drone/app/gitspace/orchestrator/orchestrator_trigger.go

// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package orchestrator

import (
	"context"
	"fmt"
	"time"

	events "github.com/harness/gitness/app/events/gitspace"
	"github.com/harness/gitness/app/gitspace/infrastructure"
	"github.com/harness/gitness/app/gitspace/orchestrator/container"
	"github.com/harness/gitness/app/gitspace/orchestrator/ide"
	"github.com/harness/gitness/app/gitspace/platformconnector"
	"github.com/harness/gitness/app/gitspace/scm"
	"github.com/harness/gitness/app/gitspace/secret"
	"github.com/harness/gitness/app/store"
	"github.com/harness/gitness/types"
	"github.com/harness/gitness/types/enum"

	"github.com/gotidy/ptr"
	"github.com/rs/zerolog/log"
	"golang.org/x/exp/slices"
)

const harnessUser = "harness"

type Config struct {
	DefaultBaseImage string
}

type Orchestrator struct {
	scm                          *scm.SCM
	platformConnector            platformconnector.PlatformConnector
	infraProvisioner             infrastructure.InfraProvisioner
	containerOrchestratorFactory container.Factory
	eventReporter                *events.Reporter
	config                       *Config
	ideFactory                   ide.Factory
	secretResolverFactory        *secret.ResolverFactory
	gitspaceInstanceStore        store.GitspaceInstanceStore
}

func NewOrchestrator(
	scm *scm.SCM,
	platformConnector platformconnector.PlatformConnector,
	infraProvisioner infrastructure.InfraProvisioner,
	containerOrchestratorFactory container.Factory,
	eventReporter *events.Reporter,
	config *Config,
	ideFactory ide.Factory,
	secretResolverFactory *secret.ResolverFactory,
	gitspaceInstanceStore store.GitspaceInstanceStore,
) Orchestrator {
	return Orchestrator{
		scm:                          scm,
		platformConnector:            platformConnector,
		infraProvisioner:             infraProvisioner,
		containerOrchestratorFactory: containerOrchestratorFactory,
		eventReporter:                eventReporter,
		config:                       config,
		ideFactory:                   ideFactory,
		secretResolverFactory:        secretResolverFactory,
		gitspaceInstanceStore:        gitspaceInstanceStore,
	}
}

// TriggerStartGitspace fetches the infra resources configured for the gitspace and triggers the infra provisioning.
func (o Orchestrator) TriggerStartGitspace(
	ctx context.Context,
	gitspaceConfig types.GitspaceConfig,
) *types.GitspaceError {
	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeFetchDevcontainerStart)
	scmResolvedDetails, err := o.scm.GetSCMRepoDetails(ctx, gitspaceConfig)
	if err != nil {
		o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeFetchDevcontainerFailed)
		return &types.GitspaceError{
			Error: fmt.Errorf("failed to fetch code repo details for gitspace config ID %d: %w",
				gitspaceConfig.ID, err),
			ErrorMessage: ptr.String(err.Error()),
		}
	}
	devcontainerConfig := scmResolvedDetails.DevcontainerConfig
	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeFetchDevcontainerCompleted)
	gitspaceSpecs := devcontainerConfig.Customizations.ExtractGitspaceSpec()
	connectorRefs := getConnectorRefs(gitspaceSpecs)
	if len(connectorRefs) > 0 {
		o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeFetchConnectorsDetailsStart)
		connectors, err := o.platformConnector.FetchConnectors(
			ctx, connectorRefs, gitspaceConfig.SpacePath)
		if err != nil {
			o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeFetchConnectorsDetailsFailed)
			log.Ctx(ctx).Err(err).Msgf("failed to fetch connectors for gitspace: %v",
				connectorRefs,
			)
			return &types.GitspaceError{
				Error: fmt.Errorf("failed to fetch connectors for gitspace: %v :%w",
					connectorRefs,
					err,
				),
				ErrorMessage: ptr.String(err.Error()),
			}
		}
		o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeFetchConnectorsDetailsCompleted)
		gitspaceConfig.Connectors = connectors
	}

	requiredGitspacePorts, err := o.getPortsRequiredForGitspace(gitspaceConfig, devcontainerConfig)
	if err != nil {
		err = fmt.Errorf("cannot get the ports required for gitspace during start: %w", err)
		return &types.GitspaceError{
			Error:        err,
			ErrorMessage: ptr.String(err.Error()),
		}
	}

	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraProvisioningStart)

	opts := infrastructure.InfraEventOpts{RequiredGitspacePorts: requiredGitspacePorts}
	err = o.infraProvisioner.TriggerInfraEventWithOpts(ctx, enum.InfraEventProvision, gitspaceConfig, nil, opts)
	if err != nil {
		o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraProvisioningFailed)
		return &types.GitspaceError{
			Error: fmt.Errorf("cannot trigger provision infrastructure for ID %s: %w",
				gitspaceConfig.InfraProviderResource.UID, err),
			ErrorMessage: ptr.String(err.Error()), // TODO: Fetch explicit error msg from infra provisioner.
		}
	}

	return nil
}

// TriggerStopGitspace stops the Gitspace container and triggers infra deprovisioning to deprovision
// all the infra resources which are not required to restart the Gitspace.
func (o Orchestrator) TriggerStopGitspace(
	ctx context.Context,
	gitspaceConfig types.GitspaceConfig,
) *types.GitspaceError {
	infra, err := o.getProvisionedInfra(ctx, gitspaceConfig,
		[]enum.InfraStatus{enum.InfraStatusProvisioned, enum.InfraStatusStopped})
	if err != nil {
		infraNotFoundErr := fmt.Errorf("unable to find provisioned infra while triggering stop for gitspace "+
			"instance %s: %w", gitspaceConfig.GitspaceInstance.Identifier, err)
		return &types.GitspaceError{
			Error:        infraNotFoundErr,
			ErrorMessage: ptr.String(infraNotFoundErr.Error()), // TODO: Fetch explicit error msg
		}
	}
	if gitspaceConfig.GitspaceInstance.State == enum.GitspaceInstanceStateRunning ||
		gitspaceConfig.GitspaceInstance.State == enum.GitspaceInstanceStateStopping {
		err = o.stopGitspaceContainer(ctx, gitspaceConfig, *infra)
		if err != nil {
			return &types.GitspaceError{
				Error:        err,
				ErrorMessage: ptr.String(err.Error()), // TODO: Fetch explicit error msg
			}
		}
	}

	return nil
}

func (o Orchestrator) stopGitspaceContainer(
	ctx context.Context,
	gitspaceConfig types.GitspaceConfig,
	infra types.Infrastructure,
) error {
	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentConnectStart)

	containerOrchestrator, err := o.containerOrchestratorFactory.GetContainerOrchestrator(infra.ProviderType)
	if err != nil {
		o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentConnectFailed)
		return fmt.Errorf("couldn't get the container orchestrator: %w", err)
	}

	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentConnectCompleted)

	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceStopStart)

	// NOTE: Currently we use a static identifier as the Gitspace user.
	gitspaceConfig.GitspaceUser.Identifier = harnessUser

	err = containerOrchestrator.StopGitspace(ctx, gitspaceConfig, infra)
	if err != nil {
		o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceStopFailed)

		return fmt.Errorf("error stopping the Gitspace container: %w", err)
	}

	return nil
}

func (o Orchestrator) FinishStopGitspaceContainer(
	ctx context.Context,
	gitspaceConfig types.GitspaceConfig,
	infra types.Infrastructure,
) *types.GitspaceError {
	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceStopCompleted)

	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraStopStart)

	err := o.infraProvisioner.TriggerInfraEvent(ctx, enum.InfraEventStop, gitspaceConfig, &infra)
	if err != nil {
		o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraStopFailed)
		infraStopErr := fmt.Errorf("cannot trigger stop infrastructure with ID %s: %w",
			gitspaceConfig.InfraProviderResource.UID, err)
		return &types.GitspaceError{
			Error:        infraStopErr,
			ErrorMessage: ptr.String(infraStopErr.Error()), // TODO: Fetch explicit error msg
		}
	}
	return nil
}

func (o Orchestrator) stopAndRemoveGitspaceContainer(
	ctx context.Context,
	gitspaceConfig types.GitspaceConfig,
	infra types.Infrastructure,
	canDeleteUserData bool,
) error {
	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentConnectStart)

	containerOrchestrator, err := o.containerOrchestratorFactory.GetContainerOrchestrator(infra.ProviderType)
	if err != nil {
		o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentConnectFailed)
		return fmt.Errorf("couldn't get the container orchestrator: %w", err)
	}

	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentConnectCompleted)

	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceDeletionStart)

	// NOTE: Currently we use a static identifier as the Gitspace user.
	gitspaceConfig.GitspaceUser.Identifier = harnessUser

	err = containerOrchestrator.StopAndRemoveGitspace(ctx, gitspaceConfig, infra, canDeleteUserData)
	if err != nil {
		o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceDeletionFailed)
		log.Err(err).Msgf("error stopping the Gitspace container")
	}
	return nil
}

func (o Orchestrator) FinishStopAndRemoveGitspaceContainer(
	ctx context.Context,
	gitspaceConfig types.GitspaceConfig,
	infra types.Infrastructure,
	canDeleteUserData bool,
) *types.GitspaceError {
	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceDeletionCompleted)
	if canDeleteUserData {
		o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraDeprovisioningStart)
	} else {
		o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraResetStart)
	}

	opts := infrastructure.InfraEventOpts{CanDeleteUserData: canDeleteUserData}
	err := o.infraProvisioner.TriggerInfraEventWithOpts(ctx, enum.InfraEventDeprovision, gitspaceConfig, &infra, opts)
	if err != nil {
		if canDeleteUserData {
			o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraDeprovisioningFailed)
		} else {
			o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraResetFailed)
		}
		return &types.GitspaceError{
			Error: fmt.Errorf(
				"cannot trigger deprovision infrastructure with ID %s: %w",
				gitspaceConfig.InfraProviderResource.UID,
				err,
			),
			ErrorMessage: ptr.String(err.Error()),
		}
	}
	return nil
}

// TriggerCleanupInstanceResources cleans up all the resources exclusive to gitspace instance.
func (o Orchestrator) TriggerCleanupInstanceResources(ctx context.Context, gitspaceConfig types.GitspaceConfig) error {
	infra, err := o.getProvisionedInfra(ctx, gitspaceConfig,
		[]enum.InfraStatus{
			enum.InfraStatusProvisioned,
			enum.InfraStatusStopped,
			enum.InfraStatusPending,
			enum.InfraStatusUnknown,
			enum.InfraStatusDestroyed,
		})
	if err != nil {
		return fmt.Errorf(
			"unable to find provisioned infra while triggering cleanup for gitspace instance %s: %w",
			gitspaceConfig.GitspaceInstance.Identifier, err)
	}

	if gitspaceConfig.GitspaceInstance.State != enum.GitSpaceInstanceStateCleaning {
		return fmt.Errorf("cannot trigger cleanup, expected state: %s, actual state: %s ",
			enum.GitSpaceInstanceStateCleaning,
			gitspaceConfig.GitspaceInstance.State,
		)
	}

	o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraCleanupStart)

	err = o.infraProvisioner.TriggerInfraEvent(ctx, enum.InfraEventCleanup, gitspaceConfig, infra)
	if err != nil {
		return fmt.Errorf("cannot trigger cleanup infrastructure with ID %s: %w",
			gitspaceConfig.InfraProviderResource.UID,
			err,
		)
	}

	return nil
}

// TriggerDeleteGitspace removes the Gitspace container and triggers infra deprovisioning to deprovision
// the infra resources.
// canDeleteUserData = false -> trigger deprovision of all resources except storage associated to user data.
// canDeleteUserData = true -> trigger deprovision of all resources.
func (o Orchestrator) TriggerDeleteGitspace(
	ctx context.Context,
	gitspaceConfig types.GitspaceConfig,
	canDeleteUserData bool,
) error {
	infra, err := o.getProvisionedInfra(ctx, gitspaceConfig,
		[]enum.InfraStatus{
			enum.InfraStatusProvisioned,
			enum.InfraStatusStopped,
			enum.InfraStatusDestroyed,
			enum.InfraStatusError,
			enum.InfraStatusUnknown,
		})
	if err != nil {
		return fmt.Errorf(
			"unable to find provisioned infra while triggering delete for gitspace instance %s: %w",
			gitspaceConfig.GitspaceInstance.Identifier, err)
	}
	if err = o.stopAndRemoveGitspaceContainer(ctx, gitspaceConfig, *infra, canDeleteUserData); err != nil {
		log.Warn().Msgf("error stopping and removing gitspace container: %v", err)
	}

	// TODO: Add a job for cleanup of infra if stop fails
	log.Warn().Msgf(
		"Checking and force deleting the infra if required for gitspace instance %s",
		gitspaceConfig.GitspaceInstance.Identifier,
	)
	ticker := time.NewTicker(60 * time.Second)
	timeout := time.After(15 * time.Minute)
	defer ticker.Stop()
	ch := make(chan error)

	for {
		select {
		case msg := <-ch:
			if msg == nil {
				return msg
			}
		case <-ticker.C:
			instance, err := o.gitspaceInstanceStore.Find(
				ctx,
				gitspaceConfig.GitspaceInstance.ID,
			)
			if err != nil {
				return fmt.Errorf(
					"failed to find gitspace instance %s: %w",
					gitspaceConfig.GitspaceInstance.Identifier,
					err,
				)
			}
			if instance.State == enum.GitspaceInstanceStateDeleted ||
				instance.State == enum.GitspaceInstanceStateCleaned {
				return nil
			}
		case <-timeout:
			opts := infrastructure.InfraEventOpts{CanDeleteUserData: true}
			err := o.infraProvisioner.TriggerInfraEventWithOpts(
				ctx,
				enum.InfraEventDeprovision,
				gitspaceConfig,
				infra,
				opts,
			)
			if err != nil {
				if canDeleteUserData {
					o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraDeprovisioningFailed)
				} else {
					o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraResetFailed)
				}
				return fmt.Errorf(
					"cannot trigger deprovision infrastructure with gitspace identifier %s: %w",
					gitspaceConfig.GitspaceInstance.Identifier,
					err,
				)
			}
			return nil
		}
	}
}

func (o Orchestrator) emitGitspaceEvent(
	ctx context.Context,
	config types.GitspaceConfig,
	eventType enum.GitspaceEventType,
) {
	o.eventReporter.EmitGitspaceEvent(
		ctx,
		events.GitspaceEvent,
		&events.GitspaceEventPayload{
			QueryKey:   config.Identifier,
			EntityID:   config.GitspaceInstance.ID,
			EntityType: enum.GitspaceEntityTypeGitspaceInstance,
			EventType:  eventType,
			Timestamp:  time.Now().UnixNano(),
		})
}

func (o Orchestrator) getPortsRequiredForGitspace(
	gitspaceConfig types.GitspaceConfig,
	devcontainerConfig types.DevcontainerConfig,
) ([]types.GitspacePort, error) {
	// TODO: What if the required ports in the config have deviated from when the last instance was created?
	resolvedIDE, err := o.ideFactory.GetIDE(gitspaceConfig.IDE)
	if err != nil {
		return nil, fmt.Errorf("unable to get IDE service while checking required Gitspace ports: %w", err)
	}
	idePort := resolvedIDE.Port()
	gitspacePorts := []types.GitspacePort{*idePort}
	forwardPorts := container.ExtractForwardPorts(devcontainerConfig)

	for _, port := range forwardPorts {
		gitspacePorts = append(gitspacePorts, types.GitspacePort{
			Port:     port,
			Protocol: enum.CommunicationProtocolHTTP,
		})
	}
	return gitspacePorts, nil
}

// GetGitspaceLogs fetches gitspace's start/stop logs.
func (o Orchestrator) GetGitspaceLogs(ctx context.Context, gitspaceConfig types.GitspaceConfig) (string, error) {
	if gitspaceConfig.GitspaceInstance == nil {
		return "", fmt.Errorf("gitspace %s is not setup yet, please try later", gitspaceConfig.Identifier)
	}
	infra, err := o.getProvisionedInfra(ctx, gitspaceConfig, []enum.InfraStatus{enum.InfraStatusProvisioned})
	if err != nil {
		return "", fmt.Errorf(
			"unable to find provisioned infra while fetching logs for gitspace instance %s: %w",
			gitspaceConfig.GitspaceInstance.Identifier, err)
	}

	containerOrchestrator, err := o.containerOrchestratorFactory.GetContainerOrchestrator(infra.ProviderType)
	if err != nil {
		return "", fmt.Errorf("couldn't get the container orchestrator: %w", err)
	}

	// NOTE: Currently we use a static identifier as the Gitspace user.
	gitspaceConfig.GitspaceUser.Identifier = harnessUser
	logs, err := containerOrchestrator.StreamLogs(ctx, gitspaceConfig, *infra)
	if err != nil {
		return "", fmt.Errorf("error while fetching logs from container orchestrator: %w", err)
	}

	return logs, nil
}

func (o Orchestrator) getProvisionedInfra(
	ctx context.Context,
	gitspaceConfig types.GitspaceConfig,
	expectedStatus []enum.InfraStatus,
) (*types.Infrastructure, error) {
	infra, err := o.infraProvisioner.Find(ctx, gitspaceConfig)
	if err != nil {
		return nil, fmt.Errorf("cannot find the provisioned infra: %w", err)
	}

	if !slices.Contains(expectedStatus, infra.Status) {
		return nil, fmt.Errorf("expected infra state in %v, actual state is: %s", expectedStatus, infra.Status)
	}

	if infra.Storage == "" {
		log.Warn().Msgf("couldn't find the storage for resource ID %s", gitspaceConfig.InfraProviderResource.UID)
	}

	return infra, nil
}