diff --git a/app/api/controller/gitspace/action.go b/app/api/controller/gitspace/action.go index d73671457..1eaa08943 100644 --- a/app/api/controller/gitspace/action.go +++ b/app/api/controller/gitspace/action.go @@ -34,7 +34,7 @@ import ( const defaultAccessKey = "Harness@123" const defaultMachineUser = "harness" -const gitspaceTimedOutInMintues = 5 +const gitspaceTimedOutInMintues = 10 type ActionInput struct { Action enum.GitspaceActionType `json:"action"` @@ -109,29 +109,61 @@ func (c *Controller) startGitspaceAction( } config.GitspaceInstance = newGitspaceInstance config.State, _ = enum.GetGitspaceStateFromInstance(newGitspaceInstance.State) - contextWithoutCancel := context.WithoutCancel(ctx) - go func() { - err := c.startAsyncOperation(contextWithoutCancel, config) - if err != nil { - c.emitGitspaceConfigEvent(contextWithoutCancel, config, enum.GitspaceEventTypeGitspaceActionStartFailed) - log.Err(err).Msg("start operation failed") - } - }() + c.submitAsyncOps(ctx, config, enum.GitspaceActionTypeStart) return nil } -func (c *Controller) startAsyncOperation( - ctx context.Context, +func (c *Controller) asyncOperation( + ctxWithTimedOut context.Context, config *types.GitspaceConfig, -) error { - updatedGitspace, orchestrateErr := c.orchestrator.StartGitspace(ctx, config) - if err := c.gitspaceInstanceStore.Update(ctx, updatedGitspace); err != nil { - return fmt.Errorf("failed to update gitspace %w %w", err, orchestrateErr) + action enum.GitspaceActionType, + errChannel chan error, +) { + var orchestrateErr error + switch action { + case enum.GitspaceActionTypeStart: + orchestrateErr = c.orchestrator.StartGitspace(ctxWithTimedOut, config) + case enum.GitspaceActionTypeStop: + orchestrateErr = c.orchestrator.StopGitspace(ctxWithTimedOut, config) } if orchestrateErr != nil { - return fmt.Errorf("failed to find start gitspace : %s %w", config.Identifier, orchestrateErr) + errChannel <- fmt.Errorf("failed to find start/stop gitspace : %s %w", config.Identifier, orchestrateErr) } - return nil + close(errChannel) +} + +func (c *Controller) submitAsyncOps( + ctx context.Context, + config *types.GitspaceConfig, + action enum.GitspaceActionType, +) { + submitCtx := context.WithoutCancel(ctx) + ttlExecuteContext, cancel := context.WithTimeout(submitCtx, gitspaceTimedOutInMintues*time.Minute) + // submit an async task with a TTL + errChannel := make(chan error) + go c.asyncOperation(ttlExecuteContext, config, action, errChannel) + // wait execution completion for the specified time or mark it as an error + var err error + go func() { + defer c.updateGitspaceInstance(submitCtx, config) + select { + case <-ttlExecuteContext.Done(): + if ttlExecuteContext.Err() != nil { + err = ttlExecuteContext.Err() + } + case err = <-errChannel: + } + if err != nil { + log.Err(err).Msgf("error during async execution for %s", config.GitspaceInstance.Identifier) + switch action { + case enum.GitspaceActionTypeStart: + c.emitGitspaceConfigEvent(ttlExecuteContext, config, enum.GitspaceEventTypeGitspaceActionStartFailed) + case enum.GitspaceActionTypeStop: + c.emitGitspaceConfigEvent(ttlExecuteContext, config, enum.GitspaceEventTypeGitspaceActionStopFailed) + } + } + cancel() + }() } func (c *Controller) createGitspaceInstance(config *types.GitspaceConfig) (*types.GitspaceInstance, error) { @@ -203,36 +235,7 @@ func (c *Controller) stopGitspaceAction( return fmt.Errorf("failed to update gitspace config for stopping %s %w", config.Identifier, err) } config.State, _ = enum.GetGitspaceStateFromInstance(savedGitspaceInstance.State) - contextWithoutCancel := context.WithoutCancel(ctx) - go func() { - err := c.stopAsyncOperation(contextWithoutCancel, config) - if err != nil { - c.emitGitspaceConfigEvent(contextWithoutCancel, config, enum.GitspaceEventTypeGitspaceActionStopFailed) - log.Err(err).Msg("stop operation failed") - } - }() - return err -} - -func (c *Controller) stopAsyncOperation( - ctx context.Context, - config *types.GitspaceConfig, -) error { - savedGitspace := config.GitspaceInstance - updatedGitspace, orchestrateErr := c.orchestrator.StopGitspace(ctx, config) - if updatedGitspace != nil { - if err := c.gitspaceInstanceStore.Update(ctx, updatedGitspace); err != nil { - return fmt.Errorf( - "unable to update the gitspace with config id %s %w %w", - savedGitspace.Identifier, - err, - orchestrateErr) - } - if orchestrateErr != nil { - return fmt.Errorf( - "failed to stop gitspace instance with ID %s %w", savedGitspace.Identifier, orchestrateErr) - } - } + c.submitAsyncOps(ctx, config, enum.GitspaceActionTypeStop) return nil } @@ -260,3 +263,14 @@ func (c *Controller) emitGitspaceConfigEvent( Timestamp: time.Now().UnixNano(), }) } + +func (c *Controller) updateGitspaceInstance( + ctx context.Context, + config *types.GitspaceConfig, +) { + err := c.gitspaceInstanceStore.Update(ctx, config.GitspaceInstance) + if err != nil { + log.Err(err).Msgf( + "failed to update gitspace instance during exec %q", config.GitspaceInstance.Identifier) + } +} diff --git a/app/gitspace/orchestrator/orchestrator.go b/app/gitspace/orchestrator/orchestrator.go index 3acce9802..186104d6a 100644 --- a/app/gitspace/orchestrator/orchestrator.go +++ b/app/gitspace/orchestrator/orchestrator.go @@ -24,22 +24,13 @@ type Orchestrator interface { // StartGitspace is responsible for all the operations necessary to create the Gitspace container. It fetches the // devcontainer.json from the code repo, provisions infra using the infra provisioner and setting up the Gitspace // through the container orchestrator. - StartGitspace( - ctx context.Context, - gitspaceConfig *types.GitspaceConfig, - ) (*types.GitspaceInstance, error) + StartGitspace(ctx context.Context, gitspaceConfig *types.GitspaceConfig) error // StopGitspace is responsible for stopping a running Gitspace. It stops the Gitspace container and unprovisions // all the infra resources which are not required to restart the Gitspace. - StopGitspace( - ctx context.Context, - gitspaceConfig *types.GitspaceConfig, - ) (*types.GitspaceInstance, error) + StopGitspace(ctx context.Context, gitspaceConfig *types.GitspaceConfig) error // DeleteGitspace is responsible for deleting a Gitspace. It stops the Gitspace container and unprovisions // all the infra resources. - DeleteGitspace( - ctx context.Context, - gitspaceConfig *types.GitspaceConfig, - ) (*types.GitspaceInstance, error) + DeleteGitspace(ctx context.Context, gitspaceConfig *types.GitspaceConfig) (*types.GitspaceInstance, error) } diff --git a/app/gitspace/orchestrator/orchestrator_impl.go b/app/gitspace/orchestrator/orchestrator_impl.go index 3f429d385..8623b54f7 100644 --- a/app/gitspace/orchestrator/orchestrator_impl.go +++ b/app/gitspace/orchestrator/orchestrator_impl.go @@ -61,7 +61,7 @@ func NewOrchestrator( func (o orchestrator) StartGitspace( ctx context.Context, gitspaceConfig *types.GitspaceConfig, -) (*types.GitspaceInstance, error) { +) error { gitspaceInstance := gitspaceConfig.GitspaceInstance gitspaceInstance.State = enum.GitspaceInstanceStateError @@ -71,8 +71,7 @@ func (o orchestrator) StartGitspace( if err != nil { o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeFetchDevcontainerFailed) - return gitspaceInstance, - fmt.Errorf("failed to fetch code repo details for gitspace config ID %d", gitspaceConfig.ID) + return fmt.Errorf("failed to fetch code repo details for gitspace config ID %d", gitspaceConfig.ID) } if devcontainerConfig == nil { @@ -84,7 +83,7 @@ func (o orchestrator) StartGitspace( infraProviderResource, err := o.infraProviderResourceStore.Find(ctx, gitspaceConfig.InfraProviderResourceID) if err != nil { - return gitspaceInstance, fmt.Errorf("cannot get the infraprovider resource for ID %d: %w", + return fmt.Errorf("cannot get the infraprovider resource for ID %d: %w", gitspaceConfig.InfraProviderResourceID, err) } @@ -94,7 +93,7 @@ func (o orchestrator) StartGitspace( if err != nil { o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraProvisioningFailed) - return gitspaceInstance, fmt.Errorf( + return fmt.Errorf( "cannot provision infrastructure for ID %d: %w", gitspaceConfig.InfraProviderResourceID, err) } @@ -107,7 +106,7 @@ func (o orchestrator) StartGitspace( if err != nil { o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentConnectFailed) - return gitspaceInstance, fmt.Errorf("couldn't call the agent health API: %w", err) + return fmt.Errorf("couldn't call the agent health API: %w", err) } o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentConnectCompleted) @@ -118,7 +117,7 @@ func (o orchestrator) StartGitspace( if err != nil { o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceCreationFailed) - return gitspaceInstance, fmt.Errorf("couldn't call the agent start API: %w", err) + return fmt.Errorf("couldn't call the agent start API: %w", err) } o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceCreationCompleted) @@ -155,25 +154,25 @@ func (o orchestrator) StartGitspace( o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStartCompleted) - return gitspaceInstance, nil + return nil } func (o orchestrator) StopGitspace( ctx context.Context, gitspaceConfig *types.GitspaceConfig, -) (*types.GitspaceInstance, error) { +) error { gitspaceInstance := gitspaceConfig.GitspaceInstance gitspaceInstance.State = enum.GitspaceInstanceStateError infraProviderResource, err := o.infraProviderResourceStore.Find(ctx, gitspaceConfig.InfraProviderResourceID) if err != nil { - return nil, fmt.Errorf( + return fmt.Errorf( "cannot get the infraProviderResource with ID %d: %w", gitspaceConfig.InfraProviderResourceID, err) } infra, err := o.infraProvisioner.Find(ctx, infraProviderResource, gitspaceConfig) if err != nil { - return gitspaceInstance, fmt.Errorf("cannot find the provisioned infra: %w", err) + return fmt.Errorf("cannot find the provisioned infra: %w", err) } o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentConnectStart) @@ -182,7 +181,7 @@ func (o orchestrator) StopGitspace( if err != nil { o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentConnectFailed) - return gitspaceConfig.GitspaceInstance, fmt.Errorf("couldn't call the agent health API: %w", err) + return fmt.Errorf("couldn't call the agent health API: %w", err) } o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentConnectCompleted) @@ -193,7 +192,7 @@ func (o orchestrator) StopGitspace( if err != nil { o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceDeletionFailed) - return gitspaceInstance, fmt.Errorf("error stopping the Gitspace container: %w", err) + return fmt.Errorf("error stopping the Gitspace container: %w", err) } o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeAgentGitspaceDeletionCompleted) @@ -204,7 +203,7 @@ func (o orchestrator) StopGitspace( if err != nil { o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeInfraUnprovisioningFailed) - return gitspaceInstance, fmt.Errorf( + return fmt.Errorf( "cannot stop provisioned infrastructure with ID %d: %w", gitspaceConfig.InfraProviderResourceID, err) } @@ -214,7 +213,7 @@ func (o orchestrator) StopGitspace( o.emitGitspaceEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStopCompleted) - return gitspaceInstance, err + return err } func (o orchestrator) DeleteGitspace(