From a515a5729943279c9b02c87627cbfa674aa905ee Mon Sep 17 00:00:00 2001 From: Ansuman Satapathy Date: Thu, 5 Dec 2024 05:25:02 +0000 Subject: [PATCH] feat: [CDE-545]: check during provision retry for latest instance (#3114) * feat: [CDE-545]: check during provision retry for latest instance * feat: [CDE-545]: check during provision retry for latest instance --- app/api/controller/gitspace/action.go | 13 ++++--- app/services/gitspace/action_start.go | 2 +- app/services/gitspace/action_stop.go | 4 +- app/services/gitspace/actions.go | 43 +++++++++------------- app/services/gitspaceinfraevent/handler.go | 31 +++++++++------- app/services/gitspaceinfraevent/service.go | 2 +- infraprovider/docker_provider.go | 3 +- types/config.go | 2 +- types/gitspace.go | 6 +-- 9 files changed, 52 insertions(+), 54 deletions(-) diff --git a/app/api/controller/gitspace/action.go b/app/api/controller/gitspace/action.go index 86a9e6980..3f1ac7f97 100644 --- a/app/api/controller/gitspace/action.go +++ b/app/api/controller/gitspace/action.go @@ -52,12 +52,13 @@ func (c *Controller) Action( } gitspaceConfig, err := c.gitspaceConfigStore.FindByIdentifier(ctx, space.ID, in.Identifier) - gitspaceConfig.SpacePath = space.Path - gitspaceConfig.SpaceID = space.ID if err != nil { return nil, fmt.Errorf("failed to find gitspace config: %w", err) } + gitspaceConfig.SpacePath = space.Path + gitspaceConfig.SpaceID = space.ID + // check if it's an internal repo if gitspaceConfig.CodeRepo.Type == enum.CodeRepoTypeGitness { if gitspaceConfig.CodeRepo.Ref == nil { @@ -87,12 +88,12 @@ func (c *Controller) Action( return nil, err } - c.gitspaceSvc.EmitGitspaceConfigEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStart) - err = c.gitspaceSvc.StartGitspaceAction(ctx, gitspaceConfig) + c.gitspaceSvc.EmitGitspaceConfigEvent(ctx, *gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStart) + err = c.gitspaceSvc.StartGitspaceAction(ctx, *gitspaceConfig) return gitspaceConfig, err case enum.GitspaceActionTypeStop: - c.gitspaceSvc.EmitGitspaceConfigEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStop) - err = c.gitspaceSvc.StopGitspaceAction(ctx, gitspaceConfig, time.Now()) + c.gitspaceSvc.EmitGitspaceConfigEvent(ctx, *gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStop) + err = c.gitspaceSvc.StopGitspaceAction(ctx, *gitspaceConfig, time.Now()) return gitspaceConfig, err default: return nil, fmt.Errorf("unknown action %s on gitspace : %s", string(in.Action), gitspaceConfig.Identifier) diff --git a/app/services/gitspace/action_start.go b/app/services/gitspace/action_start.go index d9d91da2c..5c3d29744 100644 --- a/app/services/gitspace/action_start.go +++ b/app/services/gitspace/action_start.go @@ -26,7 +26,7 @@ import ( func (c *Service) StartGitspaceAction( ctx context.Context, - config *types.GitspaceConfig, + config types.GitspaceConfig, ) error { savedGitspaceInstance, err := c.gitspaceInstanceStore.FindLatestByGitspaceConfigID(ctx, config.ID) if err != nil && !errors.Is(err, store.ErrResourceNotFound) { diff --git a/app/services/gitspace/action_stop.go b/app/services/gitspace/action_stop.go index ec20bc22d..8eaa6190a 100644 --- a/app/services/gitspace/action_stop.go +++ b/app/services/gitspace/action_stop.go @@ -25,7 +25,7 @@ import ( func (c *Service) StopGitspaceAction( ctx context.Context, - config *types.GitspaceConfig, + config types.GitspaceConfig, now time.Time, ) error { savedGitspaceInstance, err := c.gitspaceInstanceStore.FindLatestByGitspaceConfigID(ctx, config.ID) @@ -55,7 +55,7 @@ func (c *Service) StopGitspaceAction( func (c *Service) GitspaceAutostopAction( ctx context.Context, - config *types.GitspaceConfig, + config types.GitspaceConfig, now time.Time, ) error { c.EmitGitspaceConfigEvent(ctx, config, enum.GitspaceEventTypeGitspaceAutoStop) diff --git a/app/services/gitspace/actions.go b/app/services/gitspace/actions.go index 6d89521f4..77b8e0a2c 100644 --- a/app/services/gitspace/actions.go +++ b/app/services/gitspace/actions.go @@ -39,7 +39,7 @@ const gitspaceInstanceCleaningTimedOutMins = 15 func (c *Service) gitspaceBusyOperation( ctx context.Context, - config *types.GitspaceConfig, + config types.GitspaceConfig, ) error { if config.GitspaceInstance == nil || !config.GitspaceInstance.State.IsBusyStatus() { return nil @@ -50,30 +50,35 @@ func (c *Service) gitspaceBusyOperation( return usererror.NewWithPayload(http.StatusForbidden, fmt.Sprintf( "Last session for this gitspace is still %s", config.GitspaceInstance.State)) } - config.GitspaceInstance.State = enum.GitspaceInstanceStateError if err := c.UpdateInstance(ctx, config.GitspaceInstance); err != nil { return fmt.Errorf("failed to update gitspace config for %s: %w", config.Identifier, err) } - return nil } func (c *Service) submitAsyncOps( ctx context.Context, - config *types.GitspaceConfig, + config types.GitspaceConfig, action enum.GitspaceActionType, ) { + switch action { + case enum.GitspaceActionTypeStart: + config.GitspaceInstance.State = enum.GitspaceInstanceStateStarting + case enum.GitspaceActionTypeStop: + config.GitspaceInstance.State = enum.GitspaceInstanceStateStopping + } + if updateErr := c.UpdateInstance(ctx, config.GitspaceInstance); updateErr != nil { + log.Err(updateErr).Msgf( + "failed to update gitspace instance during exec %s", config.GitspaceInstance.Identifier) + } errChannel := make(chan *types.GitspaceError) - submitCtx := context.WithoutCancel(ctx) - gitspaceTimedOutInMins := time.Duration(c.config.Gitspace.ProvisionTimeoutInMins) * time.Minute + gitspaceTimedOutInMins := time.Duration(c.config.Gitspace.InfraTimeoutInMins) * time.Minute ttlExecuteContext, cancel := context.WithTimeout(submitCtx, gitspaceTimedOutInMins) - go c.asyncOperation(ttlExecuteContext, *config, action, errChannel) - + go c.triggerOrchestrator(ttlExecuteContext, config, action, errChannel) var err *types.GitspaceError - go func() { select { case <-ttlExecuteContext.Done(): @@ -106,35 +111,21 @@ func (c *Service) submitAsyncOps( }() } -func (c *Service) asyncOperation( +func (c *Service) triggerOrchestrator( ctxWithTimedOut context.Context, config types.GitspaceConfig, action enum.GitspaceActionType, errChannel chan *types.GitspaceError, ) { defer close(errChannel) - var orchestrateErr *types.GitspaceError switch action { case enum.GitspaceActionTypeStart: - config.GitspaceInstance.State = enum.GitspaceInstanceStateStarting - err := c.UpdateInstance(ctxWithTimedOut, config.GitspaceInstance) - if err != nil { - log.Err(err).Msgf( - "failed to update gitspace instance during exec %s", config.GitspaceInstance.Identifier) - } orchestrateErr = c.orchestrator.TriggerStartGitspace(ctxWithTimedOut, config) case enum.GitspaceActionTypeStop: - config.GitspaceInstance.State = enum.GitspaceInstanceStateStopping - err := c.UpdateInstance(ctxWithTimedOut, config.GitspaceInstance) - if err != nil { - log.Err(err).Msgf( - "failed to update gitspace instance during exec %s", config.GitspaceInstance.Identifier) - } orchestrateErr = c.orchestrator.TriggerStopGitspace(ctxWithTimedOut, config) } - if orchestrateErr != nil { orchestrateErr.Error = fmt.Errorf("failed to start/stop gitspace: %s %w", config.Identifier, orchestrateErr.Error) @@ -142,7 +133,7 @@ func (c *Service) asyncOperation( } } -func (c *Service) buildGitspaceInstance(config *types.GitspaceConfig) (*types.GitspaceInstance, error) { +func (c *Service) buildGitspaceInstance(config types.GitspaceConfig) (*types.GitspaceInstance, error) { gitspaceMachineUser := defaultMachineUser now := time.Now().UnixMilli() suffixUID, err := gonanoid.Generate(AllowedUIDAlphabet, 6) @@ -177,7 +168,7 @@ func (c *Service) buildGitspaceInstance(config *types.GitspaceConfig) (*types.Gi func (c *Service) EmitGitspaceConfigEvent( ctx context.Context, - config *types.GitspaceConfig, + config types.GitspaceConfig, eventType enum.GitspaceEventType, ) { c.eventReporter.EmitGitspaceEvent(ctx, events.GitspaceEvent, &events.GitspaceEventPayload{ diff --git a/app/services/gitspaceinfraevent/handler.go b/app/services/gitspaceinfraevent/handler.go index 726a47d1d..50a6ead76 100644 --- a/app/services/gitspaceinfraevent/handler.go +++ b/app/services/gitspaceinfraevent/handler.go @@ -28,14 +28,15 @@ import ( "github.com/rs/zerolog/log" ) -func (s *Service) handleGitspaceInfraEvent( +func (s *Service) handleGitspaceInfraResumeEvent( ctx context.Context, event *events.Event[*gitspaceInfraEvents.GitspaceInfraEventPayload], ) error { payload := event.Payload - + ctxWithTimedOut, cancel := context.WithTimeout(ctx, time.Duration(s.config.TimeoutInMins)*time.Minute) + defer cancel() config, fetchErr := s.getConfig( - ctx, payload.Infra.SpacePath, payload.Infra.GitspaceConfigIdentifier) + ctxWithTimedOut, payload.Infra.SpacePath, payload.Infra.GitspaceConfigIdentifier) if fetchErr != nil { return fetchErr } @@ -43,7 +44,7 @@ func (s *Service) handleGitspaceInfraEvent( instance := config.GitspaceInstance if payload.Infra.GitspaceInstanceIdentifier != "" { gitspaceInstance, err := s.gitspaceSvc.FindInstanceByIdentifier( - ctx, + ctxWithTimedOut, payload.Infra.GitspaceInstanceIdentifier, payload.Infra.SpacePath, ) @@ -56,7 +57,7 @@ func (s *Service) handleGitspaceInfraEvent( } defer func() { - updateErr := s.gitspaceSvc.UpdateInstance(ctx, instance) + updateErr := s.gitspaceSvc.UpdateInstance(ctxWithTimedOut, instance) if updateErr != nil { log.Err(updateErr).Msgf("failed to update gitspace instance") } @@ -66,9 +67,12 @@ func (s *Service) handleGitspaceInfraEvent( switch payload.Type { case enum.InfraEventProvision: - updatedInstance, resumeStartErr := s.orchestrator.ResumeStartGitspace(ctx, *config, payload.Infra) + if config.GitspaceInstance.Identifier != payload.Infra.GitspaceInstanceIdentifier { + return fmt.Errorf("gitspace instance is not latest, stopping provisioning") + } + updatedInstance, resumeStartErr := s.orchestrator.ResumeStartGitspace(ctxWithTimedOut, *config, payload.Infra) if resumeStartErr != nil { - s.emitGitspaceConfigEvent(ctx, config, enum.GitspaceEventTypeGitspaceActionStartFailed) + s.emitGitspaceConfigEvent(ctxWithTimedOut, config, enum.GitspaceEventTypeGitspaceActionStartFailed) updatedInstance.ErrorMessage = resumeStartErr.ErrorMessage err = fmt.Errorf("failed to resume start gitspace: %w", resumeStartErr.Error) } @@ -76,9 +80,9 @@ func (s *Service) handleGitspaceInfraEvent( instance = &updatedInstance case enum.InfraEventStop: - instanceState, resumeStopErr := s.orchestrator.ResumeStopGitspace(ctx, *config, payload.Infra) + instanceState, resumeStopErr := s.orchestrator.ResumeStopGitspace(ctxWithTimedOut, *config, payload.Infra) if resumeStopErr != nil { - s.emitGitspaceConfigEvent(ctx, config, enum.GitspaceEventTypeGitspaceActionStopFailed) + s.emitGitspaceConfigEvent(ctxWithTimedOut, config, enum.GitspaceEventTypeGitspaceActionStopFailed) instance.ErrorMessage = resumeStopErr.ErrorMessage err = fmt.Errorf("failed to resume stop gitspace: %w", resumeStopErr.Error) } @@ -86,12 +90,12 @@ func (s *Service) handleGitspaceInfraEvent( instance.State = instanceState case enum.InfraEventDeprovision: - instanceState, resumeDeleteErr := s.orchestrator.ResumeDeleteGitspace(ctx, *config, payload.Infra) + instanceState, resumeDeleteErr := s.orchestrator.ResumeDeleteGitspace(ctxWithTimedOut, *config, payload.Infra) if resumeDeleteErr != nil { err = fmt.Errorf("failed to resume delete gitspace: %w", resumeDeleteErr) } else if config.IsMarkedForDeletion { config.IsDeleted = true - updateErr := s.gitspaceSvc.UpdateConfig(ctx, config) + updateErr := s.gitspaceSvc.UpdateConfig(ctxWithTimedOut, config) if updateErr != nil { err = fmt.Errorf("failed to delete gitspace config with ID: %s %w", config.Identifier, updateErr) } @@ -99,9 +103,10 @@ func (s *Service) handleGitspaceInfraEvent( instance.State = instanceState case enum.InfraEventCleanup: - instanceState, resumeCleanupErr := s.orchestrator.ResumeCleanupInstanceResources(ctx, *config, payload.Infra) + instanceState, resumeCleanupErr := s.orchestrator.ResumeCleanupInstanceResources( + ctxWithTimedOut, *config, payload.Infra) if resumeCleanupErr != nil { - s.emitGitspaceConfigEvent(ctx, config, enum.GitspaceEventTypeInfraCleanupFailed) + s.emitGitspaceConfigEvent(ctxWithTimedOut, config, enum.GitspaceEventTypeInfraCleanupFailed) err = fmt.Errorf("failed to resume cleanup gitspace: %w", resumeCleanupErr) } diff --git a/app/services/gitspaceinfraevent/service.go b/app/services/gitspaceinfraevent/service.go index d47bcf968..0ee216c56 100644 --- a/app/services/gitspaceinfraevent/service.go +++ b/app/services/gitspaceinfraevent/service.go @@ -65,7 +65,7 @@ func NewService( stream.WithMaxRetries(config.MaxRetries), )) - _ = r.RegisterGitspaceInfraEvent(service.handleGitspaceInfraEvent) + _ = r.RegisterGitspaceInfraEvent(service.handleGitspaceInfraResumeEvent) return nil }) diff --git a/infraprovider/docker_provider.go b/infraprovider/docker_provider.go index 3d05c87c1..00ed23b6b 100644 --- a/infraprovider/docker_provider.go +++ b/infraprovider/docker_provider.go @@ -55,7 +55,7 @@ func (d DockerProvider) Provision( spaceID int64, spacePath string, gitspaceConfigIdentifier string, - _ string, + gitspaceInstanceIdentifier string, _ int, requiredGitspacePorts []types.GitspacePort, inputParameters []types.InfraProviderParameter, @@ -83,6 +83,7 @@ func (d DockerProvider) Provision( infrastructure.SpaceID = spaceID infrastructure.SpacePath = spacePath infrastructure.GitspaceConfigIdentifier = gitspaceConfigIdentifier + infrastructure.GitspaceInstanceIdentifier = gitspaceInstanceIdentifier storageName, err := d.createNamedVolume(ctx, spacePath, gitspaceConfigIdentifier, dockerClient) if err != nil { diff --git a/types/config.go b/types/config.go index 9c9cb09ad..1501e544f 100644 --- a/types/config.go +++ b/types/config.go @@ -423,7 +423,7 @@ type Config struct { AgentPort int `envconfig:"GITNESS_GITSPACE_AGENT_PORT" default:"8083"` - ProvisionTimeoutInMins int `envconfig:"GITNESS_PROVISION_TIMEOUT_IN_MINS" default:"60"` + InfraTimeoutInMins int `envconfig:"GITNESS_INFRA_TIMEOUT_IN_MINS" default:"60"` BusyActionInMins int `envconfig:"GITNESS_BUSY_ACTION_IN_MINS" default:"15"` diff --git a/types/gitspace.go b/types/gitspace.go index 04cf8bcfd..937551cc5 100644 --- a/types/gitspace.go +++ b/types/gitspace.go @@ -122,13 +122,13 @@ func (g *GitspaceInstance) GetGitspaceState() (enum.GitspaceStateType, error) { enum.GitspaceInstanceStateUnknown: return enum.GitspaceStateError, nil case enum.GitspaceInstanceStateStarting: - if g.LastUsed != nil && lastUpdateTimeExceeded(*g.LastUsed) { + if g.LastUsed != nil && lastUpdateTimeExceeds10Mins(*g.LastUsed) { return enum.GitspaceStateError, nil } return enum.GitspaceStateStarting, nil case enum.GitspaceInstanceStateStopping, enum.GitSpaceInstanceStateCleaning: - if g.ActiveTimeEnded != nil && lastUpdateTimeExceeded(*g.ActiveTimeEnded) { + if g.ActiveTimeEnded != nil && lastUpdateTimeExceeds10Mins(*g.ActiveTimeEnded) { return enum.GitspaceStateError, nil } return enum.GitspaceStateStopping, nil @@ -139,7 +139,7 @@ func (g *GitspaceInstance) GetGitspaceState() (enum.GitspaceStateType, error) { } } -func lastUpdateTimeExceeded(lastUpdateTime int64) bool { +func lastUpdateTimeExceeds10Mins(lastUpdateTime int64) bool { duration := time.Minute * 10 return time.Since(time.UnixMilli(lastUpdateTime)) > duration }