USEFULL
/
drone
mirror of https://github.com/harness/drone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: [CDE-545]: check during provision retry for latest instance (#3114) 

* feat: [CDE-545]: check during provision retry for latest instance
* feat: [CDE-545]: check during provision retry for latest instance
pull/3597/head
Ansuman Satapathy 2024-12-05 05:25:02 +00:00 committed by Harness
parent a5d442f289
commit a515a57299
9 changed files with 52 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
app/api/controller/gitspace/action.go
View File

@ -52,12 +52,13 @@ func (c *Controller) Action(
}
gitspaceConfig, err := c.gitspaceConfigStore.FindByIdentifier(ctx, space.ID, in.Identifier)
gitspaceConfig.SpacePath = space.Path
gitspaceConfig.SpaceID = space.ID
if err != nil {
return nil, fmt.Errorf("failed to find gitspace config: %w", err)
}
gitspaceConfig.SpacePath = space.Path
gitspaceConfig.SpaceID = space.ID
// check if it's an internal repo
if gitspaceConfig.CodeRepo.Type == enum.CodeRepoTypeGitness {
if gitspaceConfig.CodeRepo.Ref == nil {
@ -87,12 +88,12 @@ func (c *Controller) Action(
return nil, err
}
c.gitspaceSvc.EmitGitspaceConfigEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStart)
err = c.gitspaceSvc.StartGitspaceAction(ctx, gitspaceConfig)
c.gitspaceSvc.EmitGitspaceConfigEvent(ctx, *gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStart)
err = c.gitspaceSvc.StartGitspaceAction(ctx, *gitspaceConfig)
return gitspaceConfig, err
case enum.GitspaceActionTypeStop:
c.gitspaceSvc.EmitGitspaceConfigEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStop)
err = c.gitspaceSvc.StopGitspaceAction(ctx, gitspaceConfig, time.Now())
c.gitspaceSvc.EmitGitspaceConfigEvent(ctx, *gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStop)
err = c.gitspaceSvc.StopGitspaceAction(ctx, *gitspaceConfig, time.Now())
return gitspaceConfig, err
default:
return nil, fmt.Errorf("unknown action %s on gitspace : %s", string(in.Action), gitspaceConfig.Identifier)

2
app/services/gitspace/action_start.go
View File

@ -26,7 +26,7 @@ import (
func (c *Service) StartGitspaceAction(
ctx context.Context,
config *types.GitspaceConfig,
config types.GitspaceConfig,
) error {
savedGitspaceInstance, err := c.gitspaceInstanceStore.FindLatestByGitspaceConfigID(ctx, config.ID)
if err != nil && !errors.Is(err, store.ErrResourceNotFound) {

4
app/services/gitspace/action_stop.go
View File

@ -25,7 +25,7 @@ import (
func (c *Service) StopGitspaceAction(
ctx context.Context,
config *types.GitspaceConfig,
config types.GitspaceConfig,
now time.Time,
) error {
savedGitspaceInstance, err := c.gitspaceInstanceStore.FindLatestByGitspaceConfigID(ctx, config.ID)
@ -55,7 +55,7 @@ func (c *Service) StopGitspaceAction(
func (c *Service) GitspaceAutostopAction(
ctx context.Context,
config *types.GitspaceConfig,
config types.GitspaceConfig,
now time.Time,
) error {
c.EmitGitspaceConfigEvent(ctx, config, enum.GitspaceEventTypeGitspaceAutoStop)

43
app/services/gitspace/actions.go
View File

@ -39,7 +39,7 @@ const gitspaceInstanceCleaningTimedOutMins = 15
func (c *Service) gitspaceBusyOperation(
ctx context.Context,
config *types.GitspaceConfig,
config types.GitspaceConfig,
) error {
if config.GitspaceInstance == nil || !config.GitspaceInstance.State.IsBusyStatus() {
return nil
@ -50,30 +50,35 @@ func (c *Service) gitspaceBusyOperation(
return usererror.NewWithPayload(http.StatusForbidden, fmt.Sprintf(
"Last session for this gitspace is still %s", config.GitspaceInstance.State))
}
config.GitspaceInstance.State = enum.GitspaceInstanceStateError
if err := c.UpdateInstance(ctx, config.GitspaceInstance); err != nil {
return fmt.Errorf("failed to update gitspace config for %s: %w", config.Identifier, err)
}
return nil
}
func (c *Service) submitAsyncOps(
ctx context.Context,
config *types.GitspaceConfig,
config types.GitspaceConfig,
action enum.GitspaceActionType,
) {
switch action {
case enum.GitspaceActionTypeStart:
config.GitspaceInstance.State = enum.GitspaceInstanceStateStarting
case enum.GitspaceActionTypeStop:
config.GitspaceInstance.State = enum.GitspaceInstanceStateStopping
}
if updateErr := c.UpdateInstance(ctx, config.GitspaceInstance); updateErr != nil {
log.Err(updateErr).Msgf(
"failed to update gitspace instance during exec %s", config.GitspaceInstance.Identifier)
}
errChannel := make(chan *types.GitspaceError)
submitCtx := context.WithoutCancel(ctx)
gitspaceTimedOutInMins := time.Duration(c.config.Gitspace.ProvisionTimeoutInMins) * time.Minute
gitspaceTimedOutInMins := time.Duration(c.config.Gitspace.InfraTimeoutInMins) * time.Minute
ttlExecuteContext, cancel := context.WithTimeout(submitCtx, gitspaceTimedOutInMins)
go c.asyncOperation(ttlExecuteContext, *config, action, errChannel)
go c.triggerOrchestrator(ttlExecuteContext, config, action, errChannel)
var err *types.GitspaceError
go func() {
select {
case <-ttlExecuteContext.Done():
@ -106,35 +111,21 @@ func (c *Service) submitAsyncOps(
}()
}
func (c *Service) asyncOperation(
func (c *Service) triggerOrchestrator(
ctxWithTimedOut context.Context,
config types.GitspaceConfig,
action enum.GitspaceActionType,
errChannel chan *types.GitspaceError,
) {
defer close(errChannel)
var orchestrateErr *types.GitspaceError
switch action {
case enum.GitspaceActionTypeStart:
config.GitspaceInstance.State = enum.GitspaceInstanceStateStarting
err := c.UpdateInstance(ctxWithTimedOut, config.GitspaceInstance)
if err != nil {
log.Err(err).Msgf(
"failed to update gitspace instance during exec %s", config.GitspaceInstance.Identifier)
}
orchestrateErr = c.orchestrator.TriggerStartGitspace(ctxWithTimedOut, config)
case enum.GitspaceActionTypeStop:
config.GitspaceInstance.State = enum.GitspaceInstanceStateStopping
err := c.UpdateInstance(ctxWithTimedOut, config.GitspaceInstance)
if err != nil {
log.Err(err).Msgf(
"failed to update gitspace instance during exec %s", config.GitspaceInstance.Identifier)
}
orchestrateErr = c.orchestrator.TriggerStopGitspace(ctxWithTimedOut, config)
}
if orchestrateErr != nil {
orchestrateErr.Error =
fmt.Errorf("failed to start/stop gitspace: %s %w", config.Identifier, orchestrateErr.Error)
@ -142,7 +133,7 @@ func (c *Service) asyncOperation(
}
}
func (c *Service) buildGitspaceInstance(config *types.GitspaceConfig) (*types.GitspaceInstance, error) {
func (c *Service) buildGitspaceInstance(config types.GitspaceConfig) (*types.GitspaceInstance, error) {
gitspaceMachineUser := defaultMachineUser
now := time.Now().UnixMilli()
suffixUID, err := gonanoid.Generate(AllowedUIDAlphabet, 6)
@ -177,7 +168,7 @@ func (c *Service) buildGitspaceInstance(config *types.GitspaceConfig) (*types.Gi
func (c *Service) EmitGitspaceConfigEvent(
ctx context.Context,
config *types.GitspaceConfig,
config types.GitspaceConfig,
eventType enum.GitspaceEventType,
) {
c.eventReporter.EmitGitspaceEvent(ctx, events.GitspaceEvent, &events.GitspaceEventPayload{

31
app/services/gitspaceinfraevent/handler.go
View File

@ -28,14 +28,15 @@ import (
"github.com/rs/zerolog/log"
)
func (s *Service) handleGitspaceInfraEvent(
func (s *Service) handleGitspaceInfraResumeEvent(
ctx context.Context,
event *events.Event[*gitspaceInfraEvents.GitspaceInfraEventPayload],
) error {
payload := event.Payload
ctxWithTimedOut, cancel := context.WithTimeout(ctx, time.Duration(s.config.TimeoutInMins)*time.Minute)
defer cancel()
config, fetchErr := s.getConfig(
ctx, payload.Infra.SpacePath, payload.Infra.GitspaceConfigIdentifier)
ctxWithTimedOut, payload.Infra.SpacePath, payload.Infra.GitspaceConfigIdentifier)
if fetchErr != nil {
return fetchErr
}
@ -43,7 +44,7 @@ func (s *Service) handleGitspaceInfraEvent(
instance := config.GitspaceInstance
if payload.Infra.GitspaceInstanceIdentifier != "" {
gitspaceInstance, err := s.gitspaceSvc.FindInstanceByIdentifier(
ctx,
ctxWithTimedOut,
payload.Infra.GitspaceInstanceIdentifier,
payload.Infra.SpacePath,
)
@ -56,7 +57,7 @@ func (s *Service) handleGitspaceInfraEvent(
}
defer func() {
updateErr := s.gitspaceSvc.UpdateInstance(ctx, instance)
updateErr := s.gitspaceSvc.UpdateInstance(ctxWithTimedOut, instance)
if updateErr != nil {
log.Err(updateErr).Msgf("failed to update gitspace instance")
}
@ -66,9 +67,12 @@ func (s *Service) handleGitspaceInfraEvent(
switch payload.Type {
case enum.InfraEventProvision:
updatedInstance, resumeStartErr := s.orchestrator.ResumeStartGitspace(ctx, *config, payload.Infra)
if config.GitspaceInstance.Identifier != payload.Infra.GitspaceInstanceIdentifier {
return fmt.Errorf("gitspace instance is not latest, stopping provisioning")
}
updatedInstance, resumeStartErr := s.orchestrator.ResumeStartGitspace(ctxWithTimedOut, *config, payload.Infra)
if resumeStartErr != nil {
s.emitGitspaceConfigEvent(ctx, config, enum.GitspaceEventTypeGitspaceActionStartFailed)
s.emitGitspaceConfigEvent(ctxWithTimedOut, config, enum.GitspaceEventTypeGitspaceActionStartFailed)
updatedInstance.ErrorMessage = resumeStartErr.ErrorMessage
err = fmt.Errorf("failed to resume start gitspace: %w", resumeStartErr.Error)
}
@ -76,9 +80,9 @@ func (s *Service) handleGitspaceInfraEvent(
instance = &updatedInstance
case enum.InfraEventStop:
instanceState, resumeStopErr := s.orchestrator.ResumeStopGitspace(ctx, *config, payload.Infra)
instanceState, resumeStopErr := s.orchestrator.ResumeStopGitspace(ctxWithTimedOut, *config, payload.Infra)
if resumeStopErr != nil {
s.emitGitspaceConfigEvent(ctx, config, enum.GitspaceEventTypeGitspaceActionStopFailed)
s.emitGitspaceConfigEvent(ctxWithTimedOut, config, enum.GitspaceEventTypeGitspaceActionStopFailed)
instance.ErrorMessage = resumeStopErr.ErrorMessage
err = fmt.Errorf("failed to resume stop gitspace: %w", resumeStopErr.Error)
}
@ -86,12 +90,12 @@ func (s *Service) handleGitspaceInfraEvent(
instance.State = instanceState
case enum.InfraEventDeprovision:
instanceState, resumeDeleteErr := s.orchestrator.ResumeDeleteGitspace(ctx, *config, payload.Infra)
instanceState, resumeDeleteErr := s.orchestrator.ResumeDeleteGitspace(ctxWithTimedOut, *config, payload.Infra)
if resumeDeleteErr != nil {
err = fmt.Errorf("failed to resume delete gitspace: %w", resumeDeleteErr)
} else if config.IsMarkedForDeletion {
config.IsDeleted = true
updateErr := s.gitspaceSvc.UpdateConfig(ctx, config)
updateErr := s.gitspaceSvc.UpdateConfig(ctxWithTimedOut, config)
if updateErr != nil {
err = fmt.Errorf("failed to delete gitspace config with ID: %s %w", config.Identifier, updateErr)
}
@ -99,9 +103,10 @@ func (s *Service) handleGitspaceInfraEvent(
instance.State = instanceState
case enum.InfraEventCleanup:
instanceState, resumeCleanupErr := s.orchestrator.ResumeCleanupInstanceResources(ctx, *config, payload.Infra)
instanceState, resumeCleanupErr := s.orchestrator.ResumeCleanupInstanceResources(
ctxWithTimedOut, *config, payload.Infra)
if resumeCleanupErr != nil {
s.emitGitspaceConfigEvent(ctx, config, enum.GitspaceEventTypeInfraCleanupFailed)
s.emitGitspaceConfigEvent(ctxWithTimedOut, config, enum.GitspaceEventTypeInfraCleanupFailed)
err = fmt.Errorf("failed to resume cleanup gitspace: %w", resumeCleanupErr)
}

2
app/services/gitspaceinfraevent/service.go
View File

@ -65,7 +65,7 @@ func NewService(
stream.WithMaxRetries(config.MaxRetries),
))
_ = r.RegisterGitspaceInfraEvent(service.handleGitspaceInfraEvent)
_ = r.RegisterGitspaceInfraEvent(service.handleGitspaceInfraResumeEvent)
return nil
})

3
infraprovider/docker_provider.go
View File

@ -55,7 +55,7 @@ func (d DockerProvider) Provision(
spaceID int64,
spacePath string,
gitspaceConfigIdentifier string,
_ string,
gitspaceInstanceIdentifier string,
_ int,
requiredGitspacePorts []types.GitspacePort,
inputParameters []types.InfraProviderParameter,
@ -83,6 +83,7 @@ func (d DockerProvider) Provision(
infrastructure.SpaceID = spaceID
infrastructure.SpacePath = spacePath
infrastructure.GitspaceConfigIdentifier = gitspaceConfigIdentifier
infrastructure.GitspaceInstanceIdentifier = gitspaceInstanceIdentifier
storageName, err := d.createNamedVolume(ctx, spacePath, gitspaceConfigIdentifier, dockerClient)
if err != nil {

2
types/config.go
View File

@ -423,7 +423,7 @@ type Config struct {
AgentPort int `envconfig:"GITNESS_GITSPACE_AGENT_PORT" default:"8083"`
ProvisionTimeoutInMins int `envconfig:"GITNESS_PROVISION_TIMEOUT_IN_MINS" default:"60"`
InfraTimeoutInMins int `envconfig:"GITNESS_INFRA_TIMEOUT_IN_MINS" default:"60"`
BusyActionInMins int `envconfig:"GITNESS_BUSY_ACTION_IN_MINS" default:"15"`

6
types/gitspace.go
View File

@ -122,13 +122,13 @@ func (g *GitspaceInstance) GetGitspaceState() (enum.GitspaceStateType, error) {
enum.GitspaceInstanceStateUnknown:
return enum.GitspaceStateError, nil
case enum.GitspaceInstanceStateStarting:
if g.LastUsed != nil && lastUpdateTimeExceeded(*g.LastUsed) {
if g.LastUsed != nil && lastUpdateTimeExceeds10Mins(*g.LastUsed) {
return enum.GitspaceStateError, nil
}
return enum.GitspaceStateStarting, nil
case enum.GitspaceInstanceStateStopping,
enum.GitSpaceInstanceStateCleaning:
if g.ActiveTimeEnded != nil && lastUpdateTimeExceeded(*g.ActiveTimeEnded) {
if g.ActiveTimeEnded != nil && lastUpdateTimeExceeds10Mins(*g.ActiveTimeEnded) {
return enum.GitspaceStateError, nil
}
return enum.GitspaceStateStopping, nil
@ -139,7 +139,7 @@ func (g *GitspaceInstance) GetGitspaceState() (enum.GitspaceStateType, error) {
}
}
func lastUpdateTimeExceeded(lastUpdateTime int64) bool {
func lastUpdateTimeExceeds10Mins(lastUpdateTime int64) bool {
duration := time.Minute * 10
return time.Since(time.UnixMilli(lastUpdateTime)) > duration
}