USEFULL
/
drone
mirror of https://github.com/harness/drone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: [CDE-137]: made gitspace actions async and refactored orchestrator to handle errors correctly (#2194) 

* feat: [CDE-137]: made gitspace actions async and refactored orchestrator to handle errors correctly
unified-ui
Ansuman Satapathy 2024-07-10 10:44:36 +00:00 committed by Harness
parent 87157de7fa
commit 61f1bc55fe
3 changed files with 72 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
app/api/controller/gitspace/action.go
View File

@ -66,14 +66,14 @@ func (c *Controller) Action(
switch in.Action {
case enum.GitspaceActionTypeStart:
c.emitGitspaceConfigEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStart)
gitspace, err := c.startGitspace(ctx, gitspaceConfig)
gitspace, err := c.startGitspaceAction(ctx, gitspaceConfig)
if err != nil {
c.emitGitspaceConfigEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStartFailed)
}
return gitspace, err
case enum.GitspaceActionTypeStop:
c.emitGitspaceConfigEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStop)
gitspace, err := c.stopGitspace(ctx, gitspaceConfig)
gitspace, err := c.stopGitspaceAction(ctx, gitspaceConfig)
if err != nil {
c.emitGitspaceConfigEvent(ctx, gitspaceConfig, enum.GitspaceEventTypeGitspaceActionStopFailed)
}
@ -83,7 +83,10 @@ func (c *Controller) Action(
}
}
func (c *Controller) startGitspace(ctx context.Context, config *types.GitspaceConfig) (*types.GitspaceConfig, error) {
func (c *Controller) startGitspaceAction(
ctx context.Context,
config *types.GitspaceConfig,
) (*types.GitspaceConfig, error) {
savedGitspaceInstance, err := c.gitspaceInstanceStore.FindLatestByGitspaceConfigID(ctx, config.ID, config.SpaceID)
const resourceNotFoundErr = "Failed to find gitspace: resource not found"
if err != nil && err.Error() != resourceNotFoundErr { // TODO fix this
@ -109,12 +112,24 @@ func (c *Controller) startGitspace(ctx context.Context, config *types.GitspaceCo
return nil, fmt.Errorf("failed to find gitspace with config ID : %s %w", config.Identifier, err)
}
config.GitspaceInstance = newGitspaceInstance
updatedGitspace, err := c.orchestrator.StartGitspace(ctx, config)
if err != nil {
return nil, fmt.Errorf("failed to find start gitspace : %s %w", config.Identifier, err)
config.State, _ = enum.GetGitspaceStateFromInstance(newGitspaceInstance.State)
ctx2 := context.WithoutCancel(ctx)
go func() {
_, _ = c.startAsyncOperation(ctx2, config)
}()
return config, nil
}
func (c *Controller) startAsyncOperation(
ctx context.Context,
config *types.GitspaceConfig,
) (*types.GitspaceConfig, error) {
updatedGitspace, orchestrateErr := c.orchestrator.StartGitspace(ctx, config)
if err := c.gitspaceInstanceStore.Update(ctx, updatedGitspace); err != nil {
return nil, fmt.Errorf("failed to update gitspace %w %w", err, orchestrateErr)
}
if err = c.gitspaceInstanceStore.Update(ctx, updatedGitspace); err != nil {
return nil, fmt.Errorf("failed to update gitspace %w", err)
if orchestrateErr != nil {
return nil, fmt.Errorf("failed to find start gitspace : %s %w", config.Identifier, orchestrateErr)
}
config.GitspaceInstance = updatedGitspace
config.State, _ = enum.GetGitspaceStateFromInstance(updatedGitspace.State)
@ -156,8 +171,9 @@ func (c *Controller) gitspaceBusyOperation(
if config.GitspaceInstance == nil {
return config, nil
}
const timedOutInSeconds = 5
if config.GitspaceInstance.State.IsBusyStatus() &&
time.Since(time.UnixMilli(config.Updated)) <= (10*60*1000) {
time.Since(time.UnixMilli(config.GitspaceInstance.Updated)).Milliseconds() <= (timedOutInSeconds*60*1000) {
return nil, fmt.Errorf("gitspace start/stop is already pending for : %q", config.Identifier)
} else if config.GitspaceInstance.State.IsBusyStatus() {
config.GitspaceInstance.State = enum.GitspaceInstanceStateError
@ -168,7 +184,10 @@ func (c *Controller) gitspaceBusyOperation(
return config, nil
}
func (c *Controller) stopGitspace(ctx context.Context, config *types.GitspaceConfig) (*types.GitspaceConfig, error) {
func (c *Controller) stopGitspaceAction(
ctx context.Context,
config *types.GitspaceConfig,
) (*types.GitspaceConfig, error) {
savedGitspace, err := c.gitspaceInstanceStore.FindLatestByGitspaceConfigID(ctx, config.ID, config.SpaceID)
if err != nil {
return nil, fmt.Errorf("failed to find gitspace with config ID : %s %w", config.Identifier, err)
@ -186,17 +205,35 @@ func (c *Controller) stopGitspace(ctx context.Context, config *types.GitspaceCon
if err = c.gitspaceInstanceStore.Update(ctx, config.GitspaceInstance); err != nil {
return nil, fmt.Errorf("failed to update gitspace config for stopping %s %w", config.Identifier, err)
}
if updatedGitspace, stopErr := c.orchestrator.StopGitspace(ctx, config); stopErr != nil {
return nil, fmt.Errorf(
"failed to stop gitspace instance with ID %s %w", savedGitspace.Identifier, stopErr)
} else if updatedGitspace != nil {
if stopErr = c.gitspaceInstanceStore.Update(ctx, updatedGitspace); stopErr != nil {
config.State, _ = enum.GetGitspaceStateFromInstance(savedGitspace.State)
ctx2 := context.WithoutCancel(ctx)
go func() {
_, _ = c.stopAsyncOperation(ctx2, config)
}()
return config, err
}
func (c *Controller) stopAsyncOperation(
ctx context.Context,
config *types.GitspaceConfig,
) (*types.GitspaceConfig, error) {
savedGitspace := config.GitspaceInstance
updatedGitspace, orchestrateErr := c.orchestrator.StopGitspace(ctx, config)
if updatedGitspace != nil {
if err := c.gitspaceInstanceStore.Update(ctx, updatedGitspace); err != nil {
return nil, fmt.Errorf(
"unable to update the gitspace with config id %s %w", savedGitspace.Identifier, stopErr)
"unable to update the gitspace with config id %s %w %w",
savedGitspace.Identifier,
err,
orchestrateErr)
}
if orchestrateErr != nil {
return nil, fmt.Errorf(
"failed to stop gitspace instance with ID %s %w", savedGitspace.Identifier, orchestrateErr)
}
config.GitspaceInstance = updatedGitspace
config.State, _ = enum.GetGitspaceStateFromInstance(updatedGitspace.State)
}
config.GitspaceInstance = updatedGitspace
config.State, _ = enum.GetGitspaceStateFromInstance(updatedGitspace.State)
return config, nil
}

27
app/gitspace/orchestrator/orchestrator_impl.go
View File

@ -57,6 +57,8 @@ func (o orchestrator) StartGitspace(
ctx context.Context,
gitspaceConfig *types.GitspaceConfig,
) (*types.GitspaceInstance, error) {
gitspaceInstance := gitspaceConfig.GitspaceInstance
gitspaceInstance.State = enum.GitspaceInstanceStateError
devcontainerConfig, err := o.scm.DevcontainerConfig(ctx, gitspaceConfig)
if err != nil {
log.Warn().Err(err).Msg("devcontainerConfig fetch failed.")
@ -69,24 +71,19 @@ func (o orchestrator) StartGitspace(
infraProviderResource, err := o.infraProviderResourceStore.Find(ctx, gitspaceConfig.InfraProviderResourceID)
if err != nil {
return nil, fmt.Errorf("cannot get the infraProviderResource for ID %d: %w",
return gitspaceInstance, fmt.Errorf("cannot get the infraProviderResource for ID %d: %w",
gitspaceConfig.InfraProviderResourceID, err)
}
infra, err := o.infraProvisioner.Provision(ctx, infraProviderResource, gitspaceConfig)
if err != nil {
return nil, fmt.Errorf("cannot provision infrastructure for ID %d: %w",
return gitspaceInstance, fmt.Errorf("cannot provision infrastructure for ID %d: %w",
gitspaceConfig.InfraProviderResourceID, err)
}
gitspaceInstance := gitspaceConfig.GitspaceInstance
err = o.containerOrchestrator.Status(ctx, infra)
gitspaceInstance.State = enum.GitspaceInstanceStateError
if err != nil {
return gitspaceInstance, fmt.Errorf("couldn't call the agent health API: %w", err)
}
startResponse, err := o.containerOrchestrator.StartGitspace(ctx, gitspaceConfig, devcontainerConfig, infra)
if err != nil {
return gitspaceInstance, fmt.Errorf("couldn't call the agent start API: %w", err)
@ -141,6 +138,8 @@ func (o orchestrator) StopGitspace(
ctx context.Context,
gitspaceConfig *types.GitspaceConfig,
) (*types.GitspaceInstance, error) {
gitspaceInstance := gitspaceConfig.GitspaceInstance
gitspaceInstance.State = enum.GitspaceInstanceStateError
infraProviderResource, err := o.infraProviderResourceStore.Find(ctx, gitspaceConfig.InfraProviderResourceID)
if err != nil {
return nil, fmt.Errorf(
@ -149,21 +148,19 @@ func (o orchestrator) StopGitspace(
infra, err := o.infraProvisioner.Find(ctx, infraProviderResource, gitspaceConfig)
if err != nil {
return nil, fmt.Errorf("cannot find the provisioned infra: %w", err)
return gitspaceInstance, fmt.Errorf("cannot find the provisioned infra: %w", err)
}
err = o.containerOrchestrator.StopGitspace(ctx, gitspaceConfig, infra)
if err != nil {
return nil, fmt.Errorf("error stopping the Gitspace container: %w", err)
return gitspaceInstance, fmt.Errorf("error stopping the Gitspace container: %w", err)
}
_, err = o.infraProvisioner.Stop(ctx, infraProviderResource, gitspaceConfig)
if err != nil {
return nil, fmt.Errorf(
return gitspaceInstance, fmt.Errorf(
"cannot stop provisioned infrastructure with ID %d: %w", gitspaceConfig.InfraProviderResourceID, err)
}
gitspaceInstance := gitspaceConfig.GitspaceInstance
gitspaceInstance.State = enum.GitspaceInstanceStateDeleted
return gitspaceInstance, err
}
@ -173,13 +170,15 @@ func (o orchestrator) DeleteGitspace(
gitspaceConfig *types.GitspaceConfig,
) (*types.GitspaceInstance, error) {
gitspaceInstance := gitspaceConfig.GitspaceInstance
currentState := gitspaceInstance.State
gitspaceInstance.State = enum.GitspaceInstanceStateError
infraProviderResource, err := o.infraProviderResourceStore.Find(ctx, gitspaceConfig.InfraProviderResourceID)
if err != nil {
return nil, fmt.Errorf(
"cannot get the infraProviderResource with ID %d: %w", gitspaceConfig.InfraProviderResourceID, err)
}
if gitspaceInstance.State == enum.GitspaceInstanceStateRunning ||
gitspaceInstance.State == enum.GitspaceInstanceStateUnknown {
if currentState == enum.GitspaceInstanceStateRunning ||
currentState == enum.GitspaceInstanceStateUnknown {
infra, err := o.infraProvisioner.Find(ctx, infraProviderResource, gitspaceConfig)
if err != nil {
return nil, fmt.Errorf("cannot find the provisioned infra: %w", err)

5
types/enum/gitspace_state_type.go
View File

@ -42,7 +42,7 @@ const (
func GetGitspaceStateFromInstance(
instanceState GitspaceInstanceStateType) (GitspaceStateType, error) {
switch instanceState { //nolint:exhaustive
switch instanceState {
case GitspaceInstanceStateRunning:
return GitspaceStateRunning, nil
case GitspaceInstanceStateDeleted:
@ -53,6 +53,9 @@ func GetGitspaceStateFromInstance(
return GitspaceStateStopping, nil
case GitspaceInstanceStateUninitialized:
return GitspaceStateUninitialized, nil
case GitspaceInstanceStateError,
GitspaceInstanceStateUnknown:
return GitspaceStateError, nil
default:
return GitspaceStateError, fmt.Errorf("unsupported gitspace instance state %s", string(instanceState))
}