From a1c91d1aa6952714bf92117aae59d1fac0939009 Mon Sep 17 00:00:00 2001 From: Ansuman Satapathy Date: Mon, 16 Dec 2024 06:28:45 +0000 Subject: [PATCH] fix: [CDE-555]: run lifecycle commands in paralell and allow object types for it (#3161) * fix: [CDE-555]: run commands in parallel * fix: [CDE-555]: run commands in parallel * fix: [CDE-555]: run commands in parallel * fix: [CDE-555]: run commands in parallel * fix: [CDE-555]: run commands in parallel * fix: [CDE-555]: run commands in paralell * fix: [CDE-555]: run commands in paralell * fix: [CDE-555]: update parsing and add discriminator * fix: [CDE-555]: update parsing and add discriminator --- .../container/devcontainer_step_utils.go | 49 +++++++-- types/devcontainer_config.go | 99 ++++++++++--------- 2 files changed, 94 insertions(+), 54 deletions(-) diff --git a/app/gitspace/orchestrator/container/devcontainer_step_utils.go b/app/gitspace/orchestrator/container/devcontainer_step_utils.go index db5a51ab0..c6c07d096 100644 --- a/app/gitspace/orchestrator/container/devcontainer_step_utils.go +++ b/app/gitspace/orchestrator/container/devcontainer_step_utils.go @@ -17,6 +17,7 @@ package container import ( "context" "fmt" + "sync" "github.com/harness/gitness/app/gitspace/orchestrator/common" "github.com/harness/gitness/app/gitspace/orchestrator/devcontainer" @@ -53,6 +54,7 @@ func ValidateSupportedOS( return nil } +// ExecuteLifecycleCommands executes commands in parallel, logs with numbers, and prefixes all logs. func ExecuteLifecycleCommands( ctx context.Context, exec devcontainer.Exec, @@ -65,21 +67,48 @@ func ExecuteLifecycleCommands( gitspaceLogger.Info(fmt.Sprintf("No %s commands provided, skipping execution", actionType)) return nil } - for _, command := range commands { - gitspaceLogger.Info(fmt.Sprintf("Executing %s command: %s", actionType, command)) - gitspaceLogger.Info(fmt.Sprintf("%s command execution output...", actionType)) + gitspaceLogger.Info(fmt.Sprintf("Executing %s commands: %v", actionType, commands)) - exec.DefaultWorkingDir = codeRepoDir - err := common.ExecuteCommandInHomeDirAndLog(ctx, &exec, command, false, gitspaceLogger, true) - if err != nil { - return logStreamWrapError( - gitspaceLogger, fmt.Sprintf("Error while executing %s command: %s", actionType, command), err) - } - gitspaceLogger.Info(fmt.Sprintf("Completed execution %s command: %s", actionType, command)) + // Create a WaitGroup to wait for all goroutines to finish. + var wg sync.WaitGroup + + // Iterate over commands and execute them in parallel using goroutines. + for index, command := range commands { + // Increment the WaitGroup counter. + wg.Add(1) + + // Execute each command in a new goroutine. + go func(index int, command string) { + // Decrement the WaitGroup counter when the goroutine finishes. + defer wg.Done() + + // Number the command in the logs and prefix all logs. + commandNumber := index + 1 // Starting from 1 for numbering + logPrefix := fmt.Sprintf("Command #%d - ", commandNumber) + + // Log command execution details. + gitspaceLogger.Info(fmt.Sprintf("%sExecuting %s command: %s", logPrefix, actionType, command)) + exec.DefaultWorkingDir = codeRepoDir + err := common.ExecuteCommandInHomeDirAndLog(ctx, &exec, command, false, gitspaceLogger, true) + if err != nil { + // Log the error if there is any issue with executing the command. + _ = logStreamWrapError(gitspaceLogger, fmt.Sprintf("%sError while executing %s command: %s", + logPrefix, actionType, command), err) + return + } + + // Log completion of the command execution. + gitspaceLogger.Info(fmt.Sprintf( + "%sCompleted execution %s command: %s", logPrefix, actionType, command)) + }(index, command) } + // Wait for all goroutines to finish. + wg.Wait() + return nil } + func CloneCode( ctx context.Context, exec *devcontainer.Exec, diff --git a/types/devcontainer_config.go b/types/devcontainer_config.go index 4ddb5efa1..f4151dd16 100644 --- a/types/devcontainer_config.go +++ b/types/devcontainer_config.go @@ -20,77 +20,82 @@ import ( "strings" ) +//nolint:tagliatelle type DevcontainerConfig struct { Image string `json:"image,omitempty"` - PostCreateCommand LifecycleCommand `json:"postCreateCommand,omitempty"` //nolint:tagliatelle - PostStartCommand LifecycleCommand `json:"postStartCommand,omitempty"` //nolint:tagliatelle - ForwardPorts []json.Number `json:"forwardPorts,omitempty"` //nolint:tagliatelle - ContainerEnv map[string]string `json:"containerEnv,omitempty"` //nolint:tagliatelle + PostCreateCommand LifecycleCommand `json:"postCreateCommand,omitempty"` + PostStartCommand LifecycleCommand `json:"postStartCommand,omitempty"` + ForwardPorts []json.Number `json:"forwardPorts,omitempty"` + ContainerEnv map[string]string `json:"containerEnv,omitempty"` Customizations DevContainerConfigCustomizations `json:"customizations,omitempty"` - RunArgs []string `json:"runArgs,omitempty"` //nolint:tagliatelle - ContainerUser string `json:"containerUser,omitempty"` //nolint:tagliatelle - RemoteUser string `json:"remoteUser,omitempty"` //nolint:tagliatelle + RunArgs []string `json:"runArgs,omitempty"` + ContainerUser string `json:"containerUser,omitempty"` + RemoteUser string `json:"remoteUser,omitempty"` } -// LifecycleCommand supports multiple formats for lifecycle commands. +//nolint:tagliatelle type LifecycleCommand struct { - CommandString string `json:"commandString,omitempty"` //nolint:tagliatelle - CommandArray []string `json:"commandArray,omitempty"` //nolint:tagliatelle + CommandString string `json:"commandString,omitempty"` + CommandArray []string `json:"commandArray,omitempty"` // Map to store commands by tags - CommandMap map[string]string `json:"commandMap,omitempty"` //nolint:tagliatelle + CommandMap map[string]string `json:"commandMap,omitempty"` + CommandMapArray map[string][]string `json:"commandMapArray,omitempty"` + Discriminator string `json:"-"` // Tracks the original type for proper re-marshaling } -// UnmarshalJSON custom unmarshal method for LifecycleCommand. -func (lc *LifecycleCommand) UnmarshalJSON(data []byte) error { - // Define a helper struct to match the object format - type Alias LifecycleCommand - var alias Alias - if err := json.Unmarshal(data, &alias); err == nil { - *lc = LifecycleCommand(alias) - return nil - } +// Constants for discriminator values. +const ( + TypeString = "string" + TypeArray = "array" + TypeCommandMapString = "commandMap" + TypeCommandMapArray = "commandMapArray" +) +func (lc *LifecycleCommand) UnmarshalJSON(data []byte) error { // Try to unmarshal as a single string var commandStr string if err := json.Unmarshal(data, &commandStr); err == nil { lc.CommandString = commandStr + lc.Discriminator = TypeString return nil } - // Try to unmarshal as an array of strings var commandArr []string if err := json.Unmarshal(data, &commandArr); err == nil { lc.CommandArray = commandArr + lc.Discriminator = TypeArray return nil } - // Try to unmarshal as a map of commands (tags to commands) - var commandMap map[string]interface{} + var commandMap map[string]string if err := json.Unmarshal(data, &commandMap); err == nil { - validatedCommands := make(map[string]string) - for tag, value := range commandMap { - switch v := value.(type) { - case string: - validatedCommands[tag] = v - case []interface{}: - var strArray []string - for _, item := range v { - if str, ok := item.(string); ok { - strArray = append(strArray, str) - } else { - return errors.New("invalid array type in command map") - } - } - validatedCommands[tag] = strings.Join(strArray, " ") - default: - return errors.New("map values must be string or []string") - } - } - lc.CommandMap = validatedCommands + lc.CommandMap = commandMap + lc.Discriminator = TypeCommandMapString return nil } + // Try to unmarshal as a CommandMapArray + var commandMapArray map[string][]string + if err := json.Unmarshal(data, &commandMapArray); err == nil { + lc.CommandMapArray = commandMapArray + lc.Discriminator = TypeCommandMapArray + return nil + } + return errors.New("invalid format: must be string, []string, map[string]string, or map[string][]string") +} - return errors.New("invalid format: must be string, []string, or map[string]string | map[string][]string") +func (lc *LifecycleCommand) MarshalJSON() ([]byte, error) { + switch lc.Discriminator { + case TypeString: + return json.Marshal(lc.CommandString) + case TypeArray: + return json.Marshal(lc.CommandArray) + case TypeCommandMapString: + return json.Marshal(lc.CommandMap) + case TypeCommandMapArray: + return json.Marshal(lc.CommandMapArray) + default: + return nil, errors.New("unknown type for LifecycleCommand") + } } // ToCommandArray converts the LifecycleCommand into a slice of full commands. @@ -106,6 +111,12 @@ func (lc *LifecycleCommand) ToCommandArray() []string { commands = append(commands, command) } return commands + case lc.CommandMapArray != nil: + var commands []string + for _, commandArray := range lc.CommandMapArray { + commands = append(commands, strings.Join(commandArray, " ")) + } + return commands default: return nil }