// Copyright 2023 Harness, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package container import ( "context" "fmt" "io" "os" "path/filepath" "strings" "github.com/harness/gitness/app/gitspace/logutil" "github.com/harness/gitness/infraprovider" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" dockerTypes "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/strslice" "github.com/docker/docker/client" "github.com/docker/go-connections/nat" "github.com/rs/zerolog/log" ) var _ Orchestrator = (*EmbeddedDockerOrchestrator)(nil) const ( loggingKey = "gitspace.container" sshPort = "22/tcp" catchAllIP = "0.0.0.0" catchAllPort = "0" containerStateRunning = "running" containerStateRemoved = "removed" templateCloneGit = "clone_git.sh" templateSetupSSHServer = "setup_ssh_server.sh" gitspacesDir = "gitspaces" ) type Config struct { DefaultBaseImage string DefaultBindMountTargetPath string DefaultBindMountSourceBasePath string DefaultBindMountSourceBasePathAbsolute string } type EmbeddedDockerOrchestrator struct { dockerClientFactory *infraprovider.DockerClientFactory vsCodeService *VSCode vsCodeWebService *VSCodeWeb config *Config statefulLogger *logutil.StatefulLogger } func NewEmbeddedDockerOrchestrator( dockerClientFactory *infraprovider.DockerClientFactory, vsCodeService *VSCode, vsCodeWebService *VSCodeWeb, config *Config, statefulLogger *logutil.StatefulLogger, ) Orchestrator { return &EmbeddedDockerOrchestrator{ dockerClientFactory: dockerClientFactory, vsCodeService: vsCodeService, vsCodeWebService: vsCodeWebService, config: config, statefulLogger: statefulLogger, } } // StartGitspace checks if the Gitspace is already running by checking its entry in a map. If is it running, // it returns, else, it creates a new Gitspace container by using the provided image. If the provided image is // nil, it uses a default image read from Gitness config. Post creation it runs the postCreate command and clones // the code inside the container. It uses the IDE service to setup the relevant IDE and also installs SSH server // inside the container. func (e *EmbeddedDockerOrchestrator) StartGitspace( ctx context.Context, gitspaceConfig *types.GitspaceConfig, devcontainerConfig *types.DevcontainerConfig, infra *infraprovider.Infrastructure, ) (*StartResponse, error) { containerName := getGitspaceContainerName(gitspaceConfig) log := log.Ctx(ctx).With().Str(loggingKey, containerName).Logger() dockerClient, err := e.dockerClientFactory.NewDockerClient(ctx, infra) if err != nil { return nil, fmt.Errorf("error getting docker client from docker client factory: %w", err) } defer func() { closingErr := dockerClient.Close() if closingErr != nil { log.Warn().Err(closingErr).Msg("failed to close docker client") } }() log.Debug().Msg("checking current state of gitspace") state, err := e.containerState(ctx, containerName, dockerClient) if err != nil { return nil, err } var usedPorts map[enum.IDEType]string var containerID string switch state { case containerStateRunning: log.Debug().Msg("gitspace is already running") ideService, startErr := e.getIDEService(gitspaceConfig) if startErr != nil { return nil, startErr } id, ports, startErr := e.getContainerInfo(ctx, containerName, dockerClient, ideService) if startErr != nil { return nil, startErr } usedPorts = ports containerID = id case containerStateRemoved: log.Debug().Msg("gitspace is not running, starting it...") ideService, startErr := e.getIDEService(gitspaceConfig) if startErr != nil { return nil, startErr } logStreamInstance, loggerErr := e.statefulLogger.CreateLogStream(ctx, gitspaceConfig.ID) if loggerErr != nil { return nil, fmt.Errorf("error getting log stream for gitspace ID %d: %w", gitspaceConfig.ID, loggerErr) } defer func() { loggerErr = logStreamInstance.Flush() if loggerErr != nil { log.Warn().Err(loggerErr).Msgf("failed to flush log stream for gitspace ID %d", gitspaceConfig.ID) } }() startErr = e.startGitspace( ctx, gitspaceConfig, devcontainerConfig, containerName, dockerClient, ideService, logStreamInstance, ) if startErr != nil { return nil, fmt.Errorf("failed to start gitspace %s: %w", containerName, startErr) } id, ports, startErr := e.getContainerInfo(ctx, containerName, dockerClient, ideService) if startErr != nil { return nil, startErr } containerID = id usedPorts = ports // TODO: Add gitspace status reporting. log.Debug().Msg("started gitspace") default: return nil, fmt.Errorf("gitspace %s is in a bad state: %s", containerName, state) } return &StartResponse{ ContainerID: containerID, ContainerName: containerName, WorkingDirectory: strings.TrimPrefix(e.config.DefaultBindMountTargetPath, "/"), PortsUsed: usedPorts, }, nil } func (e *EmbeddedDockerOrchestrator) startGitspace( ctx context.Context, gitspaceConfig *types.GitspaceConfig, devcontainerConfig *types.DevcontainerConfig, containerName string, dockerClient *client.Client, ideService IDE, logStreamInstance *logutil.LogStreamInstance, ) error { var imageName = devcontainerConfig.Image if imageName == "" { imageName = e.config.DefaultBaseImage } err := e.pullImage(ctx, imageName, dockerClient, logStreamInstance) if err != nil { return err } err = e.createContainer(ctx, gitspaceConfig, dockerClient, imageName, containerName, ideService, logStreamInstance) if err != nil { return err } var devcontainer = &Devcontainer{ ContainerName: containerName, DockerClient: dockerClient, WorkingDir: e.config.DefaultBindMountTargetPath, } err = e.executePostCreateCommand(ctx, devcontainerConfig, devcontainer, logStreamInstance) if err != nil { return err } err = e.cloneCode(ctx, gitspaceConfig, devcontainerConfig, devcontainer, logStreamInstance) if err != nil { return err } err = e.setupSSHServer(ctx, gitspaceConfig.GitspaceInstance, devcontainer, logStreamInstance) if err != nil { return err } err = e.setupIDE(ctx, devcontainer, ideService, logStreamInstance) if err != nil { return err } return nil } func (e *EmbeddedDockerOrchestrator) setupIDE( ctx context.Context, devcontainer *Devcontainer, ideService IDE, logStreamInstance *logutil.LogStreamInstance, ) error { loggingErr := logStreamInstance.Write("Setting up IDE inside container: " + string(ideService.Type())) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } output, err := ideService.Setup(ctx, devcontainer) if err != nil { loggingErr = logStreamInstance.Write("Error while setting up IDE inside container: " + err.Error()) err = fmt.Errorf("failed to setup IDE for gitspace %s: %w", devcontainer.ContainerName, err) if loggingErr != nil { err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr) } return err } loggingErr = logStreamInstance.Write("IDE setup output...\n" + string(output)) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } loggingErr = logStreamInstance.Write("Successfully set up IDE inside container") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } return nil } func (e *EmbeddedDockerOrchestrator) getContainerInfo( ctx context.Context, containerName string, dockerClient *client.Client, ideService IDE, ) (string, map[enum.IDEType]string, error) { inspectResp, err := dockerClient.ContainerInspect(ctx, containerName) if err != nil { return "", nil, fmt.Errorf("could not inspect container %s: %w", containerName, err) } usedPorts := map[enum.IDEType]string{} for port, bindings := range inspectResp.NetworkSettings.Ports { if port == sshPort { usedPorts[enum.IDETypeVSCode] = bindings[0].HostPort } if port == nat.Port(ideService.PortAndProtocol()) { usedPorts[ideService.Type()] = bindings[0].HostPort } } return inspectResp.ID, usedPorts, nil } func (e *EmbeddedDockerOrchestrator) getIDEService(gitspaceConfig *types.GitspaceConfig) (IDE, error) { var ideService IDE switch gitspaceConfig.IDE { case enum.IDETypeVSCode: ideService = e.vsCodeService case enum.IDETypeVSCodeWeb: ideService = e.vsCodeWebService default: return nil, fmt.Errorf("unsupported IDE: %s", gitspaceConfig.IDE) } return ideService, nil } func (e *EmbeddedDockerOrchestrator) setupSSHServer( ctx context.Context, gitspaceInstance *types.GitspaceInstance, devcontainer *Devcontainer, logStreamInstance *logutil.LogStreamInstance, ) error { sshServerScript, err := GenerateScriptFromTemplate( templateSetupSSHServer, &SetupSSHServerPayload{ Username: "harness", Password: *gitspaceInstance.AccessKey, WorkingDirectory: devcontainer.WorkingDir, }) if err != nil { return fmt.Errorf( "failed to generate scipt to setup ssh server from template %s: %w", templateSetupSSHServer, err) } loggingErr := logStreamInstance.Write("Installing ssh-server inside container") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } output, err := devcontainer.ExecuteCommand(ctx, sshServerScript, false) if err != nil { loggingErr = logStreamInstance.Write("Error while installing ssh-server inside container: " + err.Error()) err = fmt.Errorf("failed to setup SSH server: %w", err) if loggingErr != nil { err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr) } return err } loggingErr = logStreamInstance.Write("SSH server installation output...\n" + string(output)) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } loggingErr = logStreamInstance.Write("Successfully installed ssh-server inside container") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } return nil } func (e *EmbeddedDockerOrchestrator) cloneCode( ctx context.Context, gitspaceConfig *types.GitspaceConfig, devcontainerConfig *types.DevcontainerConfig, devcontainer *Devcontainer, logStreamInstance *logutil.LogStreamInstance, ) error { var devcontainerPresent = "true" if devcontainerConfig.Image == "" { devcontainerPresent = "false" } gitCloneScript, err := GenerateScriptFromTemplate( templateCloneGit, &CloneGitPayload{ RepoURL: gitspaceConfig.CodeRepoURL, DevcontainerPresent: devcontainerPresent, Image: e.config.DefaultBaseImage, }) if err != nil { return fmt.Errorf("failed to generate scipt to clone git from template %s: %w", templateCloneGit, err) } loggingErr := logStreamInstance.Write("Cloning git repo inside container: " + gitspaceConfig.CodeRepoURL) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } output, err := devcontainer.ExecuteCommand(ctx, gitCloneScript, false) if err != nil { loggingErr = logStreamInstance.Write("Error while cloning git repo inside container: " + err.Error()) err = fmt.Errorf("failed to clone code: %w", err) if loggingErr != nil { err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr) } return err } loggingErr = logStreamInstance.Write("Cloning git repo output...\n" + string(output)) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } loggingErr = logStreamInstance.Write("Successfully cloned git repo inside container") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } return nil } func (e *EmbeddedDockerOrchestrator) executePostCreateCommand( ctx context.Context, devcontainerConfig *types.DevcontainerConfig, devcontainer *Devcontainer, logStreamInstance *logutil.LogStreamInstance, ) error { if devcontainerConfig.PostCreateCommand == "" { loggingErr := logStreamInstance.Write("No post-create command provided, skipping execution") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } return nil } loggingErr := logStreamInstance.Write("Executing postCreate command: " + devcontainerConfig.PostCreateCommand) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } output, err := devcontainer.ExecuteCommand(ctx, devcontainerConfig.PostCreateCommand, false) if err != nil { loggingErr = logStreamInstance.Write("Error while executing postCreate command") err = fmt.Errorf("failed to execute postCreate command %q: %w", devcontainerConfig.PostCreateCommand, err) if loggingErr != nil { err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr) } return err } loggingErr = logStreamInstance.Write("Post create command execution output...\n" + string(output)) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } loggingErr = logStreamInstance.Write("Successfully executed postCreate command") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } return nil } func (e *EmbeddedDockerOrchestrator) createContainer( ctx context.Context, gitspaceConfig *types.GitspaceConfig, dockerClient *client.Client, imageName string, containerName string, ideService IDE, logStreamInstance *logutil.LogStreamInstance, ) error { portUsedByIDE := ideService.PortAndProtocol() exposedPorts := nat.PortSet{ sshPort: struct{}{}, } hostPortBindings := []nat.PortBinding{ { HostIP: catchAllIP, HostPort: catchAllPort, }, } portBindings := nat.PortMap{ sshPort: hostPortBindings, } if portUsedByIDE != "" { natPort := nat.Port(portUsedByIDE) exposedPorts[natPort] = struct{}{} portBindings[natPort] = hostPortBindings } entryPoint := make(strslice.StrSlice, 0) entryPoint = append(entryPoint, "sleep") commands := make(strslice.StrSlice, 0) commands = append(commands, "infinity") bindMountSourcePath := filepath.Join( e.config.DefaultBindMountSourceBasePath, gitspacesDir, gitspaceConfig.SpacePath, gitspaceConfig.Identifier, ) absoluteBindMountSourcePath := filepath.Join( e.config.DefaultBindMountSourceBasePathAbsolute, gitspacesDir, gitspaceConfig.SpacePath, gitspaceConfig.Identifier, ) loggingErr := logStreamInstance.Write( "Creating bind mount source directory: " + bindMountSourcePath + " (" + absoluteBindMountSourcePath + " )") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } err := os.MkdirAll(bindMountSourcePath, os.ModePerm) if err != nil { loggingErr = logStreamInstance.Write("Error while creating bind mount source directory: " + err.Error()) err = fmt.Errorf( "could not create bind mount source path %s: %w", bindMountSourcePath, err) if loggingErr != nil { err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr) } return err } loggingErr = logStreamInstance.Write("Successfully created bind mount source directory") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } loggingErr = logStreamInstance.Write("Creating container: " + containerName) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } resp2, err := dockerClient.ContainerCreate(ctx, &container.Config{ Image: imageName, Entrypoint: entryPoint, Cmd: commands, ExposedPorts: exposedPorts, }, &container.HostConfig{ PortBindings: portBindings, Mounts: []mount.Mount{ { Type: mount.TypeBind, Source: absoluteBindMountSourcePath, Target: e.config.DefaultBindMountTargetPath, }, }, }, nil, containerName) if err != nil { loggingErr = logStreamInstance.Write("Error while creating container: " + err.Error()) err = fmt.Errorf("could not create container %s: %w", containerName, err) if loggingErr != nil { err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr) } return err } loggingErr = logStreamInstance.Write("Successfully created container") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } loggingErr = logStreamInstance.Write("Starting container: " + containerName) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } err = dockerClient.ContainerStart(ctx, resp2.ID, dockerTypes.ContainerStartOptions{}) if err != nil { loggingErr = logStreamInstance.Write("Error while creating container: " + err.Error()) err = fmt.Errorf("could not start container %s: %w", containerName, err) if loggingErr != nil { err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr) } return err } loggingErr = logStreamInstance.Write("Successfully started container") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } return nil } func (e *EmbeddedDockerOrchestrator) pullImage( ctx context.Context, imageName string, dockerClient *client.Client, logStreamInstance *logutil.LogStreamInstance, ) error { loggingErr := logStreamInstance.Write("Pulling image: " + imageName) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } pullResponse, err := dockerClient.ImagePull(ctx, imageName, dockerTypes.ImagePullOptions{}) defer func() { closingErr := pullResponse.Close() if closingErr != nil { log.Warn().Err(closingErr).Msg("failed to close image pull response") } }() if err != nil { loggingErr = logStreamInstance.Write("Error while pulling image: " + err.Error()) err = fmt.Errorf("could not pull image %s: %w", imageName, err) if loggingErr != nil { err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr) } return err } // NOTE: It is necessary to read all the data in pullResponse to ensure the image has been completely downloaded. // If the execution proceeds before the response is completed, the container will not find the required image. output, err := io.ReadAll(pullResponse) if err != nil { loggingErr = logStreamInstance.Write("Error while parsing image pull response: " + err.Error()) err = fmt.Errorf("error while parsing pull image output %s: %w", imageName, err) if loggingErr != nil { err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr) } return err } loggingErr = logStreamInstance.Write(string(output)) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } loggingErr = logStreamInstance.Write("Successfully pulled image") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } return nil } // StopGitspace checks if the Gitspace container is running. If yes, it stops and removes the container. // Else it returns. func (e EmbeddedDockerOrchestrator) StopGitspace( ctx context.Context, gitspaceConfig *types.GitspaceConfig, infra *infraprovider.Infrastructure, ) error { containerName := getGitspaceContainerName(gitspaceConfig) log := log.Ctx(ctx).With().Str(loggingKey, containerName).Logger() dockerClient, err := e.dockerClientFactory.NewDockerClient(ctx, infra) if err != nil { return fmt.Errorf("error getting docker client from docker client factory: %w", err) } defer func() { closingErr := dockerClient.Close() if closingErr != nil { log.Warn().Err(closingErr).Msg("failed to close docker client") } }() log.Debug().Msg("checking current state of gitspace") state, err := e.containerState(ctx, containerName, dockerClient) if err != nil { return err } if state == containerStateRemoved { log.Debug().Msg("gitspace is already stopped") return nil } log.Debug().Msg("stopping gitspace") logStreamInstance, loggerErr := e.statefulLogger.CreateLogStream(ctx, gitspaceConfig.ID) if loggerErr != nil { return fmt.Errorf("error getting log stream for gitspace ID %d: %w", gitspaceConfig.ID, loggerErr) } defer func() { loggerErr = logStreamInstance.Flush() if loggerErr != nil { log.Warn().Err(loggerErr).Msgf("failed to flush log stream for gitspace ID %d", gitspaceConfig.ID) } }() err = e.stopGitspace(ctx, containerName, dockerClient, logStreamInstance) if err != nil { return fmt.Errorf("failed to stop gitspace %s: %w", containerName, err) } log.Debug().Msg("stopped gitspace") return nil } func (e EmbeddedDockerOrchestrator) stopGitspace( ctx context.Context, containerName string, dockerClient *client.Client, logStreamInstance *logutil.LogStreamInstance, ) error { loggingErr := logStreamInstance.Write("Stopping container: " + containerName) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } err := dockerClient.ContainerStop(ctx, containerName, nil) if err != nil { loggingErr = logStreamInstance.Write("Error while stopping container: " + err.Error()) err = fmt.Errorf("could not stop container %s: %w", containerName, err) if loggingErr != nil { err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr) } return err } loggingErr = logStreamInstance.Write("Successfully stopped container") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } loggingErr = logStreamInstance.Write("Removing container: " + containerName) if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } err = dockerClient.ContainerRemove(ctx, containerName, dockerTypes.ContainerRemoveOptions{Force: true}) if err != nil { loggingErr = logStreamInstance.Write("Error while removing container: " + err.Error()) err = fmt.Errorf("could not remove container %s: %w", containerName, err) if loggingErr != nil { err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr) } return err } loggingErr = logStreamInstance.Write("Successfully removed container") if loggingErr != nil { return fmt.Errorf("logging error: %w", loggingErr) } return nil } func getGitspaceContainerName(config *types.GitspaceConfig) string { return "gitspace-" + config.UserID + "-" + config.Identifier } // Status is NOOP for EmbeddedDockerOrchestrator as the docker host is verified by the infra provisioner. func (e *EmbeddedDockerOrchestrator) Status(_ context.Context, _ *infraprovider.Infrastructure) error { return nil } func (e *EmbeddedDockerOrchestrator) containerState( ctx context.Context, containerName string, dockerClient *client.Client, ) (string, error) { var args = filters.NewArgs() args.Add("name", containerName) containers, err := dockerClient.ContainerList(ctx, dockerTypes.ContainerListOptions{All: true, Filters: args}) if err != nil { return "", fmt.Errorf("could not list container %s: %w", containerName, err) } if len(containers) == 0 { return containerStateRemoved, nil } return containers[0].State, nil }