drone/app/gitspace/orchestrator/container/embedded_docker.go

810 lines
22 KiB
Go

// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package container
import (
"context"
"fmt"
"io"
"github.com/harness/gitness/app/gitspace/logutil"
"github.com/harness/gitness/infraprovider"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
dockerTypes "github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/client"
"github.com/docker/go-connections/nat"
"github.com/rs/zerolog/log"
)
var _ Orchestrator = (*EmbeddedDockerOrchestrator)(nil)
const (
loggingKey = "gitspace.container"
catchAllIP = "0.0.0.0"
catchAllPort = "0"
containerStateRunning = "running"
containerStateRemoved = "removed"
containerStateStopped = "exited"
templateCloneGit = "clone_git.sh"
templateSetupSSHServer = "setup_ssh_server.sh"
)
type Config struct {
DefaultBaseImage string
}
type EmbeddedDockerOrchestrator struct {
dockerClientFactory *infraprovider.DockerClientFactory
vsCodeService *VSCode
vsCodeWebService *VSCodeWeb
config *Config
statefulLogger *logutil.StatefulLogger
}
func NewEmbeddedDockerOrchestrator(
dockerClientFactory *infraprovider.DockerClientFactory,
vsCodeService *VSCode,
vsCodeWebService *VSCodeWeb,
config *Config,
statefulLogger *logutil.StatefulLogger,
) Orchestrator {
return &EmbeddedDockerOrchestrator{
dockerClientFactory: dockerClientFactory,
vsCodeService: vsCodeService,
vsCodeWebService: vsCodeWebService,
config: config,
statefulLogger: statefulLogger,
}
}
// CreateAndStartGitspace starts an exited container and starts a new container if the container is removed.
// If the container is newly created, it clones the code, sets up the IDE and executes the postCreateCommand.
// It returns the container ID, name and ports used.
// It returns an error if the container is not running, exited or removed.
func (e *EmbeddedDockerOrchestrator) CreateAndStartGitspace(
ctx context.Context,
gitspaceConfig *types.GitspaceConfig,
devcontainerConfig *types.DevcontainerConfig,
infra *infraprovider.Infrastructure,
repoName string,
) (*StartResponse, error) {
containerName := getGitspaceContainerName(gitspaceConfig)
log := log.Ctx(ctx).With().Str(loggingKey, containerName).Logger()
dockerClient, err := e.dockerClientFactory.NewDockerClient(ctx, infra)
if err != nil {
return nil, fmt.Errorf("error getting docker client from docker client factory: %w", err)
}
defer func() {
closingErr := dockerClient.Close()
if closingErr != nil {
log.Warn().Err(closingErr).Msg("failed to close docker client")
}
}()
log.Debug().Msg("checking current state of gitspace")
state, err := e.containerState(ctx, containerName, dockerClient)
if err != nil {
return nil, err
}
ideService, err := e.getIDEService(gitspaceConfig)
if err != nil {
return nil, err
}
switch state {
case containerStateRunning:
log.Debug().Msg("gitspace is already running")
case containerStateStopped:
log.Debug().Msg("gitspace is stopped, starting it")
logStreamInstance, loggerErr := e.statefulLogger.CreateLogStream(ctx, gitspaceConfig.ID)
if loggerErr != nil {
return nil, fmt.Errorf("error getting log stream for gitspace ID %d: %w", gitspaceConfig.ID, loggerErr)
}
defer func() {
loggerErr = logStreamInstance.Flush()
if loggerErr != nil {
log.Warn().Err(loggerErr).Msgf("failed to flush log stream for gitspace ID %d", gitspaceConfig.ID)
}
}()
startErr := e.startContainer(ctx, dockerClient, containerName, logStreamInstance)
if startErr != nil {
return nil, startErr
}
// TODO: Add gitspace status reporting.
log.Debug().Msg("started gitspace")
case containerStateRemoved:
log.Debug().Msg("gitspace is removed, creating it...")
logStreamInstance, loggerErr := e.statefulLogger.CreateLogStream(ctx, gitspaceConfig.ID)
if loggerErr != nil {
return nil, fmt.Errorf("error getting log stream for gitspace ID %d: %w", gitspaceConfig.ID, loggerErr)
}
defer func() {
loggerErr = logStreamInstance.Flush()
if loggerErr != nil {
log.Warn().Err(loggerErr).Msgf("failed to flush log stream for gitspace ID %d", gitspaceConfig.ID)
}
}()
workingDirectory := "/" + repoName
startErr := e.startGitspace(
ctx,
gitspaceConfig,
devcontainerConfig,
containerName,
dockerClient,
ideService,
logStreamInstance,
infra.Storage,
workingDirectory,
)
if startErr != nil {
return nil, fmt.Errorf("failed to start gitspace %s: %w", containerName, startErr)
}
// TODO: Add gitspace status reporting.
log.Debug().Msg("started gitspace")
default:
return nil, fmt.Errorf("gitspace %s is in a bad state: %s", containerName, state)
}
id, ports, startErr := e.getContainerInfo(ctx, containerName, dockerClient, ideService)
if startErr != nil {
return nil, startErr
}
return &StartResponse{
ContainerID: id,
ContainerName: containerName,
PortsUsed: ports,
}, nil
}
func (e *EmbeddedDockerOrchestrator) startGitspace(
ctx context.Context,
gitspaceConfig *types.GitspaceConfig,
devcontainerConfig *types.DevcontainerConfig,
containerName string,
dockerClient *client.Client,
ideService IDE,
logStreamInstance *logutil.LogStreamInstance,
volumeName string,
workingDirectory string,
) error {
var imageName = devcontainerConfig.Image
if imageName == "" {
imageName = e.config.DefaultBaseImage
}
err := e.pullImage(ctx, imageName, dockerClient, logStreamInstance)
if err != nil {
return err
}
err = e.createContainer(
ctx,
dockerClient,
imageName,
containerName,
ideService,
logStreamInstance,
volumeName,
workingDirectory,
)
if err != nil {
return err
}
err = e.startContainer(ctx, dockerClient, containerName, logStreamInstance)
if err != nil {
return err
}
var devcontainer = &Devcontainer{
ContainerName: containerName,
DockerClient: dockerClient,
WorkingDir: workingDirectory,
}
err = e.cloneCode(ctx, gitspaceConfig, devcontainerConfig, devcontainer, logStreamInstance)
if err != nil {
return err
}
err = e.setupIDE(ctx, gitspaceConfig.GitspaceInstance, devcontainer, ideService, logStreamInstance)
if err != nil {
return err
}
err = e.executePostCreateCommand(ctx, devcontainerConfig, devcontainer, logStreamInstance)
if err != nil {
return err
}
return nil
}
func (e *EmbeddedDockerOrchestrator) setupIDE(
ctx context.Context,
gitspaceInstance *types.GitspaceInstance,
devcontainer *Devcontainer,
ideService IDE,
logStreamInstance *logutil.LogStreamInstance,
) error {
loggingErr := logStreamInstance.Write("Setting up IDE inside container: " + string(ideService.Type()))
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
output, err := ideService.Setup(ctx, devcontainer, gitspaceInstance)
if err != nil {
loggingErr = logStreamInstance.Write("Error while setting up IDE inside container: " + err.Error())
err = fmt.Errorf("failed to setup IDE for gitspace %s: %w", devcontainer.ContainerName, err)
if loggingErr != nil {
err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr)
}
return err
}
loggingErr = logStreamInstance.Write("IDE setup output...\n" + string(output))
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
loggingErr = logStreamInstance.Write("Successfully set up IDE inside container")
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
return nil
}
func (e *EmbeddedDockerOrchestrator) getContainerInfo(
ctx context.Context,
containerName string,
dockerClient *client.Client,
ideService IDE,
) (string, map[enum.IDEType]string, error) {
inspectResp, err := dockerClient.ContainerInspect(ctx, containerName)
if err != nil {
return "", nil, fmt.Errorf("could not inspect container %s: %w", containerName, err)
}
usedPorts := map[enum.IDEType]string{}
for port, bindings := range inspectResp.NetworkSettings.Ports {
if port == nat.Port(ideService.PortAndProtocol()) {
usedPorts[ideService.Type()] = bindings[0].HostPort
}
}
return inspectResp.ID, usedPorts, nil
}
func (e *EmbeddedDockerOrchestrator) getIDEService(gitspaceConfig *types.GitspaceConfig) (IDE, error) {
var ideService IDE
switch gitspaceConfig.IDE {
case enum.IDETypeVSCode:
ideService = e.vsCodeService
case enum.IDETypeVSCodeWeb:
ideService = e.vsCodeWebService
default:
return nil, fmt.Errorf("unsupported IDE: %s", gitspaceConfig.IDE)
}
return ideService, nil
}
func (e *EmbeddedDockerOrchestrator) cloneCode(
ctx context.Context,
gitspaceConfig *types.GitspaceConfig,
devcontainerConfig *types.DevcontainerConfig,
devcontainer *Devcontainer,
logStreamInstance *logutil.LogStreamInstance,
) error {
var devcontainerPresent = "true"
if devcontainerConfig.Image == "" {
devcontainerPresent = "false"
}
gitCloneScript, err := GenerateScriptFromTemplate(
templateCloneGit, &CloneGitPayload{
RepoURL: gitspaceConfig.CodeRepoURL,
DevcontainerPresent: devcontainerPresent,
Image: e.config.DefaultBaseImage,
Branch: gitspaceConfig.Branch,
})
if err != nil {
return fmt.Errorf("failed to generate scipt to clone git from template %s: %w", templateCloneGit, err)
}
loggingErr := logStreamInstance.Write(
"Cloning git repo inside container: " + gitspaceConfig.CodeRepoURL + " branch: " + gitspaceConfig.Branch)
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
output, err := devcontainer.ExecuteCommand(ctx, gitCloneScript, false)
if err != nil {
loggingErr = logStreamInstance.Write("Error while cloning git repo inside container: " + err.Error())
err = fmt.Errorf("failed to clone code: %w", err)
if loggingErr != nil {
err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr)
}
return err
}
loggingErr = logStreamInstance.Write("Cloning git repo output...\n" + string(output))
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
loggingErr = logStreamInstance.Write("Successfully cloned git repo inside container")
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
return nil
}
func (e *EmbeddedDockerOrchestrator) executePostCreateCommand(
ctx context.Context,
devcontainerConfig *types.DevcontainerConfig,
devcontainer *Devcontainer,
logStreamInstance *logutil.LogStreamInstance,
) error {
if devcontainerConfig.PostCreateCommand == "" {
loggingErr := logStreamInstance.Write("No post-create command provided, skipping execution")
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
return nil
}
loggingErr := logStreamInstance.Write("Executing postCreate command: " + devcontainerConfig.PostCreateCommand)
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
output, err := devcontainer.ExecuteCommand(ctx, devcontainerConfig.PostCreateCommand, false)
if err != nil {
loggingErr = logStreamInstance.Write("Error while executing postCreate command")
err = fmt.Errorf("failed to execute postCreate command %q: %w", devcontainerConfig.PostCreateCommand, err)
if loggingErr != nil {
err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr)
}
return err
}
loggingErr = logStreamInstance.Write("Post create command execution output...\n" + string(output))
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
loggingErr = logStreamInstance.Write("Successfully executed postCreate command")
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
return nil
}
func (e *EmbeddedDockerOrchestrator) startContainer(
ctx context.Context,
dockerClient *client.Client,
containerName string,
logStreamInstance *logutil.LogStreamInstance,
) error {
loggingErr := logStreamInstance.Write("Starting container: " + containerName)
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
err := dockerClient.ContainerStart(ctx, containerName, dockerTypes.ContainerStartOptions{})
if err != nil {
loggingErr = logStreamInstance.Write("Error while creating container: " + err.Error())
err = fmt.Errorf("could not start container %s: %w", containerName, err)
if loggingErr != nil {
err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr)
}
return err
}
loggingErr = logStreamInstance.Write("Successfully started container")
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
return nil
}
func (e *EmbeddedDockerOrchestrator) createContainer(
ctx context.Context,
dockerClient *client.Client,
imageName string,
containerName string,
ideService IDE,
logStreamInstance *logutil.LogStreamInstance,
volumeName string,
workingDirectory string,
) error {
portUsedByIDE := ideService.PortAndProtocol()
hostPortBindings := []nat.PortBinding{
{
HostIP: catchAllIP,
HostPort: catchAllPort,
},
}
exposedPorts := nat.PortSet{}
portBindings := nat.PortMap{}
if portUsedByIDE != "" {
natPort := nat.Port(portUsedByIDE)
exposedPorts[natPort] = struct{}{}
portBindings[natPort] = hostPortBindings
}
entryPoint := []string{"/bin/bash", "-c", `trap "exit 0" 15; sleep infinity`}
loggingErr := logStreamInstance.Write("Creating container: " + containerName)
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
_, err := dockerClient.ContainerCreate(ctx, &container.Config{
Image: imageName,
Entrypoint: entryPoint,
ExposedPorts: exposedPorts,
}, &container.HostConfig{
PortBindings: portBindings,
Mounts: []mount.Mount{
{
Type: mount.TypeVolume,
Source: volumeName,
Target: workingDirectory,
},
},
}, nil, containerName)
if err != nil {
loggingErr = logStreamInstance.Write("Error while creating container: " + err.Error())
err = fmt.Errorf("could not create container %s: %w", containerName, err)
if loggingErr != nil {
err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr)
}
return err
}
return nil
}
func (e *EmbeddedDockerOrchestrator) pullImage(
ctx context.Context,
imageName string,
dockerClient *client.Client,
logStreamInstance *logutil.LogStreamInstance,
) error {
loggingErr := logStreamInstance.Write("Pulling image: " + imageName)
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
pullResponse, err := dockerClient.ImagePull(ctx, imageName, dockerTypes.ImagePullOptions{})
defer func() {
closingErr := pullResponse.Close()
if closingErr != nil {
log.Warn().Err(closingErr).Msg("failed to close image pull response")
}
}()
if err != nil {
loggingErr = logStreamInstance.Write("Error while pulling image: " + err.Error())
err = fmt.Errorf("could not pull image %s: %w", imageName, err)
if loggingErr != nil {
err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr)
}
return err
}
// NOTE: It is necessary to read all the data in pullResponse to ensure the image has been completely downloaded.
// If the execution proceeds before the response is completed, the container will not find the required image.
output, err := io.ReadAll(pullResponse)
if err != nil {
loggingErr = logStreamInstance.Write("Error while parsing image pull response: " + err.Error())
err = fmt.Errorf("error while parsing pull image output %s: %w", imageName, err)
if loggingErr != nil {
err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr)
}
return err
}
loggingErr = logStreamInstance.Write(string(output))
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
loggingErr = logStreamInstance.Write("Successfully pulled image")
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
return nil
}
// StopGitspace stops a container. If it is removed, it returns an error.
func (e EmbeddedDockerOrchestrator) StopGitspace(
ctx context.Context,
gitspaceConfig *types.GitspaceConfig,
infra *infraprovider.Infrastructure,
) error {
containerName := getGitspaceContainerName(gitspaceConfig)
log := log.Ctx(ctx).With().Str(loggingKey, containerName).Logger()
dockerClient, err := e.dockerClientFactory.NewDockerClient(ctx, infra)
if err != nil {
return fmt.Errorf("error getting docker client from docker client factory: %w", err)
}
defer func() {
closingErr := dockerClient.Close()
if closingErr != nil {
log.Warn().Err(closingErr).Msg("failed to close docker client")
}
}()
log.Debug().Msg("checking current state of gitspace")
state, err := e.containerState(ctx, containerName, dockerClient)
if err != nil {
return err
}
if state == containerStateRemoved {
return fmt.Errorf("gitspace %s is removed", containerName)
}
if state == containerStateStopped {
log.Debug().Msg("gitspace is already stopped")
return nil
}
log.Debug().Msg("stopping gitspace")
logStreamInstance, loggerErr := e.statefulLogger.CreateLogStream(ctx, gitspaceConfig.ID)
if loggerErr != nil {
return fmt.Errorf("error getting log stream for gitspace ID %d: %w", gitspaceConfig.ID, loggerErr)
}
defer func() {
loggerErr = logStreamInstance.Flush()
if loggerErr != nil {
log.Warn().Err(loggerErr).Msgf("failed to flush log stream for gitspace ID %d", gitspaceConfig.ID)
}
}()
err = e.stopContainer(ctx, containerName, dockerClient, logStreamInstance)
if err != nil {
return fmt.Errorf("failed to stop gitspace %s: %w", containerName, err)
}
log.Debug().Msg("stopped gitspace")
return nil
}
func (e EmbeddedDockerOrchestrator) stopContainer(
ctx context.Context,
containerName string,
dockerClient *client.Client,
logStreamInstance *logutil.LogStreamInstance,
) error {
loggingErr := logStreamInstance.Write("Stopping container: " + containerName)
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
err := dockerClient.ContainerStop(ctx, containerName, nil)
if err != nil {
loggingErr = logStreamInstance.Write("Error while stopping container: " + err.Error())
err = fmt.Errorf("could not stop container %s: %w", containerName, err)
if loggingErr != nil {
err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr)
}
return err
}
loggingErr = logStreamInstance.Write("Successfully stopped container")
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
return nil
}
func getGitspaceContainerName(config *types.GitspaceConfig) string {
return "gitspace-" + config.UserID + "-" + config.Identifier
}
// Status is NOOP for EmbeddedDockerOrchestrator as the docker host is verified by the infra provisioner.
func (e *EmbeddedDockerOrchestrator) Status(_ context.Context, _ *infraprovider.Infrastructure) error {
return nil
}
func (e *EmbeddedDockerOrchestrator) containerState(
ctx context.Context,
containerName string,
dockerClient *client.Client,
) (string, error) {
var args = filters.NewArgs()
args.Add("name", containerName)
containers, err := dockerClient.ContainerList(ctx, dockerTypes.ContainerListOptions{All: true, Filters: args})
if err != nil {
return "", fmt.Errorf("could not list container %s: %w", containerName, err)
}
if len(containers) == 0 {
return containerStateRemoved, nil
}
return containers[0].State, nil
}
// StopAndRemoveGitspace stops the container if not stopped and removes it.
// If the container is already removed, it returns.
func (e *EmbeddedDockerOrchestrator) StopAndRemoveGitspace(
ctx context.Context,
gitspaceConfig *types.GitspaceConfig,
infra *infraprovider.Infrastructure,
) error {
containerName := getGitspaceContainerName(gitspaceConfig)
log := log.Ctx(ctx).With().Str(loggingKey, containerName).Logger()
dockerClient, err := e.dockerClientFactory.NewDockerClient(ctx, infra)
if err != nil {
return fmt.Errorf("error getting docker client from docker client factory: %w", err)
}
defer func() {
closingErr := dockerClient.Close()
if closingErr != nil {
log.Warn().Err(closingErr).Msg("failed to close docker client")
}
}()
log.Debug().Msg("checking current state of gitspace")
state, err := e.containerState(ctx, containerName, dockerClient)
if err != nil {
return err
}
if state == containerStateRemoved {
log.Debug().Msg("gitspace is already removed")
return nil
}
logStreamInstance, loggerErr := e.statefulLogger.CreateLogStream(ctx, gitspaceConfig.ID)
if loggerErr != nil {
return fmt.Errorf("error getting log stream for gitspace ID %d: %w", gitspaceConfig.ID, loggerErr)
}
defer func() {
loggerErr = logStreamInstance.Flush()
if loggerErr != nil {
log.Warn().Err(loggerErr).Msgf("failed to flush log stream for gitspace ID %d", gitspaceConfig.ID)
}
}()
if state != containerStateStopped {
log.Debug().Msg("stopping gitspace")
err = e.stopContainer(ctx, containerName, dockerClient, logStreamInstance)
if err != nil {
return fmt.Errorf("failed to stop gitspace %s: %w", containerName, err)
}
log.Debug().Msg("stopped gitspace")
}
log.Debug().Msg("removing gitspace")
err = e.removeContainer(ctx, containerName, dockerClient, logStreamInstance)
if err != nil {
return fmt.Errorf("failed to remove gitspace %s: %w", containerName, err)
}
log.Debug().Msg("removed gitspace")
return nil
}
func (e EmbeddedDockerOrchestrator) removeContainer(
ctx context.Context,
containerName string,
dockerClient *client.Client,
logStreamInstance *logutil.LogStreamInstance,
) error {
loggingErr := logStreamInstance.Write("Removing container: " + containerName)
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
err := dockerClient.ContainerRemove(ctx, containerName, dockerTypes.ContainerRemoveOptions{Force: true})
if err != nil {
loggingErr = logStreamInstance.Write("Error while removing container: " + err.Error())
err = fmt.Errorf("could not remove container %s: %w", containerName, err)
if loggingErr != nil {
err = fmt.Errorf("original error: %w; logging error: %w", err, loggingErr)
}
return err
}
loggingErr = logStreamInstance.Write("Successfully removed container")
if loggingErr != nil {
return fmt.Errorf("logging error: %w", loggingErr)
}
return nil
}