From b1d0c945d16fb4b45262312e34fb59b2fee3d54c Mon Sep 17 00:00:00 2001 From: Dhruv Dhruv Date: Tue, 14 Jan 2025 08:21:44 +0000 Subject: [PATCH] feat: [CDE-572]: Adding support for features- sorting and building. (#3248) * Addressing review comments. * feat: [CDE-572]: Adding support for features- sorting and building. --- .../orchestrator/utils/build_with_features.go | 188 +++++++++++++++- .../orchestrator/utils/download_features.go | 14 +- .../orchestrator/utils/sort_features.go | 208 ++++++++++++++++++ go.mod | 2 +- types/devcontainer_config.go | 10 + types/devcontainer_feature_config.go | 1 - types/resolved_feature.go | 10 + 7 files changed, 426 insertions(+), 7 deletions(-) create mode 100644 app/gitspace/orchestrator/utils/sort_features.go diff --git a/app/gitspace/orchestrator/utils/build_with_features.go b/app/gitspace/orchestrator/utils/build_with_features.go index 3c1a10735..c87931e6b 100644 --- a/app/gitspace/orchestrator/utils/build_with_features.go +++ b/app/gitspace/orchestrator/utils/build_with_features.go @@ -15,14 +15,28 @@ package utils import ( + "archive/tar" + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" "regexp" "strings" + + "github.com/harness/gitness/types" + + dockerTypes "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/rs/zerolog/log" ) -// ConvertOptionsToEnvVariables converts the option keys to standardised env variables using the +// convertOptionsToEnvVariables converts the option keys to standardised env variables using the // devcontainer specification to ensure uniformity in the naming and casing of the env variables. +// eg: 217yat%_fg -> _YAT_FG // Reference: https://containers.dev/implementors/features/#option-resolution -func ConvertOptionsToEnvVariables(str string) string { +func convertOptionsToEnvVariables(str string) string { // Replace all non-alphanumeric characters (excluding underscores) with '_' reNonAlnum := regexp.MustCompile(`[^\w_]`) str = reNonAlnum.ReplaceAllString(str, "_") @@ -36,3 +50,173 @@ func ConvertOptionsToEnvVariables(str string) string { return str } + +// BuildWithFeatures builds a docker image using the provided image as the base image. +// It sets some common env variables using the ARG instruction. +// For every feature, it copies the containerEnv variables using the ENV instruction and passes the resolved options +// as env variables in the RUN instruction. It further executes the install script. +func BuildWithFeatures( + ctx context.Context, + dockerClient *client.Client, + imageName string, + features []*types.ResolvedFeature, + gitspaceInstanceIdentifier string, + containerUser string, + remoteUser string, + containerUserHomeDir string, + remoteUserHomeDir string, +) (string, error) { + buildContextPath := getGitspaceInstanceDirectory(gitspaceInstanceIdentifier) + + defer func() { + err := os.RemoveAll(buildContextPath) + if err != nil { + log.Ctx(ctx).Err(err).Msgf("failed to remove build context directory %s", buildContextPath) + } + }() + + err := generateDockerFileWithFeatures(imageName, features, buildContextPath, containerUser, + containerUserHomeDir, remoteUser, remoteUserHomeDir) + if err != nil { + return "", err + } + + buildContext, err := packBuildContextDirectory(buildContextPath) + if err != nil { + return "", err + } + + newImageName := "gitspace-with-features:" + gitspaceInstanceIdentifier + buildRes, imageBuildErr := dockerClient.ImageBuild(ctx, buildContext, dockerTypes.ImageBuildOptions{ + SuppressOutput: false, + Tags: []string{newImageName}, + Version: dockerTypes.BuilderBuildKit, + }) + + defer func() { + if buildRes.Body != nil { + closeErr := buildRes.Body.Close() + if closeErr != nil { + log.Ctx(ctx).Err(closeErr).Msg("failed to close docker image build response body") + } + } + }() + + if imageBuildErr != nil { + return "", imageBuildErr + } + + _, err = io.Copy(io.Discard, buildRes.Body) + if err != nil { + return "", err + } + return newImageName, nil +} + +func packBuildContextDirectory(path string) (io.Reader, error) { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + + err := filepath.Walk(path, func(file string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + header, err := tar.FileInfoHeader(fi, file) + if err != nil { + return err + } + + header.Name, err = filepath.Rel(path, file) + if err != nil { + return err + } + + if err := tw.WriteHeader(header); err != nil { + return err + } + + if fi.Mode().IsRegular() { + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + + _, err = io.Copy(tw, f) + if err != nil { + return err + } + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk path %q: %w", path, err) + } + + if err := tw.Close(); err != nil { + return nil, fmt.Errorf("failed to close tar writer: %w", err) + } + + return &buf, nil +} + +// generateDockerFileWithFeatures creates and saves a dockerfile inside the build context directory. +func generateDockerFileWithFeatures( + imageName string, + features []*types.ResolvedFeature, + buildContextPath string, + containerUser string, + remoteUser string, + containerUserHomeDir string, + remoteUserHomeDir string, +) error { + dockerFile := fmt.Sprintf("FROM %s\nARG %s=%s\nARG %s=%s\nARG %s=%s\nARG %s=%s\nCOPY ./devcontainer-features %s", + imageName, convertOptionsToEnvVariables("_CONTAINER_USER"), containerUser, + convertOptionsToEnvVariables("_REMOTE_USER"), remoteUser, + convertOptionsToEnvVariables("_CONTAINER_USER_HOME"), containerUserHomeDir, + convertOptionsToEnvVariables("_REMOTE_USER_HOME"), remoteUserHomeDir, + "/tmp/devcontainer-features") + + for _, feature := range features { + if len(feature.DownloadedFeature.DevcontainerFeatureConfig.ContainerEnv) > 0 { + envVariables := "" + for key, value := range feature.DownloadedFeature.DevcontainerFeatureConfig.ContainerEnv { + envVariables += " " + key + "=" + value + } + dockerFile += fmt.Sprintf("\nENV%s", envVariables) + } + + finalOptionsMap := make(map[string]string) + for key, value := range feature.ResolvedOptions { + finalOptionsMap[convertOptionsToEnvVariables(key)] = value + } + + optionEnvVariables := "" + for key, value := range finalOptionsMap { + optionEnvVariables += " " + key + "=" + value + } + + installScriptPath := filepath.Join("/tmp/devcontainer-features", + getFeatureFolderNameWithTag(feature.DownloadedFeature.FeatureFolderName, feature.DownloadedFeature.Tag), + feature.DownloadedFeature.FeatureFolderName, "install.sh") + dockerFile += fmt.Sprintf("\nRUN%s chmod +x %s && %s", + optionEnvVariables, installScriptPath, installScriptPath) + } + + log.Debug().Msgf("generated dockerfile for build context %s\n%s", buildContextPath, dockerFile) + + file, err := os.OpenFile(filepath.Join(buildContextPath, "Dockerfile"), os.O_CREATE|os.O_RDWR, 0666) + if err != nil { + return fmt.Errorf("failed to create Dockerfile: %w", err) + } + defer file.Close() + + _, err = file.WriteString(dockerFile) + if err != nil { + return fmt.Errorf("failed to write content to Dockerfile: %w", err) + } + + return nil +} diff --git a/app/gitspace/orchestrator/utils/download_features.go b/app/gitspace/orchestrator/utils/download_features.go index 07a8d562f..54fa091f0 100644 --- a/app/gitspace/orchestrator/utils/download_features.go +++ b/app/gitspace/orchestrator/utils/download_features.go @@ -27,6 +27,7 @@ import ( "path/filepath" "strings" "sync" + "time" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" @@ -67,6 +68,10 @@ func DownloadFeatures( downloadQueue <- featureSource{sourceURL: key, sourceType: value.SourceType} } + // TODO: Add ctx based cancellations to the below goroutines. + + // NOTE: The following logic might see performance issues with spikes in memory and CPU usage. + // If there are such issues, we can introduce throttling on the basis of memory, CPU, etc. go func() { for source := range downloadQueue { startCh <- 1 @@ -102,6 +107,8 @@ waitLoop: default: if totalEnd > 0 && totalStart == totalEnd { break waitLoop + } else { + time.Sleep(time.Millisecond * 10) } } } @@ -183,6 +190,7 @@ func downloadFeature( downloadedFeature := types.DownloadedFeature{ FeatureFolderName: featureFolderName, + Source: source.sourceURL, SourceWithoutTag: sourceWithoutTag, Tag: featureTag, CanonicalName: canonicalName, @@ -219,10 +227,13 @@ func getDevcontainerFeatureConfig( if err != nil { return nil, fmt.Errorf("error unpacking tarball for feature %s: %w", source.sourceURL, err) } + + // Delete the tarball to avoid unnecessary packaging and copying during docker build. err = os.Remove(filepath.Join(downloadDirectory, tarballName)) if err != nil { return nil, fmt.Errorf("error deleting tarball for feature %s: %w", source.sourceURL, err) } + devcontainerFeatureRaw, err := os.ReadFile(filepath.Join(dst, "devcontainer-feature.json")) if err != nil { return nil, fmt.Errorf("error reading devcontainer-feature.json file for feature %s: %w", @@ -251,9 +262,6 @@ func getFeatureDownloadDirectory(gitspaceInstanceIdentifier, featureFolderName, } func getFeatureFolderNameWithTag(featureFolderName string, featureTag string) string { - if featureTag == "" { - return featureFolderName - } return featureFolderName + "-" + featureTag } diff --git a/app/gitspace/orchestrator/utils/sort_features.go b/app/gitspace/orchestrator/utils/sort_features.go new file mode 100644 index 000000000..4e0935453 --- /dev/null +++ b/app/gitspace/orchestrator/utils/sort_features.go @@ -0,0 +1,208 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "fmt" + "slices" + + "github.com/harness/gitness/types" +) + +type node struct { + Digest string + Dependencies map[string]bool + Priority int + Feature *types.ResolvedFeature +} + +type featureWithPriority struct { + Priority int + Feature *types.ResolvedFeature +} + +// SortFeatures sorts the features topologically, using round priorities and feature installation order override +// if provided by the user. It also keeps track of hard dependencies and soft dependencies. +// Reference: https://containers.dev/implementors/features/#dependency-installation-order-algorithm. +func SortFeatures( + featuresToBeInstalled map[string]*types.ResolvedFeature, + overrideFeatureInstallOrder []string, +) ([]*types.ResolvedFeature, error) { + sourcesWithoutTagsMappedToDigests := getSourcesWithoutTagsMappedToDigests(featuresToBeInstalled) + + adjacencyList, err := buildAdjacencyList(featuresToBeInstalled, sourcesWithoutTagsMappedToDigests, + overrideFeatureInstallOrder) + if err != nil { + return nil, err + } + + sortedFeatures, err := applyTopologicalSorting(adjacencyList) + if err != nil { + return nil, err + } + + return sortedFeatures, nil +} + +// getSourcesWithoutTagsMappedToDigests is used to map feature source (without tags) to their digests. +// Multiple features in the install set can have the same source but different options. All these features +// will be mapped to the same source and must be installed before any dependent features. +func getSourcesWithoutTagsMappedToDigests(featuresToBeInstalled map[string]*types.ResolvedFeature) map[string][]string { + sourcesWithoutTagsMappedToDigests := map[string][]string{} + for _, featureToBeInstalled := range featuresToBeInstalled { + sourceWithoutTag := featureToBeInstalled.DownloadedFeature.SourceWithoutTag + if _, initialized := sourcesWithoutTagsMappedToDigests[sourceWithoutTag]; !initialized { + sourcesWithoutTagsMappedToDigests[sourceWithoutTag] = []string{} + } + sourcesWithoutTagsMappedToDigests[sourceWithoutTag] = + append(sourcesWithoutTagsMappedToDigests[sourceWithoutTag], featureToBeInstalled.Digest) + } + return sourcesWithoutTagsMappedToDigests +} + +func buildAdjacencyList( + featuresToBeInstalled map[string]*types.ResolvedFeature, + sourcesWithoutTagsMappedToDigests map[string][]string, + overrideFeatureInstallOrder []string, +) ([]*node, error) { + counter := 0 + adjacencyList := make([]*node, 0) + for _, featureToBeInstalled := range featuresToBeInstalled { + dependencies := map[string]bool{} + + err := populateHardDependencies(featureToBeInstalled, dependencies) + if err != nil { + return nil, err + } + + populateSoftDependencies(sourcesWithoutTagsMappedToDigests, featureToBeInstalled, dependencies) + + // While the default priority is 0, it can be varied by the user through the overrideFeatureInstallOrder + // in the devcontainer.json. + // Reference: https://containers.dev/implementors/features/#overrideFeatureInstallOrder. + priority := 0 + index := slices.Index(overrideFeatureInstallOrder, featureToBeInstalled.DownloadedFeature.SourceWithoutTag) + if index > -1 { + priority = len(overrideFeatureInstallOrder) - index + counter++ + } + + graphNode := node{ + Digest: featureToBeInstalled.Digest, + Dependencies: dependencies, + Priority: priority, + Feature: featureToBeInstalled, + } + + adjacencyList = append(adjacencyList, &graphNode) + } + + // If any feature mentioned by the user in the overrideFeatureInstallOrder is not present in the install set, + // fail the flow. + difference := len(overrideFeatureInstallOrder) - counter + if difference > 0 { + return nil, fmt.Errorf("overrideFeatureInstallOrder contains %d extra features", difference) + } + + return adjacencyList, nil +} + +// populateSoftDependencies populates the digests of all the features whose source name is present in the installAfter +// property for the current feature ie which must be installed before the current feature can be installed. +// Any feature mentioned in the installAfter but not part of the install set is ignored. +func populateSoftDependencies( + sourcesWithoutTagsMappedToDigests map[string][]string, + featureToBeInstalled *types.ResolvedFeature, + dependencies map[string]bool, +) { + softDependencies := featureToBeInstalled.DownloadedFeature.DevcontainerFeatureConfig.InstallsAfter + if len(softDependencies) > 0 { + for _, softDependency := range softDependencies { + if digests, ok := sourcesWithoutTagsMappedToDigests[softDependency]; ok { + for _, digest := range digests { + if _, alreadyAdded := dependencies[digest]; !alreadyAdded { + dependencies[digest] = true + } + } + } + } + } +} + +// populateHardDependencies populates the digests of all the features which must be installed before the current +// feature can be installed. +func populateHardDependencies(featureToBeInstalled *types.ResolvedFeature, dependencies map[string]bool) error { + hardDependencies := featureToBeInstalled.DownloadedFeature.DevcontainerFeatureConfig.DependsOn + if hardDependencies != nil && len(*hardDependencies) > 0 { + for _, hardDependency := range *hardDependencies { + digest, err := calculateDigest(hardDependency.Source, hardDependency.Options) + if err != nil { + return fmt.Errorf("error calculating digest for %s: %w", hardDependency.Source, err) + } + dependencies[digest] = true + } + } + return nil +} + +func applyTopologicalSorting( + adjacencyList []*node, +) ([]*types.ResolvedFeature, error) { + sortedFeatures := make([]*types.ResolvedFeature, 0) + for len(sortedFeatures) < len(adjacencyList) { + maxPriority, eligibleFeatures := getFeaturesEligibleInThisRound(adjacencyList) + + if len(eligibleFeatures) == 0 { + return nil, fmt.Errorf("features can not be sorted") + } + + selectedFeatures := []*types.ResolvedFeature{} + + // only select those features which have the max priority, rest will be picked up in the next iteration. + for _, eligibleFeature := range eligibleFeatures { + if eligibleFeature.Priority == maxPriority { + selectedFeatures = append(selectedFeatures, eligibleFeature.Feature) + } + } + + slices.SortStableFunc(selectedFeatures, types.CompareResolvedFeature) + + for _, selectedFeature := range selectedFeatures { + sortedFeatures = append(sortedFeatures, selectedFeature) + for _, vertex := range adjacencyList { + delete(vertex.Dependencies, selectedFeature.Digest) + } + } + } + + return sortedFeatures, nil +} + +func getFeaturesEligibleInThisRound(adjacencyList []*node) (int, []featureWithPriority) { + maxPriorityInRound := 0 + eligibleFeatures := []featureWithPriority{} + for _, vertex := range adjacencyList { + if len(vertex.Dependencies) == 0 { + eligibleFeatures = append(eligibleFeatures, featureWithPriority{ + Priority: vertex.Priority, + Feature: vertex.Feature, + }) + if maxPriorityInRound < vertex.Priority { + maxPriorityInRound = vertex.Priority + } + } + } + return maxPriorityInRound, eligibleFeatures +} diff --git a/go.mod b/go.mod index c68878c06..a43fdba20 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.22.0 require ( cloud.google.com/go/storage v1.43.0 + github.com/Masterminds/semver/v3 v3.3.1 github.com/Masterminds/squirrel v1.5.4 github.com/adrg/xdg v0.5.0 github.com/aws/aws-sdk-go v1.55.2 @@ -98,7 +99,6 @@ require ( github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e // indirect github.com/BobuSumisu/aho-corasick v1.0.3 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/Masterminds/semver/v3 v3.3.1 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/antonmedv/expr v1.15.5 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect diff --git a/types/devcontainer_config.go b/types/devcontainer_config.go index e8b229f43..2665a7598 100644 --- a/types/devcontainer_config.go +++ b/types/devcontainer_config.go @@ -244,3 +244,13 @@ type Mount struct { Target string `json:"target,omitempty"` Type string `json:"type,omitempty"` } + +func (m *Mount) UnmarshalJSON(data []byte) error { + // TODO: Add support for unmarshalling mount data from a string input + var mount Mount + err := json.Unmarshal(data, &mount) + if err != nil { + return err + } + return nil +} diff --git a/types/devcontainer_feature_config.go b/types/devcontainer_feature_config.go index 5e4fbcb19..bb109ed64 100644 --- a/types/devcontainer_feature_config.go +++ b/types/devcontainer_feature_config.go @@ -32,7 +32,6 @@ type DevcontainerFeatureConfig struct { ContainerEnv map[string]string `json:"containerEnv,omitempty"` Privileged bool `json:"privileged,omitempty"` Init bool `json:"init,omitempty"` - Deprecated bool `json:"deprecated,omitempty"` CapAdd []string `json:"capAdd,omitempty"` SecurityOpt []string `json:"securityOpt,omitempty"` Entrypoint string `json:"entrypoint,omitempty"` diff --git a/types/resolved_feature.go b/types/resolved_feature.go index f36d8b3ba..49cd6d29e 100644 --- a/types/resolved_feature.go +++ b/types/resolved_feature.go @@ -15,6 +15,7 @@ package types import ( + "fmt" "sort" "strconv" "strings" @@ -31,12 +32,21 @@ type ResolvedFeature struct { type DownloadedFeature struct { FeatureFolderName string `json:"feature_folder_name,omitempty"` + Source string `json:"source,omitempty"` SourceWithoutTag string `json:"source_without_tag,omitempty"` Tag string `json:"tag,omitempty"` CanonicalName string `json:"canonical_name,omitempty"` DevcontainerFeatureConfig *DevcontainerFeatureConfig `json:"devcontainer_feature_config,omitempty"` } +func (r *ResolvedFeature) Print() string { + options := make([]string, 0, len(r.ResolvedOptions)) + for key, value := range r.ResolvedOptions { + options = append(options, fmt.Sprintf("%s=%s", key, value)) + } + return fmt.Sprintf("%s %+v", r.DownloadedFeature.Source, options) +} + // CompareResolvedFeature implements the following comparison rules. // 1. Compare and sort each Feature lexicographically by their fully qualified resource name // (For OCI-published Features, that means the ID without version or digest.). If the comparison is equal: