feat: [CDE-572]: Adding support for features- sorting and building. (#3248)

* Addressing review comments.
* feat: [CDE-572]: Adding support for features- sorting and building.

app/gitspace/orchestrator/utils/build_with_features.go

@@ -15,14 +15,28 @@
 package utils
 
 import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
+
+	"github.com/harness/gitness/types"
+
+	dockerTypes "github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+	"github.com/rs/zerolog/log"
 )
 
-// ConvertOptionsToEnvVariables converts the option keys to standardised env variables using the
+// convertOptionsToEnvVariables converts the option keys to standardised env variables using the
 // devcontainer specification to ensure uniformity in the naming and casing of the env variables.
+// eg: 217yat%_fg -> _YAT_FG
 // Reference: https://containers.dev/implementors/features/#option-resolution
-func ConvertOptionsToEnvVariables(str string) string {
+func convertOptionsToEnvVariables(str string) string {
 	// Replace all non-alphanumeric characters (excluding underscores) with '_'
 	reNonAlnum := regexp.MustCompile(`[^\w_]`)
 	str = reNonAlnum.ReplaceAllString(str, "_")
@@ -36,3 +50,173 @@ func ConvertOptionsToEnvVariables(str string) string {
 
 	return str
 }
+
+// BuildWithFeatures builds a docker image using the provided image as the base image.
+// It sets some common env variables using the ARG instruction.
+// For every feature, it copies the containerEnv variables using the ENV instruction and passes the resolved options
+// as env variables in the RUN instruction. It further executes the install script.
+func BuildWithFeatures(
+	ctx context.Context,
+	dockerClient *client.Client,
+	imageName string,
+	features []*types.ResolvedFeature,
+	gitspaceInstanceIdentifier string,
+	containerUser string,
+	remoteUser string,
+	containerUserHomeDir string,
+	remoteUserHomeDir string,
+) (string, error) {
+	buildContextPath := getGitspaceInstanceDirectory(gitspaceInstanceIdentifier)
+
+	defer func() {
+		err := os.RemoveAll(buildContextPath)
+		if err != nil {
+			log.Ctx(ctx).Err(err).Msgf("failed to remove build context directory %s", buildContextPath)
+		}
+	}()
+
+	err := generateDockerFileWithFeatures(imageName, features, buildContextPath, containerUser,
+		containerUserHomeDir, remoteUser, remoteUserHomeDir)
+	if err != nil {
+		return "", err
+	}
+
+	buildContext, err := packBuildContextDirectory(buildContextPath)
+	if err != nil {
+		return "", err
+	}
+
+	newImageName := "gitspace-with-features:" + gitspaceInstanceIdentifier
+	buildRes, imageBuildErr := dockerClient.ImageBuild(ctx, buildContext, dockerTypes.ImageBuildOptions{
+		SuppressOutput: false,
+		Tags:           []string{newImageName},
+		Version:        dockerTypes.BuilderBuildKit,
+	})
+
+	defer func() {
+		if buildRes.Body != nil {
+			closeErr := buildRes.Body.Close()
+			if closeErr != nil {
+				log.Ctx(ctx).Err(closeErr).Msg("failed to close docker image build response body")
+			}
+		}
+	}()
+
+	if imageBuildErr != nil {
+		return "", imageBuildErr
+	}
+
+	_, err = io.Copy(io.Discard, buildRes.Body)
+	if err != nil {
+		return "", err
+	}
+	return newImageName, nil
+}
+
+func packBuildContextDirectory(path string) (io.Reader, error) {
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+
+	err := filepath.Walk(path, func(file string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		header, err := tar.FileInfoHeader(fi, file)
+		if err != nil {
+			return err
+		}
+
+		header.Name, err = filepath.Rel(path, file)
+		if err != nil {
+			return err
+		}
+
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+
+		if fi.Mode().IsRegular() {
+			f, err := os.Open(file)
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+
+			_, err = io.Copy(tw, f)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to walk path %q: %w", path, err)
+	}
+
+	if err := tw.Close(); err != nil {
+		return nil, fmt.Errorf("failed to close tar writer: %w", err)
+	}
+
+	return &buf, nil
+}
+
+// generateDockerFileWithFeatures creates and saves a dockerfile inside the build context directory.
+func generateDockerFileWithFeatures(
+	imageName string,
+	features []*types.ResolvedFeature,
+	buildContextPath string,
+	containerUser string,
+	remoteUser string,
+	containerUserHomeDir string,
+	remoteUserHomeDir string,
+) error {
+	dockerFile := fmt.Sprintf("FROM %s\nARG %s=%s\nARG %s=%s\nARG %s=%s\nARG %s=%s\nCOPY ./devcontainer-features %s",
+		imageName, convertOptionsToEnvVariables("_CONTAINER_USER"), containerUser,
+		convertOptionsToEnvVariables("_REMOTE_USER"), remoteUser,
+		convertOptionsToEnvVariables("_CONTAINER_USER_HOME"), containerUserHomeDir,
+		convertOptionsToEnvVariables("_REMOTE_USER_HOME"), remoteUserHomeDir,
+		"/tmp/devcontainer-features")
+
+	for _, feature := range features {
+		if len(feature.DownloadedFeature.DevcontainerFeatureConfig.ContainerEnv) > 0 {
+			envVariables := ""
+			for key, value := range feature.DownloadedFeature.DevcontainerFeatureConfig.ContainerEnv {
+				envVariables += " " + key + "=" + value
+			}
+			dockerFile += fmt.Sprintf("\nENV%s", envVariables)
+		}
+
+		finalOptionsMap := make(map[string]string)
+		for key, value := range feature.ResolvedOptions {
+			finalOptionsMap[convertOptionsToEnvVariables(key)] = value
+		}
+
+		optionEnvVariables := ""
+		for key, value := range finalOptionsMap {
+			optionEnvVariables += " " + key + "=" + value
+		}
+
+		installScriptPath := filepath.Join("/tmp/devcontainer-features",
+			getFeatureFolderNameWithTag(feature.DownloadedFeature.FeatureFolderName, feature.DownloadedFeature.Tag),
+			feature.DownloadedFeature.FeatureFolderName, "install.sh")
+		dockerFile += fmt.Sprintf("\nRUN%s chmod +x %s && %s",
+			optionEnvVariables, installScriptPath, installScriptPath)
+	}
+
+	log.Debug().Msgf("generated dockerfile for build context %s\n%s", buildContextPath, dockerFile)
+
+	file, err := os.OpenFile(filepath.Join(buildContextPath, "Dockerfile"), os.O_CREATE|os.O_RDWR, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to create Dockerfile: %w", err)
+	}
+	defer file.Close()
+
+	_, err = file.WriteString(dockerFile)
+	if err != nil {
+		return fmt.Errorf("failed to write content to Dockerfile: %w", err)
+	}
+
+	return nil
+}

app/gitspace/orchestrator/utils/download_features.go

@@ -27,6 +27,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/harness/gitness/types"
 	"github.com/harness/gitness/types/enum"
@@ -67,6 +68,10 @@ func DownloadFeatures(
 		downloadQueue <- featureSource{sourceURL: key, sourceType: value.SourceType}
 	}
 
+	// TODO: Add ctx based cancellations to the below goroutines.
+
+	// NOTE: The following logic might see performance issues with spikes in memory and CPU usage.
+	// If there are such issues, we can introduce throttling on the basis of memory, CPU, etc.
 	go func() {
 		for source := range downloadQueue {
 			startCh <- 1
@@ -102,6 +107,8 @@ waitLoop:
 		default:
 			if totalEnd > 0 && totalStart == totalEnd {
 				break waitLoop
+			} else {
+				time.Sleep(time.Millisecond * 10)
 			}
 		}
 	}
@@ -183,6 +190,7 @@ func downloadFeature(
 
 	downloadedFeature := types.DownloadedFeature{
 		FeatureFolderName:         featureFolderName,
+		Source:                    source.sourceURL,
 		SourceWithoutTag:          sourceWithoutTag,
 		Tag:                       featureTag,
 		CanonicalName:             canonicalName,
@@ -219,10 +227,13 @@ func getDevcontainerFeatureConfig(
 	if err != nil {
 		return nil, fmt.Errorf("error unpacking tarball for feature %s: %w", source.sourceURL, err)
 	}
+
+	// Delete the tarball to avoid unnecessary packaging and copying during docker build.
 	err = os.Remove(filepath.Join(downloadDirectory, tarballName))
 	if err != nil {
 		return nil, fmt.Errorf("error deleting tarball for feature %s: %w", source.sourceURL, err)
 	}
+
 	devcontainerFeatureRaw, err := os.ReadFile(filepath.Join(dst, "devcontainer-feature.json"))
 	if err != nil {
 		return nil, fmt.Errorf("error reading devcontainer-feature.json file for feature %s: %w",
@@ -251,9 +262,6 @@ func getFeatureDownloadDirectory(gitspaceInstanceIdentifier, featureFolderName,
 }
 
 func getFeatureFolderNameWithTag(featureFolderName string, featureTag string) string {
-	if featureTag == "" {
-		return featureFolderName
-	}
 	return featureFolderName + "-" + featureTag
 }
 

app/gitspace/orchestrator/utils/sort_features.go

@@ -0,0 +1,208 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"fmt"
+	"slices"
+
+	"github.com/harness/gitness/types"
+)
+
+type node struct {
+	Digest       string
+	Dependencies map[string]bool
+	Priority     int
+	Feature      *types.ResolvedFeature
+}
+
+type featureWithPriority struct {
+	Priority int
+	Feature  *types.ResolvedFeature
+}
+
+// SortFeatures sorts the features topologically, using round priorities and feature installation order override
+// if provided by the user. It also keeps track of hard dependencies and soft dependencies.
+// Reference: https://containers.dev/implementors/features/#dependency-installation-order-algorithm.
+func SortFeatures(
+	featuresToBeInstalled map[string]*types.ResolvedFeature,
+	overrideFeatureInstallOrder []string,
+) ([]*types.ResolvedFeature, error) {
+	sourcesWithoutTagsMappedToDigests := getSourcesWithoutTagsMappedToDigests(featuresToBeInstalled)
+
+	adjacencyList, err := buildAdjacencyList(featuresToBeInstalled, sourcesWithoutTagsMappedToDigests,
+		overrideFeatureInstallOrder)
+	if err != nil {
+		return nil, err
+	}
+
+	sortedFeatures, err := applyTopologicalSorting(adjacencyList)
+	if err != nil {
+		return nil, err
+	}
+
+	return sortedFeatures, nil
+}
+
+// getSourcesWithoutTagsMappedToDigests is used to map feature source (without tags) to their digests.
+// Multiple features in the install set can have the same source but different options. All these features
+// will be mapped to the same source and must be installed before any dependent features.
+func getSourcesWithoutTagsMappedToDigests(featuresToBeInstalled map[string]*types.ResolvedFeature) map[string][]string {
+	sourcesWithoutTagsMappedToDigests := map[string][]string{}
+	for _, featureToBeInstalled := range featuresToBeInstalled {
+		sourceWithoutTag := featureToBeInstalled.DownloadedFeature.SourceWithoutTag
+		if _, initialized := sourcesWithoutTagsMappedToDigests[sourceWithoutTag]; !initialized {
+			sourcesWithoutTagsMappedToDigests[sourceWithoutTag] = []string{}
+		}
+		sourcesWithoutTagsMappedToDigests[sourceWithoutTag] =
+			append(sourcesWithoutTagsMappedToDigests[sourceWithoutTag], featureToBeInstalled.Digest)
+	}
+	return sourcesWithoutTagsMappedToDigests
+}
+
+func buildAdjacencyList(
+	featuresToBeInstalled map[string]*types.ResolvedFeature,
+	sourcesWithoutTagsMappedToDigests map[string][]string,
+	overrideFeatureInstallOrder []string,
+) ([]*node, error) {
+	counter := 0
+	adjacencyList := make([]*node, 0)
+	for _, featureToBeInstalled := range featuresToBeInstalled {
+		dependencies := map[string]bool{}
+
+		err := populateHardDependencies(featureToBeInstalled, dependencies)
+		if err != nil {
+			return nil, err
+		}
+
+		populateSoftDependencies(sourcesWithoutTagsMappedToDigests, featureToBeInstalled, dependencies)
+
+		// While the default priority is 0, it can be varied by the user through the overrideFeatureInstallOrder
+		// in the devcontainer.json.
+		// Reference: https://containers.dev/implementors/features/#overrideFeatureInstallOrder.
+		priority := 0
+		index := slices.Index(overrideFeatureInstallOrder, featureToBeInstalled.DownloadedFeature.SourceWithoutTag)
+		if index > -1 {
+			priority = len(overrideFeatureInstallOrder) - index
+			counter++
+		}
+
+		graphNode := node{
+			Digest:       featureToBeInstalled.Digest,
+			Dependencies: dependencies,
+			Priority:     priority,
+			Feature:      featureToBeInstalled,
+		}
+
+		adjacencyList = append(adjacencyList, &graphNode)
+	}
+
+	// If any feature mentioned by the user in the overrideFeatureInstallOrder is not present in the install set,
+	// fail the flow.
+	difference := len(overrideFeatureInstallOrder) - counter
+	if difference > 0 {
+		return nil, fmt.Errorf("overrideFeatureInstallOrder contains %d extra features", difference)
+	}
+
+	return adjacencyList, nil
+}
+
+// populateSoftDependencies populates the digests of all the features whose source name is present in the installAfter
+// property for the current feature ie which must be installed before the current feature can be installed.
+// Any feature mentioned in the installAfter but not part of the install set is ignored.
+func populateSoftDependencies(
+	sourcesWithoutTagsMappedToDigests map[string][]string,
+	featureToBeInstalled *types.ResolvedFeature,
+	dependencies map[string]bool,
+) {
+	softDependencies := featureToBeInstalled.DownloadedFeature.DevcontainerFeatureConfig.InstallsAfter
+	if len(softDependencies) > 0 {
+		for _, softDependency := range softDependencies {
+			if digests, ok := sourcesWithoutTagsMappedToDigests[softDependency]; ok {
+				for _, digest := range digests {
+					if _, alreadyAdded := dependencies[digest]; !alreadyAdded {
+						dependencies[digest] = true
+					}
+				}
+			}
+		}
+	}
+}
+
+// populateHardDependencies populates the digests of all the features which must be installed before the current
+// feature can be installed.
+func populateHardDependencies(featureToBeInstalled *types.ResolvedFeature, dependencies map[string]bool) error {
+	hardDependencies := featureToBeInstalled.DownloadedFeature.DevcontainerFeatureConfig.DependsOn
+	if hardDependencies != nil && len(*hardDependencies) > 0 {
+		for _, hardDependency := range *hardDependencies {
+			digest, err := calculateDigest(hardDependency.Source, hardDependency.Options)
+			if err != nil {
+				return fmt.Errorf("error calculating digest for %s: %w", hardDependency.Source, err)
+			}
+			dependencies[digest] = true
+		}
+	}
+	return nil
+}
+
+func applyTopologicalSorting(
+	adjacencyList []*node,
+) ([]*types.ResolvedFeature, error) {
+	sortedFeatures := make([]*types.ResolvedFeature, 0)
+	for len(sortedFeatures) < len(adjacencyList) {
+		maxPriority, eligibleFeatures := getFeaturesEligibleInThisRound(adjacencyList)
+
+		if len(eligibleFeatures) == 0 {
+			return nil, fmt.Errorf("features can not be sorted")
+		}
+
+		selectedFeatures := []*types.ResolvedFeature{}
+
+		// only select those features which have the max priority, rest will be picked up in the next iteration.
+		for _, eligibleFeature := range eligibleFeatures {
+			if eligibleFeature.Priority == maxPriority {
+				selectedFeatures = append(selectedFeatures, eligibleFeature.Feature)
+			}
+		}
+
+		slices.SortStableFunc(selectedFeatures, types.CompareResolvedFeature)
+
+		for _, selectedFeature := range selectedFeatures {
+			sortedFeatures = append(sortedFeatures, selectedFeature)
+			for _, vertex := range adjacencyList {
+				delete(vertex.Dependencies, selectedFeature.Digest)
+			}
+		}
+	}
+
+	return sortedFeatures, nil
+}
+
+func getFeaturesEligibleInThisRound(adjacencyList []*node) (int, []featureWithPriority) {
+	maxPriorityInRound := 0
+	eligibleFeatures := []featureWithPriority{}
+	for _, vertex := range adjacencyList {
+		if len(vertex.Dependencies) == 0 {
+			eligibleFeatures = append(eligibleFeatures, featureWithPriority{
+				Priority: vertex.Priority,
+				Feature:  vertex.Feature,
+			})
+			if maxPriorityInRound < vertex.Priority {
+				maxPriorityInRound = vertex.Priority
+			}
+		}
+	}
+	return maxPriorityInRound, eligibleFeatures
+}

go.mod

@@ -4,6 +4,7 @@ go 1.22.0
 
 require (
 	cloud.google.com/go/storage v1.43.0
+	github.com/Masterminds/semver/v3 v3.3.1
 	github.com/Masterminds/squirrel v1.5.4
 	github.com/adrg/xdg v0.5.0
 	github.com/aws/aws-sdk-go v1.55.2
@@ -98,7 +99,6 @@ require (
 	github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e // indirect
 	github.com/BobuSumisu/aho-corasick v1.0.3 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
-	github.com/Masterminds/semver/v3 v3.3.1 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/antonmedv/expr v1.15.5 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect

types/devcontainer_config.go

@@ -244,3 +244,13 @@ type Mount struct {
 	Target string `json:"target,omitempty"`
 	Type   string `json:"type,omitempty"`
 }
+
+func (m *Mount) UnmarshalJSON(data []byte) error {
+	// TODO: Add support for unmarshalling mount data from a string input
+	var mount Mount
+	err := json.Unmarshal(data, &mount)
+	if err != nil {
+		return err
+	}
+	return nil
+}

types/devcontainer_feature_config.go

@@ -32,7 +32,6 @@ type DevcontainerFeatureConfig struct {
 	ContainerEnv      map[string]string `json:"containerEnv,omitempty"`
 	Privileged        bool              `json:"privileged,omitempty"`
 	Init              bool              `json:"init,omitempty"`
-	Deprecated        bool              `json:"deprecated,omitempty"`
 	CapAdd            []string          `json:"capAdd,omitempty"`
 	SecurityOpt       []string          `json:"securityOpt,omitempty"`
 	Entrypoint        string            `json:"entrypoint,omitempty"`

types/resolved_feature.go

@@ -15,6 +15,7 @@
 package types
 
 import (
+	"fmt"
 	"sort"
 	"strconv"
 	"strings"
@@ -31,12 +32,21 @@ type ResolvedFeature struct {
 
 type DownloadedFeature struct {
 	FeatureFolderName         string                     `json:"feature_folder_name,omitempty"`
+	Source                    string                     `json:"source,omitempty"`
 	SourceWithoutTag          string                     `json:"source_without_tag,omitempty"`
 	Tag                       string                     `json:"tag,omitempty"`
 	CanonicalName             string                     `json:"canonical_name,omitempty"`
 	DevcontainerFeatureConfig *DevcontainerFeatureConfig `json:"devcontainer_feature_config,omitempty"`
 }
 
+func (r *ResolvedFeature) Print() string {
+	options := make([]string, 0, len(r.ResolvedOptions))
+	for key, value := range r.ResolvedOptions {
+		options = append(options, fmt.Sprintf("%s=%s", key, value))
+	}
+	return fmt.Sprintf("%s %+v", r.DownloadedFeature.Source, options)
+}
+
 // CompareResolvedFeature implements the following comparison rules.
 // 1. Compare and sort each Feature lexicographically by their fully qualified resource name
 // (For OCI-published Features, that means the ID without version or digest.). If the comparison is equal: