diff --git a/app/gitspace/orchestrator/utils/build_with_features.go b/app/gitspace/orchestrator/utils/build_with_features.go new file mode 100644 index 000000000..3c1a10735 --- /dev/null +++ b/app/gitspace/orchestrator/utils/build_with_features.go @@ -0,0 +1,38 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "regexp" + "strings" +) + +// ConvertOptionsToEnvVariables converts the option keys to standardised env variables using the +// devcontainer specification to ensure uniformity in the naming and casing of the env variables. +// Reference: https://containers.dev/implementors/features/#option-resolution +func ConvertOptionsToEnvVariables(str string) string { + // Replace all non-alphanumeric characters (excluding underscores) with '_' + reNonAlnum := regexp.MustCompile(`[^\w_]`) + str = reNonAlnum.ReplaceAllString(str, "_") + + // Replace leading digits or underscores with a single '_' + reLeadingDigitsOrUnderscores := regexp.MustCompile(`^[\d_]+`) + str = reLeadingDigitsOrUnderscores.ReplaceAllString(str, "_") + + // Convert the string to uppercase + str = strings.ToUpper(str) + + return str +} diff --git a/app/gitspace/orchestrator/utils/download_features.go b/app/gitspace/orchestrator/utils/download_features.go new file mode 100644 index 000000000..07a8d562f --- /dev/null +++ b/app/gitspace/orchestrator/utils/download_features.go @@ -0,0 +1,378 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "archive/tar" + "context" + "encoding/json" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" + + "github.com/tidwall/jsonc" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/registry/remote" +) + +type featureSource struct { + sourceURL string + sourceType enum.FeatureSourceType +} + +// DownloadFeatures downloads the user specified features and all the features on which the user defined features +// depend. It does it by checking the dependencies of a downloaded feature from the devcontainer-feature.json +// and adding that feature to the download queue, if it is not already marked for download. +func DownloadFeatures( + ctx context.Context, + gitspaceInstanceIdentifier string, + features types.Features, +) (*map[string]*types.DownloadedFeature, error) { + downloadedFeatures := sync.Map{} + featuresToBeDownloaded := sync.Map{} + downloadQueue := make(chan featureSource, 100) + errorCh := make(chan error, 100) + + // startCh and endCh are used to check if all the goroutines spawned to download features have completed or not. + // Whenever a new goroutine is spawned, it increments the counter listening to the startCh. + // Upon completion, it increments the counter listening to the endCh. + // When the start count == end count and end count is > 0, it means all the goroutines have completed execution. + startCh := make(chan int, 100) + endCh := make(chan int, 100) + + for key, value := range features { + featuresToBeDownloaded.Store(key, value) + downloadQueue <- featureSource{sourceURL: key, sourceType: value.SourceType} + } + + go func() { + for source := range downloadQueue { + startCh <- 1 + go func(source featureSource) { + defer func(endCh chan int) { endCh <- 1 }(endCh) + err := downloadFeature(ctx, gitspaceInstanceIdentifier, &source, &featuresToBeDownloaded, + downloadQueue, &downloadedFeatures) + errorCh <- err + }(source) + } + }() + + var totalStart int + var totalEnd int + var downloadError error +waitLoop: + for { + select { + case start := <-startCh: + totalStart += start + case end := <-endCh: + totalEnd += end + case err := <-errorCh: + if err == nil { + continue + } + if downloadError != nil { + downloadError = fmt.Errorf("error downloading features: %w\n%w", err, downloadError) + } else { + downloadError = fmt.Errorf("error downloading features: %w", err) + } + + default: + if totalEnd > 0 && totalStart == totalEnd { + break waitLoop + } + } + } + + close(startCh) + close(endCh) + close(downloadQueue) + + if downloadError != nil { + return nil, downloadError + } + + var finalMap = make(map[string]*types.DownloadedFeature) + + var saveToMapFunc = func(key, value any) bool { + finalMap[key.(string)] = value.(*types.DownloadedFeature) // nolint:errcheck + return true + } + + downloadedFeatures.Range(saveToMapFunc) + return &finalMap, nil +} + +// downloadFeature downloads a single feature. Depending on the source type, it either downloads it from an OCI +// repo or an HTTP(S) URL. It then fetches its devcontainer-feature.json and also checks which dependencies need +// to be downloaded for this feature. +func downloadFeature( + ctx context.Context, + gitspaceInstanceIdentifier string, + source *featureSource, + featuresToBeDownloaded *sync.Map, + downloadQueue chan featureSource, + downloadedFeatures *sync.Map, +) error { + var tarballName string + var featureFolderName string + var featureTag string + var sourceWithoutTag string + var canonicalName string + var downloadDirectory string + switch source.sourceType { + case enum.FeatureSourceTypeOCI: + sourceWithoutTag = strings.SplitN(source.sourceURL, ":", 2)[0] + partialFeatureNameWithTag := filepath.Base(source.sourceURL) + parts := strings.SplitN(partialFeatureNameWithTag, ":", 2) + tarballName = fmt.Sprintf("devcontainer-feature-%s.tgz", parts[0]) + featureFolderName = strings.TrimSuffix(tarballName, ".tgz") + featureTag = parts[1] + downloadDirectory = getFeatureDownloadDirectory(gitspaceInstanceIdentifier, featureFolderName, featureTag) + contentDigest, err := downloadTarballFromOCIRepo(ctx, source.sourceURL, downloadDirectory) + if err != nil { + return fmt.Errorf("error downloading oci artifact for feature %s: %w", source.sourceURL, err) + } + canonicalName = contentDigest + + case enum.FeatureSourceTypeTarball: + sourceWithoutTag = source.sourceURL + canonicalName = source.sourceURL + featureTag = types.FeatureDefaultTag // using static tag for comparison + tarballURL, err := url.Parse(source.sourceURL) + if err != nil { + return fmt.Errorf("error parsing feature URL for feature %s: %w", source.sourceURL, err) + } + tarballName = filepath.Base(tarballURL.Path) + featureFolderName = strings.TrimSuffix(tarballName, ".tgz") + downloadDirectory = getFeatureDownloadDirectory(gitspaceInstanceIdentifier, featureFolderName, featureTag) + err = downloadTarball(source.sourceURL, downloadDirectory, filepath.Join(downloadDirectory, tarballName)) + if err != nil { + return fmt.Errorf("error downloading tarball for feature %s: %w", source.sourceURL, err) + } + default: + return fmt.Errorf("unsupported feature type: %s", source.sourceType) + } + + devcontainerFeature, err := getDevcontainerFeatureConfig(downloadDirectory, featureFolderName, tarballName, source) + if err != nil { + return err + } + + downloadedFeature := types.DownloadedFeature{ + FeatureFolderName: featureFolderName, + SourceWithoutTag: sourceWithoutTag, + Tag: featureTag, + CanonicalName: canonicalName, + DevcontainerFeatureConfig: devcontainerFeature, + } + + downloadedFeatures.Store(source.sourceURL, &downloadedFeature) + + // Check all the dependencies which are required for this feature. If any, check if they are already marked + // for downloaded. If not, push to the download queue. + if downloadedFeature.DevcontainerFeatureConfig.DependsOn != nil && + len(*downloadedFeature.DevcontainerFeatureConfig.DependsOn) > 0 { + for key, value := range *downloadedFeature.DevcontainerFeatureConfig.DependsOn { + _, present := featuresToBeDownloaded.LoadOrStore(key, value) + if !present { + downloadQueue <- featureSource{sourceURL: key, sourceType: value.SourceType} + } + } + } + + return nil +} + +// getDevcontainerFeatureConfig returns the devcontainer-feature.json by unpacking the downloaded tarball, +// unmarshalling the file contents to types.DevcontainerFeatureConfig. It removes any comments before unmarshalling. +func getDevcontainerFeatureConfig( + downloadDirectory string, + featureName string, + tarballName string, + source *featureSource, +) (*types.DevcontainerFeatureConfig, error) { + dst := filepath.Join(downloadDirectory, featureName) + err := unpackTarball(filepath.Join(downloadDirectory, tarballName), dst) + if err != nil { + return nil, fmt.Errorf("error unpacking tarball for feature %s: %w", source.sourceURL, err) + } + err = os.Remove(filepath.Join(downloadDirectory, tarballName)) + if err != nil { + return nil, fmt.Errorf("error deleting tarball for feature %s: %w", source.sourceURL, err) + } + devcontainerFeatureRaw, err := os.ReadFile(filepath.Join(dst, "devcontainer-feature.json")) + if err != nil { + return nil, fmt.Errorf("error reading devcontainer-feature.json file for feature %s: %w", + source.sourceURL, err) + } + var devcontainerFeature types.DevcontainerFeatureConfig + err = json.Unmarshal(jsonc.ToJSON(devcontainerFeatureRaw), &devcontainerFeature) + if err != nil { + return nil, fmt.Errorf("error parsing devcontainer-feature.json for feature %s: %w", + source.sourceURL, err) + } + return &devcontainerFeature, nil +} + +func getGitspaceInstanceDirectory(gitspaceInstanceIdentifier string) string { + return filepath.Join("/tmp", gitspaceInstanceIdentifier) +} + +func getFeaturesDownloadDirectory(gitspaceInstanceIdentifier string) string { + return filepath.Join(getGitspaceInstanceDirectory(gitspaceInstanceIdentifier), "devcontainer-features") +} + +func getFeatureDownloadDirectory(gitspaceInstanceIdentifier, featureFolderName, featureTag string) string { + return filepath.Join(getFeaturesDownloadDirectory(gitspaceInstanceIdentifier), + getFeatureFolderNameWithTag(featureFolderName, featureTag)) +} + +func getFeatureFolderNameWithTag(featureFolderName string, featureTag string) string { + if featureTag == "" { + return featureFolderName + } + return featureFolderName + "-" + featureTag +} + +func downloadTarballFromOCIRepo(ctx context.Context, ociRepo string, filePath string) (string, error) { + parts := strings.SplitN(ociRepo, ":", 2) + if len(parts) != 2 { + return "", fmt.Errorf("invalid oci repo: %s", ociRepo) + } + fs, err := file.New(filePath) + if err != nil { + return "", err + } + defer fs.Close() + repo, err := remote.NewRepository(parts[0]) + if err != nil { + return "", err + } + tag := parts[1] + md, err := oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions) + if err != nil { + return "", err + } + return md.Digest.String(), nil +} + +func downloadTarball(url, dirPath, fileName string) error { + if _, err := os.Stat(dirPath); os.IsNotExist(err) { + err := os.MkdirAll(dirPath, fs.ModePerm) + if err != nil { + return err + } + } + out, err := os.Create(fileName) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer out.Close() + + resp, err := http.Get(url) // nolint:gosec,noctx + if err != nil { + return fmt.Errorf("failed to download tarball: %w", err) + } + defer resp.Body.Close() + + // Check the HTTP response status + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to download tarball: HTTP status %d", resp.StatusCode) + } + + // Copy the response body to the file + _, err = io.Copy(out, resp.Body) + if err != nil { + return fmt.Errorf("failed to save tarball: %w", err) + } + return nil +} + +// unpackTarball extracts a .tgz file to a specified output directory. +func unpackTarball(tarball, outputDir string) error { + // Open the tarball file + file, err := os.Open(tarball) + if err != nil { + return fmt.Errorf("failed to open tarball: %w", err) + } + defer file.Close() + + // Create a tar reader + tarReader := tar.NewReader(file) + + // Iterate through the files in the tar archive + for { + header, err := tarReader.Next() + if err == io.EOF { + break // End of tar archive + } + if err != nil { + return fmt.Errorf("failed to read tar header: %w", err) + } + + // Determine the file's full path + targetPath := filepath.Join(outputDir, header.Name) // nolint:gosec + + switch header.Typeflag { + case tar.TypeDir: + // Create the directory if it doesn't exist + if err := os.MkdirAll(targetPath, os.FileMode(header.Mode)); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + case tar.TypeReg: + // Extract the file + if err := extractFile(tarReader, targetPath, header.Mode); err != nil { + return err + } + default: + return fmt.Errorf("skipping unsupported type: %c in %s", header.Typeflag, header.Name) + } + } + return nil +} + +// extractFile writes the content of a file from the tar archive. +func extractFile(tarReader *tar.Reader, targetPath string, mode int64) error { + if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { + return fmt.Errorf("failed to create parent directories: %w", err) + } + + outFile, err := os.Create(targetPath) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer outFile.Close() + + if _, err := io.Copy(outFile, tarReader); err != nil { + return fmt.Errorf("failed to copy file content: %w", err) + } + + if err := os.Chmod(targetPath, os.FileMode(mode)); err != nil { + return fmt.Errorf("failed to set file permissions: %w", err) + } + + return nil +} diff --git a/app/gitspace/orchestrator/utils/resolve_features.go b/app/gitspace/orchestrator/utils/resolve_features.go new file mode 100644 index 000000000..ff4a0d162 --- /dev/null +++ b/app/gitspace/orchestrator/utils/resolve_features.go @@ -0,0 +1,112 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + + "github.com/harness/gitness/types" +) + +// ResolveFeatures resolves all the downloaded features ie starting from the user-specified features, it checks +// which features need to be installed with which options. A feature in considered uniquely installable if its +// id or source and the options overrides are unique. +// Reference: https://containers.dev/implementors/features/#definition-feature-equality +func ResolveFeatures( + userDefinedFeatures types.Features, + downloadedFeatures map[string]*types.DownloadedFeature, +) (map[string]*types.ResolvedFeature, error) { + featuresToBeResolved := make([]types.FeatureValue, 0) + for _, featureValue := range userDefinedFeatures { + featuresToBeResolved = append(featuresToBeResolved, *featureValue) + } + + resolvedFeatures := make(map[string]*types.ResolvedFeature) + for i := 0; i < len(featuresToBeResolved); i++ { + currentFeature := featuresToBeResolved[i] + + digest, err := calculateDigest(currentFeature.Source, currentFeature.Options) + if err != nil { + return nil, fmt.Errorf("error calculating digest for %s: %w", currentFeature.Source, err) + } + + if _, alreadyResolved := resolvedFeatures[digest]; alreadyResolved { + continue + } + + downloadedFeature := downloadedFeatures[currentFeature.Source] + resolvedOptions, err := getResolvedOptions(downloadedFeature, currentFeature) + if err != nil { + return nil, err + } + + resolvedFeature := types.ResolvedFeature{ + Digest: digest, + ResolvedOptions: resolvedOptions, + OverriddenOptions: currentFeature.Options, // used to calculate digest and sort features + DownloadedFeature: downloadedFeature, + } + + resolvedFeatures[digest] = &resolvedFeature + if resolvedFeature.DownloadedFeature.DevcontainerFeatureConfig.DependsOn != nil && + len(*resolvedFeature.DownloadedFeature.DevcontainerFeatureConfig.DependsOn) > 0 { + for _, featureValue := range *resolvedFeature.DownloadedFeature.DevcontainerFeatureConfig.DependsOn { + featuresToBeResolved = append(featuresToBeResolved, *featureValue) + } + } + } + return resolvedFeatures, nil +} + +func getResolvedOptions( + downloadedFeature *types.DownloadedFeature, + currentFeature types.FeatureValue, +) (map[string]string, error) { + resolvedOptions := make(map[string]string) + if downloadedFeature.DevcontainerFeatureConfig.Options != nil { + for optionKey, optionDefinition := range *downloadedFeature.DevcontainerFeatureConfig.Options { + var optionValue = optionDefinition.Default + if userProvidedOptionValue, ok := currentFeature.Options[optionKey]; ok { + optionValue = userProvidedOptionValue + } + stringValue, err := optionDefinition.ValidateValue(optionValue, optionKey, currentFeature.Source) + if err != nil { + return nil, err + } + resolvedOptions[optionKey] = stringValue + } + } + return resolvedOptions, nil +} + +// calculateDigest calculates a deterministic hash for a feature using its source and options overrides. +func calculateDigest(source string, optionsOverrides map[string]any) (string, error) { + data := map[string]any{ + "options": optionsOverrides, + "source": source, + } + + jsonBytes, err := json.Marshal(data) + if err != nil { + return "", fmt.Errorf("failed to serialize data: %w", err) + } + + hash := sha256.Sum256(jsonBytes) + + return hex.EncodeToString(hash[:]), nil +} diff --git a/app/gitspace/scm/scm.go b/app/gitspace/scm/scm.go index f9c245b35..1da08c5e3 100644 --- a/app/gitspace/scm/scm.go +++ b/app/gitspace/scm/scm.go @@ -26,6 +26,8 @@ import ( "github.com/harness/gitness/git/command" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" + + "github.com/tidwall/jsonc" ) var ( @@ -102,13 +104,6 @@ func (s *SCM) GetSCMRepoDetails( return resolvedDetails, nil } -func removeComments(input []byte) []byte { - blockCommentRegex := regexp.MustCompile(`(?s)/\*.*?\*/`) - input = blockCommentRegex.ReplaceAll(input, nil) - lineCommentRegex := regexp.MustCompile(`//.*`) - return lineCommentRegex.ReplaceAll(input, nil) -} - func detectDefaultGitBranch(ctx context.Context, gitRepoDir string) (string, error) { cmd := command.New("ls-remote", command.WithFlag("--symref"), @@ -192,8 +187,7 @@ func (s *SCM) getDevcontainerConfig( return config, nil // Return an empty config if the file is empty } - sanitizedJSON := removeComments(catFileOutputBytes) - if err = json.Unmarshal(sanitizedJSON, &config); err != nil { + if err = json.Unmarshal(jsonc.ToJSON(catFileOutputBytes), &config); err != nil { return config, fmt.Errorf("failed to parse devcontainer JSON: %w", err) } diff --git a/go.mod b/go.mod index eb1625974..c68878c06 100644 --- a/go.mod +++ b/go.mod @@ -71,6 +71,7 @@ require ( github.com/swaggest/swgui v1.8.1 github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.2 + github.com/tidwall/jsonc v0.3.2 github.com/unrolled/secure v1.15.0 github.com/zricethezav/gitleaks/v8 v8.18.5-0.20240912004812-e93a7c0d2604 go.starlark.net v0.0.0-20231121155337-90ade8b19d09 @@ -84,6 +85,7 @@ require ( google.golang.org/api v0.189.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/mail.v2 v2.3.1 + oras.land/oras-go/v2 v2.5.0 ) require ( @@ -96,6 +98,7 @@ require ( github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e // indirect github.com/BobuSumisu/aho-corasick v1.0.3 // indirect github.com/KyleBanks/depth v1.2.1 // indirect + github.com/Masterminds/semver/v3 v3.3.1 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/antonmedv/expr v1.15.5 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index cf57c4d10..f7231b9c7 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= +github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= @@ -760,6 +762,8 @@ github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64 github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ= github.com/swaggo/swag v1.16.2 h1:28Pp+8DkQoV+HLzLx8RGJZXNGKbFqnuvSbAAtoxiY04= github.com/swaggo/swag v1.16.2/go.mod h1:6YzXnDcpr0767iOejs318CwYkCQqyGer6BizOg03f+E= +github.com/tidwall/jsonc v0.3.2 h1:ZTKrmejRlAJYdn0kcaFqRAKlxxFIC21pYq8vLa4p2Wc= +github.com/tidwall/jsonc v0.3.2/go.mod h1:dw+3CIxqHi+t8eFSpzzMlcVYxKp08UP5CD8/uSFCyJE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= @@ -1087,6 +1091,8 @@ k8s.io/api v0.0.0-20181130031204-d04500c8c3dd/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j k8s.io/apimachinery v0.0.0-20181201231028-18a5ff3097b4/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/client-go v9.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= +oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/types/devcontainer_config.go b/types/devcontainer_config.go index 27d818f56..e8b229f43 100644 --- a/types/devcontainer_config.go +++ b/types/devcontainer_config.go @@ -17,20 +17,34 @@ package types import ( "encoding/json" "fmt" + "net/url" "strings" + + "github.com/harness/gitness/types/enum" + + "oras.land/oras-go/v2/registry" ) +const FeatureDefaultTag = "latest" + //nolint:tagliatelle type DevcontainerConfig struct { - Image string `json:"image,omitempty"` - PostCreateCommand LifecycleCommand `json:"postCreateCommand,omitempty"` - PostStartCommand LifecycleCommand `json:"postStartCommand,omitempty"` - ForwardPorts []json.Number `json:"forwardPorts,omitempty"` - ContainerEnv map[string]string `json:"containerEnv,omitempty"` - Customizations DevContainerConfigCustomizations `json:"customizations,omitempty"` - RunArgs []string `json:"runArgs,omitempty"` - ContainerUser string `json:"containerUser,omitempty"` - RemoteUser string `json:"remoteUser,omitempty"` + Image string `json:"image,omitempty"` + PostCreateCommand LifecycleCommand `json:"postCreateCommand,omitempty"` + PostStartCommand LifecycleCommand `json:"postStartCommand,omitempty"` + ForwardPorts []json.Number `json:"forwardPorts,omitempty"` + ContainerEnv map[string]string `json:"containerEnv,omitempty"` + Customizations DevContainerConfigCustomizations `json:"customizations,omitempty"` + RunArgs []string `json:"runArgs,omitempty"` + ContainerUser string `json:"containerUser,omitempty"` + RemoteUser string `json:"remoteUser,omitempty"` + Features *Features `json:"features,omitempty"` + OverrideFeatureInstallOrder []string `json:"overrideFeatureInstallOrder,omitempty"` + Privileged bool `json:"privileged,omitempty"` + Init bool `json:"init,omitempty"` + CapAdd []string `json:"capAdd,omitempty"` + SecurityOpt []string `json:"securityOpt,omitempty"` + Mounts []*Mount `json:"mounts,omitempty"` } // Constants for discriminator values. @@ -138,3 +152,95 @@ func (lc *LifecycleCommand) ToCommandArray() []string { return nil } } + +type Features map[string]*FeatureValue + +type FeatureValue struct { + Source string `json:"source,omitempty"` + SourceType enum.FeatureSourceType `json:"source_type,omitempty"` + Options map[string]any `json:"options,omitempty"` +} + +func (f *FeatureValue) UnmarshalJSON(data []byte) error { + var version string + if err := json.Unmarshal(data, &version); err == nil { + f.Options = make(map[string]any) + f.Options["version"] = version + return nil + } + + var options map[string]any + if err := json.Unmarshal(data, &options); err == nil { + for key, value := range options { + switch value.(type) { + case string, bool: + continue + default: + return fmt.Errorf("invalid type for option '%s': must be string or boolean, got %T", key, value) + } + } + f.Options = options + return nil + } + + return nil +} + +func (f *Features) UnmarshalJSON(data []byte) error { + if *f == nil { + *f = make(Features) + } + + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + for key, value := range raw { + sanitizedSource, sourceType, validationErr := validateFeatureSource(key) + if validationErr != nil { + return validationErr + } + feature := &FeatureValue{Source: sanitizedSource, SourceType: sourceType} + if err := json.Unmarshal(value, feature); err != nil { + return fmt.Errorf("failed to unmarshal feature '%s': %w", key, err) + } + (*f)[sanitizedSource] = feature + } + + return nil +} + +func validateFeatureSource(source string) (string, enum.FeatureSourceType, error) { + if _, err := registry.ParseReference(source); err == nil { + indexOfSeparator := strings.Index(source, ":") + if indexOfSeparator == -1 { + source += ":" + FeatureDefaultTag + } + return source, enum.FeatureSourceTypeOCI, nil + } + if err := validateTarballURL(source); err == nil { + return source, enum.FeatureSourceTypeTarball, nil + } + return source, enum.FeatureSourceTypeLocal, fmt.Errorf("unsupported feature source: %s", source) +} + +func validateTarballURL(source string) error { + tarballURL, err := url.Parse(source) + if err != nil { + return fmt.Errorf("parsing feature URL: %w", err) + } + if tarballURL.Scheme != "http" && tarballURL.Scheme != "https" { + return fmt.Errorf("invalid feature URL: %s", tarballURL.String()) + } + if !strings.HasSuffix(tarballURL.Path, ".tgz") { + return fmt.Errorf("invalid feature URL: %s", tarballURL.String()) + } + return nil +} + +type Mount struct { + Source string `json:"source,omitempty"` + Target string `json:"target,omitempty"` + Type string `json:"type,omitempty"` +} diff --git a/types/devcontainer_feature_config.go b/types/devcontainer_feature_config.go new file mode 100644 index 000000000..5e4fbcb19 --- /dev/null +++ b/types/devcontainer_feature_config.go @@ -0,0 +1,81 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "fmt" + "slices" + "strconv" + + "github.com/harness/gitness/types/enum" +) + +//nolint:tagliatelle +type DevcontainerFeatureConfig struct { + ID string `json:"id,omitempty"` + Version string `json:"version,omitempty"` + Name string `json:"name,omitempty"` + Options *Options `json:"options,omitempty"` + DependsOn *Features `json:"dependsOn,omitempty"` + ContainerEnv map[string]string `json:"containerEnv,omitempty"` + Privileged bool `json:"privileged,omitempty"` + Init bool `json:"init,omitempty"` + Deprecated bool `json:"deprecated,omitempty"` + CapAdd []string `json:"capAdd,omitempty"` + SecurityOpt []string `json:"securityOpt,omitempty"` + Entrypoint string `json:"entrypoint,omitempty"` + InstallsAfter []string `json:"installsAfter,omitempty"` + Mounts []*Mount `json:"mounts,omitempty"` + PostCreateCommand LifecycleCommand `json:"postCreateCommand,omitempty"` + PostStartCommand LifecycleCommand `json:"postStartCommand,omitempty"` +} + +type Options map[string]*OptionDefinition + +type OptionDefinition struct { + Type enum.FeatureOptionValueType `json:"type,omitempty"` + Proposals []string `json:"proposals,omitempty"` + Enum []string `json:"enum,omitempty"` + Default any `json:"default,omitempty"` + Description string `json:"description,omitempty"` +} + +// ValidateValue checks if the value matches the type defined in the definition. For string types, +// it also checks if it is allowed ie it is present in the enum array for the option. +// Reference: https://containers.dev/implementors/features/#options-property +func (o *OptionDefinition) ValidateValue(optionValue any, optionKey string, featureSource string) (string, error) { + switch o.Type { + case enum.FeatureOptionValueTypeBoolean: + boolValue, ok := optionValue.(bool) + if !ok { + return "", fmt.Errorf("error during resolving feature %s, option Id %s "+ + "expects boolean, got %s ", featureSource, optionKey, optionValue) + } + return strconv.FormatBool(boolValue), nil + case enum.FeatureOptionValueTypeString: + stringValue, ok := optionValue.(string) + if !ok { + return "", fmt.Errorf("error during resolving feature %s, option Id %s "+ + "expects string, got %s ", featureSource, optionKey, optionValue) + } + if len(o.Enum) > 0 && !slices.Contains(o.Enum, stringValue) { + return "", fmt.Errorf("error during resolving feature %s, option value %s "+ + "not allowed for Id %s ", featureSource, stringValue, optionKey) + } + return stringValue, nil + default: + return "", fmt.Errorf("unsupported option type %s", o.Type) + } +} diff --git a/types/enum/feature_option_value_type.go b/types/enum/feature_option_value_type.go new file mode 100644 index 000000000..bb8baccdd --- /dev/null +++ b/types/enum/feature_option_value_type.go @@ -0,0 +1,29 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enum + +type FeatureOptionValueType string + +func (FeatureOptionValueType) Enum() []interface{} { return toInterfaceSlice(featureOptionValueTypes) } + +const ( + FeatureOptionValueTypeString = "string" + FeatureOptionValueTypeBoolean = "boolean" +) + +var featureOptionValueTypes = sortEnum([]ExecutionSort{ + FeatureOptionValueTypeString, + FeatureOptionValueTypeBoolean, +}) diff --git a/types/enum/feature_source_type.go b/types/enum/feature_source_type.go new file mode 100644 index 000000000..ea3ec2332 --- /dev/null +++ b/types/enum/feature_source_type.go @@ -0,0 +1,31 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enum + +type FeatureSourceType string + +func (FeatureSourceType) Enum() []interface{} { return toInterfaceSlice(featureSourceTypes) } + +const ( + FeatureSourceTypeOCI = "oci" + FeatureSourceTypeTarball = "tarball" + FeatureSourceTypeLocal = "local" +) + +var featureSourceTypes = sortEnum([]ExecutionSort{ + FeatureSourceTypeOCI, + FeatureSourceTypeTarball, + FeatureSourceTypeLocal, +}) diff --git a/types/resolved_feature.go b/types/resolved_feature.go new file mode 100644 index 000000000..f36d8b3ba --- /dev/null +++ b/types/resolved_feature.go @@ -0,0 +1,137 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "sort" + "strconv" + "strings" + + "github.com/Masterminds/semver/v3" +) + +type ResolvedFeature struct { + ResolvedOptions map[string]string `json:"options,omitempty"` + OverriddenOptions map[string]any `json:"overridden_options,omitempty"` + Digest string `json:"string,omitempty"` + DownloadedFeature *DownloadedFeature `json:"downloaded_feature,omitempty"` +} + +type DownloadedFeature struct { + FeatureFolderName string `json:"feature_folder_name,omitempty"` + SourceWithoutTag string `json:"source_without_tag,omitempty"` + Tag string `json:"tag,omitempty"` + CanonicalName string `json:"canonical_name,omitempty"` + DevcontainerFeatureConfig *DevcontainerFeatureConfig `json:"devcontainer_feature_config,omitempty"` +} + +// CompareResolvedFeature implements the following comparison rules. +// 1. Compare and sort each Feature lexicographically by their fully qualified resource name +// (For OCI-published Features, that means the ID without version or digest.). If the comparison is equal: +// 2. Compare and sort each Feature from oldest to newest tag (latest being the “most new”). If the comparison is equal: +// 3. Compare and sort each Feature by their options by: +// 3.1 Greatest number of user-defined options (note omitting an option will default that value to the Feature’s +// default value and is not considered a user-defined option). If the comparison is equal: +// 3.2 Sort the provided option keys lexicographically. If the comparison is equal: +// 3.3 Sort the provided option values lexicographically. If the comparison is equal: +// 4. Sort Features by their canonical name (For OCI-published Features, the Feature ID resolved to the digest hash). +// 5. If there is no difference based on these comparator rules, the Features are considered equal. +// Reference: https://containers.dev/implementors/features/#definition-feature-equality (Round Stable Sort). +func CompareResolvedFeature(a, b *ResolvedFeature) int { + var comparison int + + comparison = strings.Compare(a.DownloadedFeature.SourceWithoutTag, b.DownloadedFeature.SourceWithoutTag) + if comparison != 0 { + return comparison + } + + comparison, _ = compareTags(a.DownloadedFeature.Tag, b.DownloadedFeature.Tag) + if comparison != 0 { + return comparison + } + + comparison = compareOverriddenOptions(a.OverriddenOptions, b.OverriddenOptions) + if comparison != 0 { + return comparison + } + + return strings.Compare(a.DownloadedFeature.CanonicalName, b.DownloadedFeature.CanonicalName) +} + +func compareTags(a, b string) (int, error) { + if a == FeatureDefaultTag && b == FeatureDefaultTag { + return 0, nil + } + if a == FeatureDefaultTag { + return 1, nil + } + if b == FeatureDefaultTag { + return -1, nil + } + versionA, err := semver.NewVersion(a) + if err != nil { + return 0, err + } + versionB, err := semver.NewVersion(b) + if err != nil { + return 0, err + } + return versionA.Compare(versionB), nil +} + +func compareOverriddenOptions(a, b map[string]any) int { + if len(a) != len(b) { + return len(a) - len(b) + } + + keysA, valuesA := getSortedOptions(a) + keysB, valuesB := getSortedOptions(b) + + for i := 0; i < len(keysA); i++ { + if keysA[i] == keysB[i] { + continue + } + return strings.Compare(keysA[i], keysB[i]) + } + + for i := 0; i < len(valuesA); i++ { + if valuesA[i] == valuesB[i] { + continue + } + return strings.Compare(valuesA[i], valuesB[i]) + } + + return 0 +} + +// getSortedOptions returns the keys and values of a map sorted lexicographically. +func getSortedOptions(m map[string]any) ([]string, []string) { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + sort.Strings(keys) + values := make([]string, 0, len(m)) + for _, key := range keys { + value := m[key] + switch v := value.(type) { + case string: + values = append(values, v) + case bool: + values = append(values, strconv.FormatBool(v)) + } + } + return keys, values +}