feat: [CDE-572]: Adding changes to support features. (#3244)

* Changes to support sorting and building.
* Updating go.mod
* feat: [CDE-572]: Adding changes to support features. Changes to support the features property in the devcontainer.json, download features from OCI repos and http(s) URLs as tarballs and parse them to devcontainer-feature.json, resolve them and apply user options. Also added a change to use an OS library to remove comments from the json files.
* feat: [CDE-572]: Adding changes to support features. Changes to support the features property in the devcontainer.json, download features from OCI repos and http(s) URLs as tarballs and parse them to devcontainer-feature.json, resolve them and apply user options. Also added a change to use an OS library to remove comments from the json files.
BT-10437
Dhruv Dhruv 2025-01-13 11:44:24 +00:00 committed by Harness
parent 21bb774275
commit 4d888b0719
11 changed files with 933 additions and 18 deletions

View File

@ -0,0 +1,38 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"regexp"
"strings"
)
// ConvertOptionsToEnvVariables converts the option keys to standardised env variables using the
// devcontainer specification to ensure uniformity in the naming and casing of the env variables.
// Reference: https://containers.dev/implementors/features/#option-resolution
func ConvertOptionsToEnvVariables(str string) string {
// Replace all non-alphanumeric characters (excluding underscores) with '_'
reNonAlnum := regexp.MustCompile(`[^\w_]`)
str = reNonAlnum.ReplaceAllString(str, "_")
// Replace leading digits or underscores with a single '_'
reLeadingDigitsOrUnderscores := regexp.MustCompile(`^[\d_]+`)
str = reLeadingDigitsOrUnderscores.ReplaceAllString(str, "_")
// Convert the string to uppercase
str = strings.ToUpper(str)
return str
}

View File

@ -0,0 +1,378 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"archive/tar"
"context"
"encoding/json"
"fmt"
"io"
"io/fs"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
"github.com/tidwall/jsonc"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/file"
"oras.land/oras-go/v2/registry/remote"
)
type featureSource struct {
sourceURL string
sourceType enum.FeatureSourceType
}
// DownloadFeatures downloads the user specified features and all the features on which the user defined features
// depend. It does it by checking the dependencies of a downloaded feature from the devcontainer-feature.json
// and adding that feature to the download queue, if it is not already marked for download.
func DownloadFeatures(
ctx context.Context,
gitspaceInstanceIdentifier string,
features types.Features,
) (*map[string]*types.DownloadedFeature, error) {
downloadedFeatures := sync.Map{}
featuresToBeDownloaded := sync.Map{}
downloadQueue := make(chan featureSource, 100)
errorCh := make(chan error, 100)
// startCh and endCh are used to check if all the goroutines spawned to download features have completed or not.
// Whenever a new goroutine is spawned, it increments the counter listening to the startCh.
// Upon completion, it increments the counter listening to the endCh.
// When the start count == end count and end count is > 0, it means all the goroutines have completed execution.
startCh := make(chan int, 100)
endCh := make(chan int, 100)
for key, value := range features {
featuresToBeDownloaded.Store(key, value)
downloadQueue <- featureSource{sourceURL: key, sourceType: value.SourceType}
}
go func() {
for source := range downloadQueue {
startCh <- 1
go func(source featureSource) {
defer func(endCh chan int) { endCh <- 1 }(endCh)
err := downloadFeature(ctx, gitspaceInstanceIdentifier, &source, &featuresToBeDownloaded,
downloadQueue, &downloadedFeatures)
errorCh <- err
}(source)
}
}()
var totalStart int
var totalEnd int
var downloadError error
waitLoop:
for {
select {
case start := <-startCh:
totalStart += start
case end := <-endCh:
totalEnd += end
case err := <-errorCh:
if err == nil {
continue
}
if downloadError != nil {
downloadError = fmt.Errorf("error downloading features: %w\n%w", err, downloadError)
} else {
downloadError = fmt.Errorf("error downloading features: %w", err)
}
default:
if totalEnd > 0 && totalStart == totalEnd {
break waitLoop
}
}
}
close(startCh)
close(endCh)
close(downloadQueue)
if downloadError != nil {
return nil, downloadError
}
var finalMap = make(map[string]*types.DownloadedFeature)
var saveToMapFunc = func(key, value any) bool {
finalMap[key.(string)] = value.(*types.DownloadedFeature) // nolint:errcheck
return true
}
downloadedFeatures.Range(saveToMapFunc)
return &finalMap, nil
}
// downloadFeature downloads a single feature. Depending on the source type, it either downloads it from an OCI
// repo or an HTTP(S) URL. It then fetches its devcontainer-feature.json and also checks which dependencies need
// to be downloaded for this feature.
func downloadFeature(
ctx context.Context,
gitspaceInstanceIdentifier string,
source *featureSource,
featuresToBeDownloaded *sync.Map,
downloadQueue chan featureSource,
downloadedFeatures *sync.Map,
) error {
var tarballName string
var featureFolderName string
var featureTag string
var sourceWithoutTag string
var canonicalName string
var downloadDirectory string
switch source.sourceType {
case enum.FeatureSourceTypeOCI:
sourceWithoutTag = strings.SplitN(source.sourceURL, ":", 2)[0]
partialFeatureNameWithTag := filepath.Base(source.sourceURL)
parts := strings.SplitN(partialFeatureNameWithTag, ":", 2)
tarballName = fmt.Sprintf("devcontainer-feature-%s.tgz", parts[0])
featureFolderName = strings.TrimSuffix(tarballName, ".tgz")
featureTag = parts[1]
downloadDirectory = getFeatureDownloadDirectory(gitspaceInstanceIdentifier, featureFolderName, featureTag)
contentDigest, err := downloadTarballFromOCIRepo(ctx, source.sourceURL, downloadDirectory)
if err != nil {
return fmt.Errorf("error downloading oci artifact for feature %s: %w", source.sourceURL, err)
}
canonicalName = contentDigest
case enum.FeatureSourceTypeTarball:
sourceWithoutTag = source.sourceURL
canonicalName = source.sourceURL
featureTag = types.FeatureDefaultTag // using static tag for comparison
tarballURL, err := url.Parse(source.sourceURL)
if err != nil {
return fmt.Errorf("error parsing feature URL for feature %s: %w", source.sourceURL, err)
}
tarballName = filepath.Base(tarballURL.Path)
featureFolderName = strings.TrimSuffix(tarballName, ".tgz")
downloadDirectory = getFeatureDownloadDirectory(gitspaceInstanceIdentifier, featureFolderName, featureTag)
err = downloadTarball(source.sourceURL, downloadDirectory, filepath.Join(downloadDirectory, tarballName))
if err != nil {
return fmt.Errorf("error downloading tarball for feature %s: %w", source.sourceURL, err)
}
default:
return fmt.Errorf("unsupported feature type: %s", source.sourceType)
}
devcontainerFeature, err := getDevcontainerFeatureConfig(downloadDirectory, featureFolderName, tarballName, source)
if err != nil {
return err
}
downloadedFeature := types.DownloadedFeature{
FeatureFolderName: featureFolderName,
SourceWithoutTag: sourceWithoutTag,
Tag: featureTag,
CanonicalName: canonicalName,
DevcontainerFeatureConfig: devcontainerFeature,
}
downloadedFeatures.Store(source.sourceURL, &downloadedFeature)
// Check all the dependencies which are required for this feature. If any, check if they are already marked
// for downloaded. If not, push to the download queue.
if downloadedFeature.DevcontainerFeatureConfig.DependsOn != nil &&
len(*downloadedFeature.DevcontainerFeatureConfig.DependsOn) > 0 {
for key, value := range *downloadedFeature.DevcontainerFeatureConfig.DependsOn {
_, present := featuresToBeDownloaded.LoadOrStore(key, value)
if !present {
downloadQueue <- featureSource{sourceURL: key, sourceType: value.SourceType}
}
}
}
return nil
}
// getDevcontainerFeatureConfig returns the devcontainer-feature.json by unpacking the downloaded tarball,
// unmarshalling the file contents to types.DevcontainerFeatureConfig. It removes any comments before unmarshalling.
func getDevcontainerFeatureConfig(
downloadDirectory string,
featureName string,
tarballName string,
source *featureSource,
) (*types.DevcontainerFeatureConfig, error) {
dst := filepath.Join(downloadDirectory, featureName)
err := unpackTarball(filepath.Join(downloadDirectory, tarballName), dst)
if err != nil {
return nil, fmt.Errorf("error unpacking tarball for feature %s: %w", source.sourceURL, err)
}
err = os.Remove(filepath.Join(downloadDirectory, tarballName))
if err != nil {
return nil, fmt.Errorf("error deleting tarball for feature %s: %w", source.sourceURL, err)
}
devcontainerFeatureRaw, err := os.ReadFile(filepath.Join(dst, "devcontainer-feature.json"))
if err != nil {
return nil, fmt.Errorf("error reading devcontainer-feature.json file for feature %s: %w",
source.sourceURL, err)
}
var devcontainerFeature types.DevcontainerFeatureConfig
err = json.Unmarshal(jsonc.ToJSON(devcontainerFeatureRaw), &devcontainerFeature)
if err != nil {
return nil, fmt.Errorf("error parsing devcontainer-feature.json for feature %s: %w",
source.sourceURL, err)
}
return &devcontainerFeature, nil
}
func getGitspaceInstanceDirectory(gitspaceInstanceIdentifier string) string {
return filepath.Join("/tmp", gitspaceInstanceIdentifier)
}
func getFeaturesDownloadDirectory(gitspaceInstanceIdentifier string) string {
return filepath.Join(getGitspaceInstanceDirectory(gitspaceInstanceIdentifier), "devcontainer-features")
}
func getFeatureDownloadDirectory(gitspaceInstanceIdentifier, featureFolderName, featureTag string) string {
return filepath.Join(getFeaturesDownloadDirectory(gitspaceInstanceIdentifier),
getFeatureFolderNameWithTag(featureFolderName, featureTag))
}
func getFeatureFolderNameWithTag(featureFolderName string, featureTag string) string {
if featureTag == "" {
return featureFolderName
}
return featureFolderName + "-" + featureTag
}
func downloadTarballFromOCIRepo(ctx context.Context, ociRepo string, filePath string) (string, error) {
parts := strings.SplitN(ociRepo, ":", 2)
if len(parts) != 2 {
return "", fmt.Errorf("invalid oci repo: %s", ociRepo)
}
fs, err := file.New(filePath)
if err != nil {
return "", err
}
defer fs.Close()
repo, err := remote.NewRepository(parts[0])
if err != nil {
return "", err
}
tag := parts[1]
md, err := oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions)
if err != nil {
return "", err
}
return md.Digest.String(), nil
}
func downloadTarball(url, dirPath, fileName string) error {
if _, err := os.Stat(dirPath); os.IsNotExist(err) {
err := os.MkdirAll(dirPath, fs.ModePerm)
if err != nil {
return err
}
}
out, err := os.Create(fileName)
if err != nil {
return fmt.Errorf("failed to create file: %w", err)
}
defer out.Close()
resp, err := http.Get(url) // nolint:gosec,noctx
if err != nil {
return fmt.Errorf("failed to download tarball: %w", err)
}
defer resp.Body.Close()
// Check the HTTP response status
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("failed to download tarball: HTTP status %d", resp.StatusCode)
}
// Copy the response body to the file
_, err = io.Copy(out, resp.Body)
if err != nil {
return fmt.Errorf("failed to save tarball: %w", err)
}
return nil
}
// unpackTarball extracts a .tgz file to a specified output directory.
func unpackTarball(tarball, outputDir string) error {
// Open the tarball file
file, err := os.Open(tarball)
if err != nil {
return fmt.Errorf("failed to open tarball: %w", err)
}
defer file.Close()
// Create a tar reader
tarReader := tar.NewReader(file)
// Iterate through the files in the tar archive
for {
header, err := tarReader.Next()
if err == io.EOF {
break // End of tar archive
}
if err != nil {
return fmt.Errorf("failed to read tar header: %w", err)
}
// Determine the file's full path
targetPath := filepath.Join(outputDir, header.Name) // nolint:gosec
switch header.Typeflag {
case tar.TypeDir:
// Create the directory if it doesn't exist
if err := os.MkdirAll(targetPath, os.FileMode(header.Mode)); err != nil {
return fmt.Errorf("failed to create directory: %w", err)
}
case tar.TypeReg:
// Extract the file
if err := extractFile(tarReader, targetPath, header.Mode); err != nil {
return err
}
default:
return fmt.Errorf("skipping unsupported type: %c in %s", header.Typeflag, header.Name)
}
}
return nil
}
// extractFile writes the content of a file from the tar archive.
func extractFile(tarReader *tar.Reader, targetPath string, mode int64) error {
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
return fmt.Errorf("failed to create parent directories: %w", err)
}
outFile, err := os.Create(targetPath)
if err != nil {
return fmt.Errorf("failed to create file: %w", err)
}
defer outFile.Close()
if _, err := io.Copy(outFile, tarReader); err != nil {
return fmt.Errorf("failed to copy file content: %w", err)
}
if err := os.Chmod(targetPath, os.FileMode(mode)); err != nil {
return fmt.Errorf("failed to set file permissions: %w", err)
}
return nil
}

View File

@ -0,0 +1,112 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"github.com/harness/gitness/types"
)
// ResolveFeatures resolves all the downloaded features ie starting from the user-specified features, it checks
// which features need to be installed with which options. A feature in considered uniquely installable if its
// id or source and the options overrides are unique.
// Reference: https://containers.dev/implementors/features/#definition-feature-equality
func ResolveFeatures(
userDefinedFeatures types.Features,
downloadedFeatures map[string]*types.DownloadedFeature,
) (map[string]*types.ResolvedFeature, error) {
featuresToBeResolved := make([]types.FeatureValue, 0)
for _, featureValue := range userDefinedFeatures {
featuresToBeResolved = append(featuresToBeResolved, *featureValue)
}
resolvedFeatures := make(map[string]*types.ResolvedFeature)
for i := 0; i < len(featuresToBeResolved); i++ {
currentFeature := featuresToBeResolved[i]
digest, err := calculateDigest(currentFeature.Source, currentFeature.Options)
if err != nil {
return nil, fmt.Errorf("error calculating digest for %s: %w", currentFeature.Source, err)
}
if _, alreadyResolved := resolvedFeatures[digest]; alreadyResolved {
continue
}
downloadedFeature := downloadedFeatures[currentFeature.Source]
resolvedOptions, err := getResolvedOptions(downloadedFeature, currentFeature)
if err != nil {
return nil, err
}
resolvedFeature := types.ResolvedFeature{
Digest: digest,
ResolvedOptions: resolvedOptions,
OverriddenOptions: currentFeature.Options, // used to calculate digest and sort features
DownloadedFeature: downloadedFeature,
}
resolvedFeatures[digest] = &resolvedFeature
if resolvedFeature.DownloadedFeature.DevcontainerFeatureConfig.DependsOn != nil &&
len(*resolvedFeature.DownloadedFeature.DevcontainerFeatureConfig.DependsOn) > 0 {
for _, featureValue := range *resolvedFeature.DownloadedFeature.DevcontainerFeatureConfig.DependsOn {
featuresToBeResolved = append(featuresToBeResolved, *featureValue)
}
}
}
return resolvedFeatures, nil
}
func getResolvedOptions(
downloadedFeature *types.DownloadedFeature,
currentFeature types.FeatureValue,
) (map[string]string, error) {
resolvedOptions := make(map[string]string)
if downloadedFeature.DevcontainerFeatureConfig.Options != nil {
for optionKey, optionDefinition := range *downloadedFeature.DevcontainerFeatureConfig.Options {
var optionValue = optionDefinition.Default
if userProvidedOptionValue, ok := currentFeature.Options[optionKey]; ok {
optionValue = userProvidedOptionValue
}
stringValue, err := optionDefinition.ValidateValue(optionValue, optionKey, currentFeature.Source)
if err != nil {
return nil, err
}
resolvedOptions[optionKey] = stringValue
}
}
return resolvedOptions, nil
}
// calculateDigest calculates a deterministic hash for a feature using its source and options overrides.
func calculateDigest(source string, optionsOverrides map[string]any) (string, error) {
data := map[string]any{
"options": optionsOverrides,
"source": source,
}
jsonBytes, err := json.Marshal(data)
if err != nil {
return "", fmt.Errorf("failed to serialize data: %w", err)
}
hash := sha256.Sum256(jsonBytes)
return hex.EncodeToString(hash[:]), nil
}

View File

@ -26,6 +26,8 @@ import (
"github.com/harness/gitness/git/command"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
"github.com/tidwall/jsonc"
)
var (
@ -102,13 +104,6 @@ func (s *SCM) GetSCMRepoDetails(
return resolvedDetails, nil
}
func removeComments(input []byte) []byte {
blockCommentRegex := regexp.MustCompile(`(?s)/\*.*?\*/`)
input = blockCommentRegex.ReplaceAll(input, nil)
lineCommentRegex := regexp.MustCompile(`//.*`)
return lineCommentRegex.ReplaceAll(input, nil)
}
func detectDefaultGitBranch(ctx context.Context, gitRepoDir string) (string, error) {
cmd := command.New("ls-remote",
command.WithFlag("--symref"),
@ -192,8 +187,7 @@ func (s *SCM) getDevcontainerConfig(
return config, nil // Return an empty config if the file is empty
}
sanitizedJSON := removeComments(catFileOutputBytes)
if err = json.Unmarshal(sanitizedJSON, &config); err != nil {
if err = json.Unmarshal(jsonc.ToJSON(catFileOutputBytes), &config); err != nil {
return config, fmt.Errorf("failed to parse devcontainer JSON: %w", err)
}

3
go.mod
View File

@ -71,6 +71,7 @@ require (
github.com/swaggest/swgui v1.8.1
github.com/swaggo/http-swagger v1.3.4
github.com/swaggo/swag v1.16.2
github.com/tidwall/jsonc v0.3.2
github.com/unrolled/secure v1.15.0
github.com/zricethezav/gitleaks/v8 v8.18.5-0.20240912004812-e93a7c0d2604
go.starlark.net v0.0.0-20231121155337-90ade8b19d09
@ -84,6 +85,7 @@ require (
google.golang.org/api v0.189.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/mail.v2 v2.3.1
oras.land/oras-go/v2 v2.5.0
)
require (
@ -96,6 +98,7 @@ require (
github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e // indirect
github.com/BobuSumisu/aho-corasick v1.0.3 // indirect
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/Masterminds/semver/v3 v3.3.1 // indirect
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
github.com/antonmedv/expr v1.15.5 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect

6
go.sum
View File

@ -33,6 +33,8 @@ github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4=
github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM=
github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10=
github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
@ -760,6 +762,8 @@ github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64
github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ=
github.com/swaggo/swag v1.16.2 h1:28Pp+8DkQoV+HLzLx8RGJZXNGKbFqnuvSbAAtoxiY04=
github.com/swaggo/swag v1.16.2/go.mod h1:6YzXnDcpr0767iOejs318CwYkCQqyGer6BizOg03f+E=
github.com/tidwall/jsonc v0.3.2 h1:ZTKrmejRlAJYdn0kcaFqRAKlxxFIC21pYq8vLa4p2Wc=
github.com/tidwall/jsonc v0.3.2/go.mod h1:dw+3CIxqHi+t8eFSpzzMlcVYxKp08UP5CD8/uSFCyJE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
@ -1087,6 +1091,8 @@ k8s.io/api v0.0.0-20181130031204-d04500c8c3dd/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j
k8s.io/apimachinery v0.0.0-20181201231028-18a5ff3097b4/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
k8s.io/client-go v9.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c=
oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

View File

@ -17,20 +17,34 @@ package types
import (
"encoding/json"
"fmt"
"net/url"
"strings"
"github.com/harness/gitness/types/enum"
"oras.land/oras-go/v2/registry"
)
const FeatureDefaultTag = "latest"
//nolint:tagliatelle
type DevcontainerConfig struct {
Image string `json:"image,omitempty"`
PostCreateCommand LifecycleCommand `json:"postCreateCommand,omitempty"`
PostStartCommand LifecycleCommand `json:"postStartCommand,omitempty"`
ForwardPorts []json.Number `json:"forwardPorts,omitempty"`
ContainerEnv map[string]string `json:"containerEnv,omitempty"`
Customizations DevContainerConfigCustomizations `json:"customizations,omitempty"`
RunArgs []string `json:"runArgs,omitempty"`
ContainerUser string `json:"containerUser,omitempty"`
RemoteUser string `json:"remoteUser,omitempty"`
Image string `json:"image,omitempty"`
PostCreateCommand LifecycleCommand `json:"postCreateCommand,omitempty"`
PostStartCommand LifecycleCommand `json:"postStartCommand,omitempty"`
ForwardPorts []json.Number `json:"forwardPorts,omitempty"`
ContainerEnv map[string]string `json:"containerEnv,omitempty"`
Customizations DevContainerConfigCustomizations `json:"customizations,omitempty"`
RunArgs []string `json:"runArgs,omitempty"`
ContainerUser string `json:"containerUser,omitempty"`
RemoteUser string `json:"remoteUser,omitempty"`
Features *Features `json:"features,omitempty"`
OverrideFeatureInstallOrder []string `json:"overrideFeatureInstallOrder,omitempty"`
Privileged bool `json:"privileged,omitempty"`
Init bool `json:"init,omitempty"`
CapAdd []string `json:"capAdd,omitempty"`
SecurityOpt []string `json:"securityOpt,omitempty"`
Mounts []*Mount `json:"mounts,omitempty"`
}
// Constants for discriminator values.
@ -138,3 +152,95 @@ func (lc *LifecycleCommand) ToCommandArray() []string {
return nil
}
}
type Features map[string]*FeatureValue
type FeatureValue struct {
Source string `json:"source,omitempty"`
SourceType enum.FeatureSourceType `json:"source_type,omitempty"`
Options map[string]any `json:"options,omitempty"`
}
func (f *FeatureValue) UnmarshalJSON(data []byte) error {
var version string
if err := json.Unmarshal(data, &version); err == nil {
f.Options = make(map[string]any)
f.Options["version"] = version
return nil
}
var options map[string]any
if err := json.Unmarshal(data, &options); err == nil {
for key, value := range options {
switch value.(type) {
case string, bool:
continue
default:
return fmt.Errorf("invalid type for option '%s': must be string or boolean, got %T", key, value)
}
}
f.Options = options
return nil
}
return nil
}
func (f *Features) UnmarshalJSON(data []byte) error {
if *f == nil {
*f = make(Features)
}
var raw map[string]json.RawMessage
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
for key, value := range raw {
sanitizedSource, sourceType, validationErr := validateFeatureSource(key)
if validationErr != nil {
return validationErr
}
feature := &FeatureValue{Source: sanitizedSource, SourceType: sourceType}
if err := json.Unmarshal(value, feature); err != nil {
return fmt.Errorf("failed to unmarshal feature '%s': %w", key, err)
}
(*f)[sanitizedSource] = feature
}
return nil
}
func validateFeatureSource(source string) (string, enum.FeatureSourceType, error) {
if _, err := registry.ParseReference(source); err == nil {
indexOfSeparator := strings.Index(source, ":")
if indexOfSeparator == -1 {
source += ":" + FeatureDefaultTag
}
return source, enum.FeatureSourceTypeOCI, nil
}
if err := validateTarballURL(source); err == nil {
return source, enum.FeatureSourceTypeTarball, nil
}
return source, enum.FeatureSourceTypeLocal, fmt.Errorf("unsupported feature source: %s", source)
}
func validateTarballURL(source string) error {
tarballURL, err := url.Parse(source)
if err != nil {
return fmt.Errorf("parsing feature URL: %w", err)
}
if tarballURL.Scheme != "http" && tarballURL.Scheme != "https" {
return fmt.Errorf("invalid feature URL: %s", tarballURL.String())
}
if !strings.HasSuffix(tarballURL.Path, ".tgz") {
return fmt.Errorf("invalid feature URL: %s", tarballURL.String())
}
return nil
}
type Mount struct {
Source string `json:"source,omitempty"`
Target string `json:"target,omitempty"`
Type string `json:"type,omitempty"`
}

View File

@ -0,0 +1,81 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"fmt"
"slices"
"strconv"
"github.com/harness/gitness/types/enum"
)
//nolint:tagliatelle
type DevcontainerFeatureConfig struct {
ID string `json:"id,omitempty"`
Version string `json:"version,omitempty"`
Name string `json:"name,omitempty"`
Options *Options `json:"options,omitempty"`
DependsOn *Features `json:"dependsOn,omitempty"`
ContainerEnv map[string]string `json:"containerEnv,omitempty"`
Privileged bool `json:"privileged,omitempty"`
Init bool `json:"init,omitempty"`
Deprecated bool `json:"deprecated,omitempty"`
CapAdd []string `json:"capAdd,omitempty"`
SecurityOpt []string `json:"securityOpt,omitempty"`
Entrypoint string `json:"entrypoint,omitempty"`
InstallsAfter []string `json:"installsAfter,omitempty"`
Mounts []*Mount `json:"mounts,omitempty"`
PostCreateCommand LifecycleCommand `json:"postCreateCommand,omitempty"`
PostStartCommand LifecycleCommand `json:"postStartCommand,omitempty"`
}
type Options map[string]*OptionDefinition
type OptionDefinition struct {
Type enum.FeatureOptionValueType `json:"type,omitempty"`
Proposals []string `json:"proposals,omitempty"`
Enum []string `json:"enum,omitempty"`
Default any `json:"default,omitempty"`
Description string `json:"description,omitempty"`
}
// ValidateValue checks if the value matches the type defined in the definition. For string types,
// it also checks if it is allowed ie it is present in the enum array for the option.
// Reference: https://containers.dev/implementors/features/#options-property
func (o *OptionDefinition) ValidateValue(optionValue any, optionKey string, featureSource string) (string, error) {
switch o.Type {
case enum.FeatureOptionValueTypeBoolean:
boolValue, ok := optionValue.(bool)
if !ok {
return "", fmt.Errorf("error during resolving feature %s, option Id %s "+
"expects boolean, got %s ", featureSource, optionKey, optionValue)
}
return strconv.FormatBool(boolValue), nil
case enum.FeatureOptionValueTypeString:
stringValue, ok := optionValue.(string)
if !ok {
return "", fmt.Errorf("error during resolving feature %s, option Id %s "+
"expects string, got %s ", featureSource, optionKey, optionValue)
}
if len(o.Enum) > 0 && !slices.Contains(o.Enum, stringValue) {
return "", fmt.Errorf("error during resolving feature %s, option value %s "+
"not allowed for Id %s ", featureSource, stringValue, optionKey)
}
return stringValue, nil
default:
return "", fmt.Errorf("unsupported option type %s", o.Type)
}
}

View File

@ -0,0 +1,29 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package enum
type FeatureOptionValueType string
func (FeatureOptionValueType) Enum() []interface{} { return toInterfaceSlice(featureOptionValueTypes) }
const (
FeatureOptionValueTypeString = "string"
FeatureOptionValueTypeBoolean = "boolean"
)
var featureOptionValueTypes = sortEnum([]ExecutionSort{
FeatureOptionValueTypeString,
FeatureOptionValueTypeBoolean,
})

View File

@ -0,0 +1,31 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package enum
type FeatureSourceType string
func (FeatureSourceType) Enum() []interface{} { return toInterfaceSlice(featureSourceTypes) }
const (
FeatureSourceTypeOCI = "oci"
FeatureSourceTypeTarball = "tarball"
FeatureSourceTypeLocal = "local"
)
var featureSourceTypes = sortEnum([]ExecutionSort{
FeatureSourceTypeOCI,
FeatureSourceTypeTarball,
FeatureSourceTypeLocal,
})

137
types/resolved_feature.go Normal file
View File

@ -0,0 +1,137 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"sort"
"strconv"
"strings"
"github.com/Masterminds/semver/v3"
)
type ResolvedFeature struct {
ResolvedOptions map[string]string `json:"options,omitempty"`
OverriddenOptions map[string]any `json:"overridden_options,omitempty"`
Digest string `json:"string,omitempty"`
DownloadedFeature *DownloadedFeature `json:"downloaded_feature,omitempty"`
}
type DownloadedFeature struct {
FeatureFolderName string `json:"feature_folder_name,omitempty"`
SourceWithoutTag string `json:"source_without_tag,omitempty"`
Tag string `json:"tag,omitempty"`
CanonicalName string `json:"canonical_name,omitempty"`
DevcontainerFeatureConfig *DevcontainerFeatureConfig `json:"devcontainer_feature_config,omitempty"`
}
// CompareResolvedFeature implements the following comparison rules.
// 1. Compare and sort each Feature lexicographically by their fully qualified resource name
// (For OCI-published Features, that means the ID without version or digest.). If the comparison is equal:
// 2. Compare and sort each Feature from oldest to newest tag (latest being the “most new”). If the comparison is equal:
// 3. Compare and sort each Feature by their options by:
// 3.1 Greatest number of user-defined options (note omitting an option will default that value to the Features
// default value and is not considered a user-defined option). If the comparison is equal:
// 3.2 Sort the provided option keys lexicographically. If the comparison is equal:
// 3.3 Sort the provided option values lexicographically. If the comparison is equal:
// 4. Sort Features by their canonical name (For OCI-published Features, the Feature ID resolved to the digest hash).
// 5. If there is no difference based on these comparator rules, the Features are considered equal.
// Reference: https://containers.dev/implementors/features/#definition-feature-equality (Round Stable Sort).
func CompareResolvedFeature(a, b *ResolvedFeature) int {
var comparison int
comparison = strings.Compare(a.DownloadedFeature.SourceWithoutTag, b.DownloadedFeature.SourceWithoutTag)
if comparison != 0 {
return comparison
}
comparison, _ = compareTags(a.DownloadedFeature.Tag, b.DownloadedFeature.Tag)
if comparison != 0 {
return comparison
}
comparison = compareOverriddenOptions(a.OverriddenOptions, b.OverriddenOptions)
if comparison != 0 {
return comparison
}
return strings.Compare(a.DownloadedFeature.CanonicalName, b.DownloadedFeature.CanonicalName)
}
func compareTags(a, b string) (int, error) {
if a == FeatureDefaultTag && b == FeatureDefaultTag {
return 0, nil
}
if a == FeatureDefaultTag {
return 1, nil
}
if b == FeatureDefaultTag {
return -1, nil
}
versionA, err := semver.NewVersion(a)
if err != nil {
return 0, err
}
versionB, err := semver.NewVersion(b)
if err != nil {
return 0, err
}
return versionA.Compare(versionB), nil
}
func compareOverriddenOptions(a, b map[string]any) int {
if len(a) != len(b) {
return len(a) - len(b)
}
keysA, valuesA := getSortedOptions(a)
keysB, valuesB := getSortedOptions(b)
for i := 0; i < len(keysA); i++ {
if keysA[i] == keysB[i] {
continue
}
return strings.Compare(keysA[i], keysB[i])
}
for i := 0; i < len(valuesA); i++ {
if valuesA[i] == valuesB[i] {
continue
}
return strings.Compare(valuesA[i], valuesB[i])
}
return 0
}
// getSortedOptions returns the keys and values of a map sorted lexicographically.
func getSortedOptions(m map[string]any) ([]string, []string) {
keys := make([]string, 0, len(m))
for key := range m {
keys = append(keys, key)
}
sort.Strings(keys)
values := make([]string, 0, len(m))
for _, key := range keys {
value := m[key]
switch v := value.(type) {
case string:
values = append(values, v)
case bool:
values = append(values, strconv.FormatBool(v))
}
}
return keys, values
}