[code-1822] Download repository as archive using git archive command (#2010)

app/api/controller/repo/archive.go

@@ -0,0 +1,43 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package repo
+
+import (
+	"context"
+	"io"
+
+	"github.com/harness/gitness/app/auth"
+	"github.com/harness/gitness/git"
+	"github.com/harness/gitness/git/api"
+	"github.com/harness/gitness/types/enum"
+)
+
+func (c *Controller) Archive(
+	ctx context.Context,
+	session *auth.Session,
+	repoRef string,
+	params api.ArchiveParams,
+	w io.Writer,
+) error {
+	repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
+	if err != nil {
+		return err
+	}
+
+	return c.git.Archive(ctx, git.ArchiveParams{
+		ReadParams:    git.CreateReadParams(repo),
+		ArchiveParams: params,
+	}, w)
+}

app/api/handler/repo/archive.go

@@ -0,0 +1,59 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package repo
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/harness/gitness/app/api/controller/repo"
+	"github.com/harness/gitness/app/api/render"
+	"github.com/harness/gitness/app/api/request"
+	"github.com/harness/gitness/git/api"
+)
+
+func HandleArchive(repoCtrl *repo.Controller) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		session, _ := request.AuthSessionFrom(ctx)
+		repoRef, err := request.GetRepoRefFromPath(r)
+		if err != nil {
+			render.TranslatedUserError(ctx, w, err)
+			return
+		}
+
+		params, filename := request.ParseArchiveParams(r)
+		var contentType string
+		switch params.Format {
+		case api.ArchiveFormatTar:
+			contentType = "application/tar"
+		case api.ArchiveFormatZip:
+			contentType = "application/zip"
+		case api.ArchiveFormatTarGz, api.ArchiveFormatTgz:
+			contentType = "application/gzip"
+		default:
+			contentType = "application/zip"
+		}
+
+		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename))
+		w.Header().Set("Content-Type", contentType)
+
+		err = repoCtrl.Archive(ctx, session, repoRef, params, w)
+		if err != nil {
+			render.TranslatedUserError(ctx, w, err)
+			return
+		}
+	}
+}

app/api/openapi/repo.go

@@ -210,6 +210,12 @@ type generalSettingsRequest struct {
 	reposettings.GeneralSettings
 }
 
+type archiveRequest struct {
+	repoRequest
+	GitRef string `path:"git_ref" required:"true"`
+	Format string `path:"format" required:"true"`
+}
+
 var queryParameterGitRef = openapi3.ParameterOrRef{
 	Parameter: &openapi3.Parameter{
 		Name: request.QueryParamGitRef,
@@ -513,6 +519,86 @@ var queryParameterDeletedAt = openapi3.ParameterOrRef{
 	},
 }
 
+var queryParamArchivePaths = openapi3.ParameterOrRef{
+	Parameter: &openapi3.Parameter{
+		Name: request.QueryParamArchivePaths,
+		In:   openapi3.ParameterInQuery,
+		Description: ptr.String("Without an optional path parameter, all files and subdirectories of the " +
+			"current working directory are included in the archive. If one or more paths are specified," +
+			" only these are included."),
+		Required: ptr.Bool(false),
+		Schema: &openapi3.SchemaOrRef{
+			Schema: &openapi3.Schema{
+				Type: ptrSchemaType(openapi3.SchemaTypeArray),
+				Items: &openapi3.SchemaOrRef{
+					Schema: &openapi3.Schema{
+						Type: ptrSchemaType(openapi3.SchemaTypeString),
+					},
+				},
+			},
+		},
+	},
+}
+
+var queryParamArchivePrefix = openapi3.ParameterOrRef{
+	Parameter: &openapi3.Parameter{
+		Name: request.QueryParamArchivePrefix,
+		In:   openapi3.ParameterInQuery,
+		Description: ptr.String("Prepend <prefix>/ to paths in the archive. Can be repeated; its rightmost value" +
+			" is used for all tracked files."),
+		Required: ptr.Bool(false),
+		Schema: &openapi3.SchemaOrRef{
+			Schema: &openapi3.Schema{
+				Type: ptrSchemaType(openapi3.SchemaTypeString),
+			},
+		},
+	},
+}
+
+var queryParamArchiveAttributes = openapi3.ParameterOrRef{
+	Parameter: &openapi3.Parameter{
+		Name:        request.QueryParamArchiveAttributes,
+		In:          openapi3.ParameterInQuery,
+		Description: ptr.String("Look for attributes in .gitattributes files in the working tree as well"),
+		Required:    ptr.Bool(false),
+		Schema: &openapi3.SchemaOrRef{
+			Schema: &openapi3.Schema{
+				Type: ptrSchemaType(openapi3.SchemaTypeString),
+			},
+		},
+	},
+}
+
+var queryParamArchiveTime = openapi3.ParameterOrRef{
+	Parameter: &openapi3.Parameter{
+		Name: request.QueryParamArchiveTime,
+		In:   openapi3.ParameterInQuery,
+		Description: ptr.String("Set modification time of archive entries. Without this option the committer " +
+			"time is used if <tree-ish> is a commit or tag, and the current time if it is a tree."),
+		Required: ptr.Bool(false),
+		Schema: &openapi3.SchemaOrRef{
+			Schema: &openapi3.Schema{
+				Type: ptrSchemaType(openapi3.SchemaTypeString),
+			},
+		},
+	},
+}
+
+var queryParamArchiveCompression = openapi3.ParameterOrRef{
+	Parameter: &openapi3.Parameter{
+		Name: request.QueryParamArchiveCompression,
+		In:   openapi3.ParameterInQuery,
+		Description: ptr.String("Specify compression level. Larger values allow the command to spend more" +
+			" time to compress to smaller size."),
+		Required: ptr.Bool(false),
+		Schema: &openapi3.SchemaOrRef{
+			Schema: &openapi3.Schema{
+				Type: ptrSchemaType(openapi3.SchemaTypeInteger),
+			},
+		},
+	},
+}
+
 //nolint:funlen
 func repoOperations(reflector *openapi3.Reflector) {
 	createRepository := openapi3.Operation{}
@@ -1028,4 +1114,25 @@ func repoOperations(reflector *openapi3.Reflector) {
 	_ = reflector.SetJSONResponse(&opSettingsGeneralFind, new(usererror.Error), http.StatusNotFound)
 	_ = reflector.Spec.AddOperation(
 		http.MethodGet, "/repos/{repo_ref}/settings/general", opSettingsGeneralFind)
+
+	opArchive := openapi3.Operation{}
+	opArchive.WithTags("repository")
+	opArchive.WithMapOfAnything(map[string]interface{}{"operationId": "archive"})
+	opArchive.WithParameters(
+		queryParamArchivePaths,
+		queryParamArchivePrefix,
+		queryParamArchiveAttributes,
+		queryParamArchiveTime,
+		queryParamArchiveCompression,
+	)
+	_ = reflector.SetRequest(&opArchive, new(archiveRequest), http.MethodGet)
+	_ = reflector.SetStringResponse(&opArchive, http.StatusOK, "application/zip")
+	_ = reflector.SetStringResponse(&opArchive, http.StatusOK, "application/tar")
+	_ = reflector.SetStringResponse(&opArchive, http.StatusOK, "application/gzip")
+	_ = reflector.SetJSONResponse(&opArchive, new(usererror.Error), http.StatusInternalServerError)
+	_ = reflector.SetJSONResponse(&opArchive, new(usererror.Error), http.StatusUnprocessableEntity)
+	_ = reflector.SetJSONResponse(&opArchive, new(usererror.Error), http.StatusUnauthorized)
+	_ = reflector.SetJSONResponse(&opArchive, new(usererror.Error), http.StatusForbidden)
+	_ = reflector.SetJSONResponse(&opArchive, new(usererror.Error), http.StatusNotFound)
+	_ = reflector.Spec.AddOperation(http.MethodGet, "/repos/{repo_ref}/archive/{git_ref}.{format}", opArchive)
 }

app/api/request/archive.go

@@ -0,0 +1,77 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package request
+
+import (
+	"net/http"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/harness/gitness/git/api"
+)
+
+const (
+	PathParamArchiveRefs         = "*"
+	QueryParamArchivePaths       = "path"
+	QueryParamArchivePrefix      = "prefix"
+	QueryParamArchiveAttributes  = "attributes"
+	QueryParamArchiveTime        = "time"
+	QueryParamArchiveCompression = "compression"
+)
+
+func ParseArchiveParams(r *http.Request) (api.ArchiveParams, string) {
+	// separate rev and ref part from url, for example:
+	// api/v1/repos/root/demo/+/archive/refs/heads/main.zip
+	// will produce rev=refs/heads and ref=main.zip
+	rev, filename := filepath.Split(PathParamOrEmpty(r, PathParamArchiveRefs))
+	// use ext as format specifier
+	ext := filepath.Ext(filename)
+	// get name of the ref from filename
+	name := strings.TrimSuffix(filename, ext)
+	format := strings.Replace(ext, ".", "", 1)
+	// prefix is used for git archive to prefix all paths.
+	prefix, _ := QueryParam(r, QueryParamArchivePrefix)
+	attributes, _ := QueryParam(r, QueryParamArchiveAttributes)
+
+	var mtime *time.Time
+	timeStr, _ := QueryParam(r, QueryParamArchiveTime)
+	if timeStr != "" {
+		value, err := time.Parse(time.DateTime, timeStr)
+		if err == nil {
+			mtime = &value
+		}
+	}
+
+	var compression *int
+	compressionStr, _ := QueryParam(r, QueryParamArchiveCompression)
+	if compressionStr != "" {
+		value, err := strconv.Atoi(compressionStr)
+		if err == nil {
+			compression = &value
+		}
+	}
+
+	return api.ArchiveParams{
+		Format:      api.ArchiveFormat(format),
+		Prefix:      prefix,
+		Attributes:  api.ArchiveAttribute(attributes),
+		Time:        mtime,
+		Compression: compression,
+		Treeish:     rev + name,
+		Paths:       r.URL.Query()[QueryParamArchivePaths],
+	}, filename
+}

app/router/api.go

@@ -354,6 +354,8 @@ func setupRepos(r chi.Router,
 
 			r.Get("/codeowners/validate", handlerrepo.HandleCodeOwnersValidate(repoCtrl))
 
+			r.Get(fmt.Sprintf("/archive/%s", request.PathParamArchiveRefs), handlerrepo.HandleArchive(repoCtrl))
+
 			SetupPullReq(r, pullreqCtrl)
 
 			SetupWebhook(r, webhookCtrl)

git/api/archive.go

@@ -0,0 +1,173 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/harness/gitness/errors"
+	"github.com/harness/gitness/git/command"
+)
+
+type ArchiveFormat string
+
+const (
+	ArchiveFormatTar   ArchiveFormat = "tar"
+	ArchiveFormatZip   ArchiveFormat = "zip"
+	ArchiveFormatTarGz ArchiveFormat = "tar.gz"
+	ArchiveFormatTgz   ArchiveFormat = "tgz"
+)
+
+func (f ArchiveFormat) Validate() error {
+	switch f {
+	case ArchiveFormatTar, ArchiveFormatZip, ArchiveFormatTarGz, ArchiveFormatTgz:
+		return nil
+	default:
+		return errors.InvalidArgument("git archive flag format '%s' is invalid", f)
+	}
+}
+
+type ArchiveAttribute string
+
+const (
+	ArchiveAttributeExportIgnore ArchiveAttribute = "export-ignore"
+	ArchiveAttributeExportSubst  ArchiveAttribute = "export-subst"
+)
+
+func (f ArchiveAttribute) Validate() error {
+	switch f {
+	case ArchiveAttributeExportIgnore, ArchiveAttributeExportSubst:
+		return nil
+	default:
+		return fmt.Errorf("git archive flag worktree-attributes '%s' is invalid", f)
+	}
+}
+
+type ArchiveParams struct {
+	// Format of the resulting archive. Possible values are tar, zip, tar.gz, tgz,
+	// and any format defined using the configuration option tar.<format>.command
+	// If --format is not given, and the output file is specified, the format is inferred
+	// from the filename if possible (e.g. writing to foo.zip makes the output to be in the zip format),
+	// Otherwise the output format is tar.
+	Format ArchiveFormat
+
+	// Prefix prepend <prefix>/ to paths in the archive. Can be repeated; its rightmost value is used
+	// for all tracked files.
+	Prefix string
+
+	// Write the archive to <file> instead of stdout.
+	File string
+
+	// export-ignore
+	// Files and directories with the attribute export-ignore won’t be added to archive files.
+	// See gitattributes[5] for details.
+	//
+	// export-subst
+	// If the attribute export-subst is set for a file then Git will expand several placeholders
+	// when adding this file to an archive. See gitattributes[5] for details.
+	Attributes ArchiveAttribute
+
+	// Set modification time of archive entries. Without this option the committer time is
+	// used if <tree-ish> is a commit or tag, and the current time if it is a tree.
+	Time *time.Time
+
+	// Compression is level used for tar.gz and zip packers.
+	Compression *int
+
+	// The tree or commit to produce an archive for.
+	Treeish string
+
+	// Paths is optional parameter, all files and subdirectories of the
+	// current working directory are included in the archive, if one or more paths
+	// are specified, only these are included.
+	Paths []string
+}
+
+func (p *ArchiveParams) Validate() error {
+	if p.Treeish == "" {
+		return errors.InvalidArgument("treeish field cannot be empty")
+	}
+	if err := p.Format.Validate(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (g *Git) Archive(ctx context.Context, repoPath string, params ArchiveParams, w io.Writer) error {
+	if err := params.Validate(); err != nil {
+		return err
+	}
+	cmd := command.New("archive",
+		command.WithArg(params.Treeish),
+	)
+
+	format := ArchiveFormatTar
+	if params.Format != "" {
+		format = params.Format
+	}
+	cmd.Add(command.WithFlag("--format", string(format)))
+
+	if params.Prefix != "" {
+		prefix := params.Prefix
+		if !strings.HasSuffix(params.Prefix, "/") {
+			prefix += "/"
+		}
+		cmd.Add(command.WithFlag("--prefix", prefix))
+	}
+
+	if params.File != "" {
+		cmd.Add(command.WithFlag("--output", params.File))
+	}
+
+	if params.Attributes != "" {
+		if err := params.Attributes.Validate(); err != nil {
+			return err
+		}
+		cmd.Add(command.WithFlag("--worktree-attributes", string(params.Attributes)))
+	}
+
+	if params.Time != nil {
+		cmd.Add(command.WithFlag("--mtime", fmt.Sprintf("%q", params.Time.Format(time.DateTime))))
+	}
+
+	if params.Compression != nil {
+		switch params.Format {
+		case ArchiveFormatZip:
+			// zip accepts values digit 0-9
+			if *params.Compression < 0 || *params.Compression > 9 {
+				return errors.InvalidArgument("compression level argument '%d' not supported for format 'zip'",
+					*params.Compression)
+			}
+			cmd.Add(command.WithArg(fmt.Sprintf("-%d", *params.Compression)))
+		case ArchiveFormatTarGz, ArchiveFormatTgz:
+			// tar.gz accepts number
+			cmd.Add(command.WithArg(fmt.Sprintf("-%d", *params.Compression)))
+		case ArchiveFormatTar:
+			// not usable for tar
+		}
+	}
+
+	cmd.Add(command.WithArg(params.Paths...))
+
+	if err := cmd.Run(ctx, command.WithDir(repoPath), command.WithStdout(w)); err != nil {
+		return fmt.Errorf("failed to archive repository: %w", err)
+	}
+
+	return nil
+}

git/command/builder.go

@@ -16,6 +16,7 @@ package command
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 )
 
@@ -47,6 +48,20 @@ var descriptions = map[string]builder{
 	"archive": {
 		// git-archive(1) does not support disambiguating options from paths from revisions.
 		flags: NoRefUpdates | NoEndOfOptions,
+		validatePositionalArgs: func(args []string) error {
+			for _, arg := range args {
+				if strings.HasPrefix(arg, "-") {
+					// check if the argument is a level of compression
+					if _, err := strconv.Atoi(arg[1:]); err == nil {
+						return nil
+					}
+				}
+				if err := validatePositionalArg(arg); err != nil {
+					return err
+				}
+			}
+			return nil
+		},
 	},
 	"blame": {
 		// git-blame(1) does not support disambiguating options from paths from revisions.

git/interface.go

@@ -101,4 +101,5 @@ type Interface interface {
 	 * Secret Scanning service
 	 */
 	ScanSecrets(ctx context.Context, param *ScanSecretsParams) (*ScanSecretsOutput, error)
+	Archive(ctx context.Context, params ArchiveParams, w io.Writer) error
 }

git/repo.go

@@ -536,3 +536,21 @@ func (s *Service) UpdateDefaultBranch(
 	}
 	return nil
 }
+
+type ArchiveParams struct {
+	ReadParams
+	api.ArchiveParams
+}
+
+func (s *Service) Archive(ctx context.Context, params ArchiveParams, w io.Writer) error {
+	if err := params.ReadParams.Validate(); err != nil {
+		return err
+	}
+	repoPath := getFullPathForRepo(s.reposRoot, params.RepoUID)
+	err := s.git.Archive(ctx, repoPath, params.ArchiveParams, w)
+	if err != nil {
+		return fmt.Errorf("failed to run git archive: %w", err)
+	}
+
+	return nil
+}