Add Repo Path Listing API (#1197)

app/api/controller/githook/pre_receive_scan_secrets.go

@@ -98,8 +98,8 @@ func scanSecretsInternal(ctx context.Context,
 		}
 
 		// in case the branch was just created - fallback to compare against latest default branch.
-		baseRev := refUpdate.Old.String() + "^{commit}"
-		rev := refUpdate.New.String() + "^{commit}"
+		baseRev := refUpdate.Old.String() + "^{commit}" //nolint:goconst
+		rev := refUpdate.New.String() + "^{commit}"     //nolint:goconst
 		//nolint:nestif
 		if refUpdate.Old.IsNil() {
 			if baseRevFallBack == nil {

app/api/controller/repo/create.go

@@ -127,7 +127,7 @@ func (c *Controller) Create(ctx context.Context, session *auth.Session, in *Crea
 		session.Principal,
 		audit.NewResource(audit.ResourceTypeRepository, repo.Identifier),
 		audit.ActionCreated,
-		paths.Space(repo.Path),
+		paths.Parent(repo.Path),
 		audit.WithNewObject(repo),
 	)
 	if err != nil {

app/api/controller/repo/default_branch.go

@@ -99,7 +99,7 @@ func (c *Controller) UpdateDefaultBranch(
 		session.Principal,
 		audit.NewResource(audit.ResourceTypeRepository, repo.Identifier),
 		audit.ActionUpdated,
-		paths.Space(repo.Path),
+		paths.Parent(repo.Path),
 		audit.WithOldObject(repoClone),
 		audit.WithNewObject(repo),
 	)

app/api/controller/repo/list_paths.go

@@ -0,0 +1,61 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package repo
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/harness/gitness/app/auth"
+	"github.com/harness/gitness/git"
+	"github.com/harness/gitness/types/enum"
+)
+
+type ListPathsOutput struct {
+	Files       []string `json:"files,omitempty"`
+	Directories []string `json:"directories,omitempty"`
+}
+
+// ListPaths lists the paths in the repo for a specific revision.
+func (c *Controller) ListPaths(ctx context.Context,
+	session *auth.Session,
+	repoRef string,
+	gitRef string,
+	includeDirectories bool,
+) (ListPathsOutput, error) {
+	repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
+	if err != nil {
+		return ListPathsOutput{}, err
+	}
+
+	// set gitRef to default branch in case an empty reference was provided
+	if gitRef == "" {
+		gitRef = repo.DefaultBranch
+	}
+
+	rpcOut, err := c.git.ListPaths(ctx, &git.ListPathsParams{
+		ReadParams:         git.CreateReadParams(repo),
+		GitREF:             gitRef,
+		IncludeDirectories: includeDirectories,
+	})
+	if err != nil {
+		return ListPathsOutput{}, fmt.Errorf("failed to list git paths: %w", err)
+	}
+
+	return ListPathsOutput{
+		Files:       rpcOut.Files,
+		Directories: rpcOut.Directories,
+	}, nil
+}

app/api/controller/repo/rule_create.go

@@ -120,7 +120,7 @@ func (c *Controller) RuleCreate(ctx context.Context,
 		session.Principal,
 		audit.NewResource(audit.ResourceTypeBranchRule, r.Identifier),
 		audit.ActionCreated,
-		paths.Space(repo.Path),
+		paths.Parent(repo.Path),
 		audit.WithNewObject(r),
 	)
 	if err != nil {

app/api/controller/repo/rule_delete.go

@@ -51,7 +51,7 @@ func (c *Controller) RuleDelete(ctx context.Context,
 		session.Principal,
 		audit.NewResource(audit.ResourceTypeBranchRule, r.Identifier),
 		audit.ActionDeleted,
-		paths.Space(repo.Path),
+		paths.Parent(repo.Path),
 		audit.WithOldObject(r),
 	)
 	if err != nil {

app/api/controller/repo/rule_update.go

@@ -142,7 +142,7 @@ func (c *Controller) RuleUpdate(ctx context.Context,
 		session.Principal,
 		audit.NewResource(audit.ResourceTypeBranchRule, r.Identifier),
 		audit.ActionUpdated,
-		paths.Space(repo.Path),
+		paths.Parent(repo.Path),
 		audit.WithOldObject(oldRule),
 		audit.WithNewObject(r),
 	)

app/api/controller/repo/soft_delete.go

@@ -68,7 +68,7 @@ func (c *Controller) SoftDelete(
 		session.Principal,
 		audit.NewResource(audit.ResourceTypeRepository, repo.Identifier),
 		audit.ActionDeleted,
-		paths.Space(repo.Path),
+		paths.Parent(repo.Path),
 		audit.WithOldObject(repo),
 	)
 	if err != nil {

app/api/controller/repo/update.go

@@ -80,7 +80,7 @@ func (c *Controller) Update(ctx context.Context,
 		session.Principal,
 		audit.NewResource(audit.ResourceTypeRepository, repo.Identifier),
 		audit.ActionUpdated,
-		paths.Space(repo.Path),
+		paths.Parent(repo.Path),
 		audit.WithOldObject(repoClone),
 		audit.WithNewObject(repo),
 	)

app/api/handler/repo/list_paths.go

@@ -0,0 +1,50 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package repo
+
+import (
+	"net/http"
+
+	"github.com/harness/gitness/app/api/controller/repo"
+	"github.com/harness/gitness/app/api/render"
+	"github.com/harness/gitness/app/api/request"
+)
+
+func HandleListPaths(repoCtrl *repo.Controller) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		session, _ := request.AuthSessionFrom(ctx)
+		repoRef, err := request.GetRepoRefFromPath(r)
+		if err != nil {
+			render.TranslatedUserError(ctx, w, err)
+			return
+		}
+
+		gitRef := request.GetGitRefFromQueryOrDefault(r, "")
+		includeDirectories, err := request.GetIncludeDirectoriesFromQueryOrDefault(r, false)
+		if err != nil {
+			render.TranslatedUserError(ctx, w, err)
+			return
+		}
+
+		out, err := repoCtrl.ListPaths(ctx, session, repoRef, gitRef, includeDirectories)
+		if err != nil {
+			render.TranslatedUserError(ctx, w, err)
+			return
+		}
+
+		render.JSON(w, http.StatusOK, out)
+	}
+}

app/api/openapi/repo.go

@@ -273,6 +273,21 @@ var queryParameterIncludeCommit = openapi3.ParameterOrRef{
 	},
 }
 
+var queryParameterIncludeDirectories = openapi3.ParameterOrRef{
+	Parameter: &openapi3.Parameter{
+		Name:        request.QueryParamIncludeDirectories,
+		In:          openapi3.ParameterInQuery,
+		Description: ptr.String("Indicates whether directories should be included in the response."),
+		Required:    ptr.Bool(false),
+		Schema: &openapi3.SchemaOrRef{
+			Schema: &openapi3.Schema{
+				Type:    ptrSchemaType(openapi3.SchemaTypeBoolean),
+				Default: ptrptr(false),
+			},
+		},
+	},
+}
+
 var QueryParamIncludeStats = openapi3.ParameterOrRef{
 	Parameter: &openapi3.Parameter{
 		Name:        request.QueryParamIncludeStats,
@@ -606,6 +621,18 @@ func repoOperations(reflector *openapi3.Reflector) {
 	_ = reflector.SetJSONResponse(&opGetContent, new(usererror.Error), http.StatusNotFound)
 	_ = reflector.Spec.AddOperation(http.MethodGet, "/repos/{repo_ref}/content/{path}", opGetContent)
 
+	opListPaths := openapi3.Operation{}
+	opListPaths.WithTags("repository")
+	opListPaths.WithMapOfAnything(map[string]interface{}{"operationId": "listPaths"})
+	opListPaths.WithParameters(queryParameterGitRef, queryParameterIncludeDirectories)
+	_ = reflector.SetRequest(&opListPaths, new(repoRequest), http.MethodGet)
+	_ = reflector.SetJSONResponse(&opListPaths, new(repo.ListPathsOutput), http.StatusOK)
+	_ = reflector.SetJSONResponse(&opListPaths, new(usererror.Error), http.StatusInternalServerError)
+	_ = reflector.SetJSONResponse(&opListPaths, new(usererror.Error), http.StatusUnauthorized)
+	_ = reflector.SetJSONResponse(&opListPaths, new(usererror.Error), http.StatusForbidden)
+	_ = reflector.SetJSONResponse(&opListPaths, new(usererror.Error), http.StatusNotFound)
+	_ = reflector.Spec.AddOperation(http.MethodGet, "/repos/{repo_ref}/paths", opListPaths)
+
 	opPathDetails := openapi3.Operation{}
 	opPathDetails.WithTags("repository")
 	opPathDetails.WithMapOfAnything(map[string]interface{}{"operationId": "pathDetails"})

app/api/request/git.go

@@ -27,19 +27,20 @@ import (
 )
 
 const (
-	QueryParamGitRef        = "git_ref"
-	QueryParamIncludeCommit = "include_commit"
-	PathParamCommitSHA      = "commit_sha"
-	QueryParamLineFrom      = "line_from"
-	QueryParamLineTo        = "line_to"
-	QueryParamPath          = "path"
-	QueryParamSince         = "since"
-	QueryParamUntil         = "until"
-	QueryParamCommitter     = "committer"
-	QueryParamIncludeStats  = "include_stats"
-	QueryParamInternal      = "internal"
-	QueryParamService       = "service"
-	HeaderParamGitProtocol  = "Git-Protocol"
+	QueryParamGitRef             = "git_ref"
+	QueryParamIncludeCommit      = "include_commit"
+	QueryParamIncludeDirectories = "include_directories"
+	PathParamCommitSHA           = "commit_sha"
+	QueryParamLineFrom           = "line_from"
+	QueryParamLineTo             = "line_to"
+	QueryParamPath               = "path"
+	QueryParamSince              = "since"
+	QueryParamUntil              = "until"
+	QueryParamCommitter          = "committer"
+	QueryParamIncludeStats       = "include_stats"
+	QueryParamInternal           = "internal"
+	QueryParamService            = "service"
+	HeaderParamGitProtocol       = "Git-Protocol"
 )
 
 func GetGitRefFromQueryOrDefault(r *http.Request, deflt string) string {
@@ -50,6 +51,10 @@ func GetIncludeCommitFromQueryOrDefault(r *http.Request, deflt bool) (bool, erro
 	return QueryParamAsBoolOrDefault(r, QueryParamIncludeCommit, deflt)
 }
 
+func GetIncludeDirectoriesFromQueryOrDefault(r *http.Request, deflt bool) (bool, error) {
+	return QueryParamAsBoolOrDefault(r, QueryParamIncludeDirectories, deflt)
+}
+
 func GetCommitSHAFromPath(r *http.Request) (string, error) {
 	return PathParamOrError(r, PathParamCommitSHA)
 }

app/paths/paths.go

@@ -98,7 +98,9 @@ func IsAncesterOf(path string, other string) bool {
 	)
 }
 
-func Space(repoPath string) string {
-	spacePath, _, _ := DisectLeaf(repoPath)
+// Parent returns the parent path of the provided path.
+// if the path doesn't have a parent an empty string is returned.
+func Parent(path string) string {
+	spacePath, _, _ := DisectLeaf(path)
 	return spacePath
 }

app/router/api.go

@@ -294,6 +294,7 @@ func setupRepos(r chi.Router,
 				r.Get("/*", handlerrepo.HandleGetContent(repoCtrl))
 			})
 
+			r.Get("/paths", handlerrepo.HandleListPaths(repoCtrl))
 			r.Post("/path-details", handlerrepo.HandlePathsDetails(repoCtrl))
 
 			r.Route("/blame", func(r chi.Router) {

audit/audit.go

@@ -17,6 +17,7 @@ package audit
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	"github.com/harness/gitness/types"
 )
@@ -102,7 +103,7 @@ type Event struct {
 
 func (e *Event) Validate() error {
 	if err := e.Action.Validate(); err != nil {
-		return err
+		return fmt.Errorf("invalid action: %w", err)
 	}
 	if e.User.UID == "" {
 		return ErrUserIsRequired
@@ -111,7 +112,7 @@ func (e *Event) Validate() error {
 		return ErrSpacePathIsRequired
 	}
 	if err := e.Resource.Validate(); err != nil {
-		return err
+		return fmt.Errorf("invalid resource: %w", err)
 	}
 	return nil
 }

git/api/branch.go

@@ -56,7 +56,7 @@ func (g *Git) GetBranch(
 	}
 
 	ref := GetReferenceFromBranchName(branchName)
-	commit, err := GetCommit(ctx, repoPath, ref+"^{commit}")
+	commit, err := GetCommit(ctx, repoPath, ref+"^{commit}") //nolint:goconst
 	if err != nil {
 		return nil, fmt.Errorf("failed to find the commit for the branch: %w", err)
 	}

git/api/commit.go

@@ -359,7 +359,7 @@ func gitGetRenameDetails(
 func gitLogNameStatus(ctx context.Context, repoPath string, sha sha.SHA) ([]string, error) {
 	cmd := command.New("log",
 		command.WithFlag("--name-status"),
-		command.WithFlag("--format="),
+		command.WithFlag("--format="), //nolint:goconst
 		command.WithFlag("--max-count=1"),
 		command.WithArg(sha.String()),
 	)
@@ -378,7 +378,7 @@ func gitShowNumstat(
 ) ([]string, error) {
 	cmd := command.New("show",
 		command.WithFlag("--numstat"),
-		command.WithFlag("--format="),
+		command.WithFlag("--format="), //nolint:goconst
 		command.WithArg(sha.String()),
 	)
 	output := &bytes.Buffer{}
@@ -682,7 +682,7 @@ func getCommit(
 
 	cmd := command.New("log",
 		command.WithFlag("--max-count", "1"),
-		command.WithFlag("--format="+format),
+		command.WithFlag("--format="+format), //nolint:goconst
 		command.WithArg(rev),
 	)
 	if path != "" {

git/api/format.go

@@ -35,4 +35,7 @@ const (
 
 	fmtSubject = "%s"
 	fmtBody    = "%B"
+
+	fmtFieldObjectType = "%(objecttype)"
+	fmtFieldPath       = "%(path)"
 )

git/api/tree.go

@@ -144,6 +144,9 @@ func lsTree(
 		command.WithStdout(output),
 	)
 	if err != nil {
+		if strings.Contains(err.Error(), "fatal: not a tree object") {
+			return nil, errors.InvalidArgument("revision %q does not point to a commit", rev)
+		}
 		if strings.Contains(err.Error(), "fatal: Not a valid object name") {
 			return nil, errors.NotFound("revision %q not found", rev)
 		}
@@ -234,40 +237,45 @@ func lsFile(
 
 // GetTreeNode returns the tree node at the given path as found for the provided reference.
 func (g *Git) GetTreeNode(ctx context.Context, repoPath, rev, treePath string) (*TreeNode, error) {
-	// root path (empty path) is a special case
-	if treePath == "" {
-		if repoPath == "" {
-			return nil, ErrRepositoryPathEmpty
-		}
-		cmd := command.New("show",
-			command.WithFlag("--no-patch"),
-			command.WithFlag("--format="+fmtTreeHash),
-			command.WithArg(rev+"^{commit}"),
-		)
-		output := &bytes.Buffer{}
-		err := cmd.Run(ctx, command.WithDir(repoPath), command.WithStdout(output))
+	if repoPath == "" {
+		return nil, ErrRepositoryPathEmpty
+	}
+
+	// anything that's not the root path is a simple call
+	if treePath != "" {
+		treeNode, err := lsFile(ctx, repoPath, rev, treePath)
 		if err != nil {
-			if strings.Contains(err.Error(), "ambiguous argument") {
-				return nil, errors.NotFound("could not resolve git revision: %s", rev)
-			}
-			return nil, fmt.Errorf("failed to get root tree node: %w", err)
+			return nil, fmt.Errorf("failed to get tree node: %w", err)
 		}
 
-		return &TreeNode{
-			NodeType: TreeNodeTypeTree,
-			Mode:     TreeNodeModeTree,
-			SHA:      sha.Must(output.String()),
-			Name:     "",
-			Path:     "",
-		}, err
+		return &treeNode, nil
 	}
 
-	treeNode, err := lsFile(ctx, repoPath, rev, treePath)
+	// root path (empty path) is a special case
+	cmd := command.New("show",
+		command.WithFlag("--no-patch"),
+		command.WithFlag("--format="+fmtTreeHash), //nolint:goconst
+		command.WithArg(rev+"^{commit}"),          //nolint:goconst
+	)
+	output := &bytes.Buffer{}
+	err := cmd.Run(ctx, command.WithDir(repoPath), command.WithStdout(output))
 	if err != nil {
-		return nil, fmt.Errorf("failed to get tree node: %w", err)
+		if strings.Contains(err.Error(), "expected commit type") {
+			return nil, errors.InvalidArgument("revision %q does not point to a commit", rev)
+		}
+		if strings.Contains(err.Error(), "unknown revision") {
+			return nil, errors.NotFound("revision %q not found", rev)
+		}
+		return nil, fmt.Errorf("failed to get root tree node: %w", err)
 	}
 
-	return &treeNode, nil
+	return &TreeNode{
+		NodeType: TreeNodeTypeTree,
+		Mode:     TreeNodeModeTree,
+		SHA:      sha.Must(output.String()),
+		Name:     "",
+		Path:     "",
+	}, err
 }
 
 // ListTreeNodes lists the child nodes of a tree reachable from ref via the specified path.
@@ -299,3 +307,75 @@ func (g *Git) ReadTree(
 	}
 	return nil
 }
+
+// ListPaths lists all the paths in a repo recursively (similar-ish to `ls -lR`).
+// Note: Keep method simple for now to avoid unnecessary corner cases
+// by always listing whole repo content (and not relative to any directory).
+func (g *Git) ListPaths(
+	ctx context.Context,
+	repoPath string,
+	rev string,
+	includeDirs bool,
+) (files []string, dirs []string, err error) {
+	if repoPath == "" {
+		return nil, nil, ErrRepositoryPathEmpty
+	}
+
+	// use custom ls-tree for speed up (file listing is ~10% faster, with dirs roughly the same)
+	cmd := command.New("ls-tree",
+		command.WithConfig("core.quotePath", "false"), // force printing of path in custom format without quoting
+		command.WithFlag("-z"),
+		command.WithFlag("-r"),
+		command.WithFlag("--full-name"),
+		command.WithArg(rev+"^{commit}"), //nolint:goconst enforce commit revs for now (keep it simple)
+	)
+	format := fmtFieldPath
+	if includeDirs {
+		cmd.Add(command.WithFlag("-t"))
+		format += fmtZero + fmtFieldObjectType
+	}
+	cmd.Add(command.WithFlag("--format=" + format)) //nolint:goconst
+
+	output := &bytes.Buffer{}
+	err = cmd.Run(ctx,
+		command.WithDir(repoPath),
+		command.WithStdout(output),
+	)
+	if err != nil {
+		if strings.Contains(err.Error(), "expected commit type") {
+			return nil, nil, errors.InvalidArgument("revision %q does not point to a commit", rev)
+		}
+		if strings.Contains(err.Error(), "fatal: Not a valid object name") {
+			return nil, nil, errors.NotFound("revision %q not found", rev)
+		}
+		return nil, nil, fmt.Errorf("failed to run git ls-tree: %w", err)
+	}
+
+	scanner := bufio.NewScanner(output)
+	scanner.Split(parser.ScanZeroSeparated)
+	for scanner.Scan() {
+		path := scanner.Text()
+
+		isDir := false
+		if includeDirs {
+			// custom format guarantees the object type in the next scan
+			if !scanner.Scan() {
+				return nil, nil, fmt.Errorf("unexpected output from ls-tree when getting object type: %w", scanner.Err())
+			}
+			objectType := scanner.Text()
+			isDir = strings.EqualFold(objectType, string(GitObjectTypeTree))
+		}
+
+		if isDir {
+			dirs = append(dirs, path)
+			continue
+		}
+
+		files = append(files, path)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, nil, fmt.Errorf("error reading ls-tree output: %w", err)
+	}
+
+	return files, dirs, nil
+}

git/interface.go

@@ -26,6 +26,7 @@ type Interface interface {
 	DeleteRepository(ctx context.Context, params *DeleteRepositoryParams) error
 	GetTreeNode(ctx context.Context, params *GetTreeNodeParams) (*GetTreeNodeOutput, error)
 	ListTreeNodes(ctx context.Context, params *ListTreeNodeParams) (*ListTreeNodeOutput, error)
+	ListPaths(ctx context.Context, params *ListPathsParams) (*ListPathsOutput, error)
 	GetSubmodule(ctx context.Context, params *GetSubmoduleParams) (*GetSubmoduleOutput, error)
 	GetBlob(ctx context.Context, params *GetBlobParams) (*GetBlobOutput, error)
 	CreateBranch(ctx context.Context, params *CreateBranchParams) (*CreateBranchOutput, error)

git/tree.go

@@ -150,6 +150,42 @@ func (s *Service) ListTreeNodes(ctx context.Context, params *ListTreeNodeParams)
 	}, nil
 }
 
+type ListPathsParams struct {
+	ReadParams
+	// GitREF is a git reference (branch / tag / commit SHA)
+	GitREF             string
+	IncludeDirectories bool
+}
+
+type ListPathsOutput struct {
+	Files       []string
+	Directories []string
+}
+
+func (s *Service) ListPaths(ctx context.Context, params *ListPathsParams) (*ListPathsOutput, error) {
+	if err := params.Validate(); err != nil {
+		return nil, err
+	}
+
+	repoPath := getFullPathForRepo(s.reposRoot, params.RepoUID)
+
+	files, dirs, err := s.git.ListPaths(
+		ctx,
+		repoPath,
+		params.GitREF,
+		params.IncludeDirectories,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list paths: %w", err)
+	}
+
+	return &ListPathsOutput{
+			Files:       files,
+			Directories: dirs,
+		},
+		nil
+}
+
 type PathsDetailsParams struct {
 	ReadParams
 	GitREF string