diff --git a/app/api/controller/githook/pre_receive_scan_secrets.go b/app/api/controller/githook/pre_receive_scan_secrets.go index 8ea10f652..bbcf34b17 100644 --- a/app/api/controller/githook/pre_receive_scan_secrets.go +++ b/app/api/controller/githook/pre_receive_scan_secrets.go @@ -98,8 +98,8 @@ func scanSecretsInternal(ctx context.Context, } // in case the branch was just created - fallback to compare against latest default branch. - baseRev := refUpdate.Old.String() + "^{commit}" - rev := refUpdate.New.String() + "^{commit}" + baseRev := refUpdate.Old.String() + "^{commit}" //nolint:goconst + rev := refUpdate.New.String() + "^{commit}" //nolint:goconst //nolint:nestif if refUpdate.Old.IsNil() { if baseRevFallBack == nil { diff --git a/app/api/controller/repo/create.go b/app/api/controller/repo/create.go index 4d72ba68b..b62099abb 100644 --- a/app/api/controller/repo/create.go +++ b/app/api/controller/repo/create.go @@ -127,7 +127,7 @@ func (c *Controller) Create(ctx context.Context, session *auth.Session, in *Crea session.Principal, audit.NewResource(audit.ResourceTypeRepository, repo.Identifier), audit.ActionCreated, - paths.Space(repo.Path), + paths.Parent(repo.Path), audit.WithNewObject(repo), ) if err != nil { diff --git a/app/api/controller/repo/default_branch.go b/app/api/controller/repo/default_branch.go index 74970dbd2..d2d42b2d4 100644 --- a/app/api/controller/repo/default_branch.go +++ b/app/api/controller/repo/default_branch.go @@ -99,7 +99,7 @@ func (c *Controller) UpdateDefaultBranch( session.Principal, audit.NewResource(audit.ResourceTypeRepository, repo.Identifier), audit.ActionUpdated, - paths.Space(repo.Path), + paths.Parent(repo.Path), audit.WithOldObject(repoClone), audit.WithNewObject(repo), ) diff --git a/app/api/controller/repo/list_paths.go b/app/api/controller/repo/list_paths.go new file mode 100644 index 000000000..0a388a749 --- /dev/null +++ b/app/api/controller/repo/list_paths.go @@ -0,0 +1,61 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repo + +import ( + "context" + "fmt" + + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/git" + "github.com/harness/gitness/types/enum" +) + +type ListPathsOutput struct { + Files []string `json:"files,omitempty"` + Directories []string `json:"directories,omitempty"` +} + +// ListPaths lists the paths in the repo for a specific revision. +func (c *Controller) ListPaths(ctx context.Context, + session *auth.Session, + repoRef string, + gitRef string, + includeDirectories bool, +) (ListPathsOutput, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) + if err != nil { + return ListPathsOutput{}, err + } + + // set gitRef to default branch in case an empty reference was provided + if gitRef == "" { + gitRef = repo.DefaultBranch + } + + rpcOut, err := c.git.ListPaths(ctx, &git.ListPathsParams{ + ReadParams: git.CreateReadParams(repo), + GitREF: gitRef, + IncludeDirectories: includeDirectories, + }) + if err != nil { + return ListPathsOutput{}, fmt.Errorf("failed to list git paths: %w", err) + } + + return ListPathsOutput{ + Files: rpcOut.Files, + Directories: rpcOut.Directories, + }, nil +} diff --git a/app/api/controller/repo/rule_create.go b/app/api/controller/repo/rule_create.go index 86ae87f23..4a85820af 100644 --- a/app/api/controller/repo/rule_create.go +++ b/app/api/controller/repo/rule_create.go @@ -120,7 +120,7 @@ func (c *Controller) RuleCreate(ctx context.Context, session.Principal, audit.NewResource(audit.ResourceTypeBranchRule, r.Identifier), audit.ActionCreated, - paths.Space(repo.Path), + paths.Parent(repo.Path), audit.WithNewObject(r), ) if err != nil { diff --git a/app/api/controller/repo/rule_delete.go b/app/api/controller/repo/rule_delete.go index b9b7ac142..cb561b56d 100644 --- a/app/api/controller/repo/rule_delete.go +++ b/app/api/controller/repo/rule_delete.go @@ -51,7 +51,7 @@ func (c *Controller) RuleDelete(ctx context.Context, session.Principal, audit.NewResource(audit.ResourceTypeBranchRule, r.Identifier), audit.ActionDeleted, - paths.Space(repo.Path), + paths.Parent(repo.Path), audit.WithOldObject(r), ) if err != nil { diff --git a/app/api/controller/repo/rule_update.go b/app/api/controller/repo/rule_update.go index 6aa304788..dc4011781 100644 --- a/app/api/controller/repo/rule_update.go +++ b/app/api/controller/repo/rule_update.go @@ -142,7 +142,7 @@ func (c *Controller) RuleUpdate(ctx context.Context, session.Principal, audit.NewResource(audit.ResourceTypeBranchRule, r.Identifier), audit.ActionUpdated, - paths.Space(repo.Path), + paths.Parent(repo.Path), audit.WithOldObject(oldRule), audit.WithNewObject(r), ) diff --git a/app/api/controller/repo/soft_delete.go b/app/api/controller/repo/soft_delete.go index 3168fba46..6fd09c87a 100644 --- a/app/api/controller/repo/soft_delete.go +++ b/app/api/controller/repo/soft_delete.go @@ -68,7 +68,7 @@ func (c *Controller) SoftDelete( session.Principal, audit.NewResource(audit.ResourceTypeRepository, repo.Identifier), audit.ActionDeleted, - paths.Space(repo.Path), + paths.Parent(repo.Path), audit.WithOldObject(repo), ) if err != nil { diff --git a/app/api/controller/repo/update.go b/app/api/controller/repo/update.go index 7e57c07d2..579f69f3d 100644 --- a/app/api/controller/repo/update.go +++ b/app/api/controller/repo/update.go @@ -80,7 +80,7 @@ func (c *Controller) Update(ctx context.Context, session.Principal, audit.NewResource(audit.ResourceTypeRepository, repo.Identifier), audit.ActionUpdated, - paths.Space(repo.Path), + paths.Parent(repo.Path), audit.WithOldObject(repoClone), audit.WithNewObject(repo), ) diff --git a/app/api/handler/repo/list_paths.go b/app/api/handler/repo/list_paths.go new file mode 100644 index 000000000..9667b0021 --- /dev/null +++ b/app/api/handler/repo/list_paths.go @@ -0,0 +1,50 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repo + +import ( + "net/http" + + "github.com/harness/gitness/app/api/controller/repo" + "github.com/harness/gitness/app/api/render" + "github.com/harness/gitness/app/api/request" +) + +func HandleListPaths(repoCtrl *repo.Controller) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + session, _ := request.AuthSessionFrom(ctx) + repoRef, err := request.GetRepoRefFromPath(r) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + gitRef := request.GetGitRefFromQueryOrDefault(r, "") + includeDirectories, err := request.GetIncludeDirectoriesFromQueryOrDefault(r, false) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + out, err := repoCtrl.ListPaths(ctx, session, repoRef, gitRef, includeDirectories) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + render.JSON(w, http.StatusOK, out) + } +} diff --git a/app/api/openapi/repo.go b/app/api/openapi/repo.go index 90c95f138..884936b38 100644 --- a/app/api/openapi/repo.go +++ b/app/api/openapi/repo.go @@ -273,6 +273,21 @@ var queryParameterIncludeCommit = openapi3.ParameterOrRef{ }, } +var queryParameterIncludeDirectories = openapi3.ParameterOrRef{ + Parameter: &openapi3.Parameter{ + Name: request.QueryParamIncludeDirectories, + In: openapi3.ParameterInQuery, + Description: ptr.String("Indicates whether directories should be included in the response."), + Required: ptr.Bool(false), + Schema: &openapi3.SchemaOrRef{ + Schema: &openapi3.Schema{ + Type: ptrSchemaType(openapi3.SchemaTypeBoolean), + Default: ptrptr(false), + }, + }, + }, +} + var QueryParamIncludeStats = openapi3.ParameterOrRef{ Parameter: &openapi3.Parameter{ Name: request.QueryParamIncludeStats, @@ -606,6 +621,18 @@ func repoOperations(reflector *openapi3.Reflector) { _ = reflector.SetJSONResponse(&opGetContent, new(usererror.Error), http.StatusNotFound) _ = reflector.Spec.AddOperation(http.MethodGet, "/repos/{repo_ref}/content/{path}", opGetContent) + opListPaths := openapi3.Operation{} + opListPaths.WithTags("repository") + opListPaths.WithMapOfAnything(map[string]interface{}{"operationId": "listPaths"}) + opListPaths.WithParameters(queryParameterGitRef, queryParameterIncludeDirectories) + _ = reflector.SetRequest(&opListPaths, new(repoRequest), http.MethodGet) + _ = reflector.SetJSONResponse(&opListPaths, new(repo.ListPathsOutput), http.StatusOK) + _ = reflector.SetJSONResponse(&opListPaths, new(usererror.Error), http.StatusInternalServerError) + _ = reflector.SetJSONResponse(&opListPaths, new(usererror.Error), http.StatusUnauthorized) + _ = reflector.SetJSONResponse(&opListPaths, new(usererror.Error), http.StatusForbidden) + _ = reflector.SetJSONResponse(&opListPaths, new(usererror.Error), http.StatusNotFound) + _ = reflector.Spec.AddOperation(http.MethodGet, "/repos/{repo_ref}/paths", opListPaths) + opPathDetails := openapi3.Operation{} opPathDetails.WithTags("repository") opPathDetails.WithMapOfAnything(map[string]interface{}{"operationId": "pathDetails"}) diff --git a/app/api/request/git.go b/app/api/request/git.go index 087cd702d..ec6f63b72 100644 --- a/app/api/request/git.go +++ b/app/api/request/git.go @@ -27,19 +27,20 @@ import ( ) const ( - QueryParamGitRef = "git_ref" - QueryParamIncludeCommit = "include_commit" - PathParamCommitSHA = "commit_sha" - QueryParamLineFrom = "line_from" - QueryParamLineTo = "line_to" - QueryParamPath = "path" - QueryParamSince = "since" - QueryParamUntil = "until" - QueryParamCommitter = "committer" - QueryParamIncludeStats = "include_stats" - QueryParamInternal = "internal" - QueryParamService = "service" - HeaderParamGitProtocol = "Git-Protocol" + QueryParamGitRef = "git_ref" + QueryParamIncludeCommit = "include_commit" + QueryParamIncludeDirectories = "include_directories" + PathParamCommitSHA = "commit_sha" + QueryParamLineFrom = "line_from" + QueryParamLineTo = "line_to" + QueryParamPath = "path" + QueryParamSince = "since" + QueryParamUntil = "until" + QueryParamCommitter = "committer" + QueryParamIncludeStats = "include_stats" + QueryParamInternal = "internal" + QueryParamService = "service" + HeaderParamGitProtocol = "Git-Protocol" ) func GetGitRefFromQueryOrDefault(r *http.Request, deflt string) string { @@ -50,6 +51,10 @@ func GetIncludeCommitFromQueryOrDefault(r *http.Request, deflt bool) (bool, erro return QueryParamAsBoolOrDefault(r, QueryParamIncludeCommit, deflt) } +func GetIncludeDirectoriesFromQueryOrDefault(r *http.Request, deflt bool) (bool, error) { + return QueryParamAsBoolOrDefault(r, QueryParamIncludeDirectories, deflt) +} + func GetCommitSHAFromPath(r *http.Request) (string, error) { return PathParamOrError(r, PathParamCommitSHA) } diff --git a/app/paths/paths.go b/app/paths/paths.go index 2db8e5412..591d0e276 100644 --- a/app/paths/paths.go +++ b/app/paths/paths.go @@ -98,7 +98,9 @@ func IsAncesterOf(path string, other string) bool { ) } -func Space(repoPath string) string { - spacePath, _, _ := DisectLeaf(repoPath) +// Parent returns the parent path of the provided path. +// if the path doesn't have a parent an empty string is returned. +func Parent(path string) string { + spacePath, _, _ := DisectLeaf(path) return spacePath } diff --git a/app/router/api.go b/app/router/api.go index 6dbc848d0..f21dbbf9e 100644 --- a/app/router/api.go +++ b/app/router/api.go @@ -294,6 +294,7 @@ func setupRepos(r chi.Router, r.Get("/*", handlerrepo.HandleGetContent(repoCtrl)) }) + r.Get("/paths", handlerrepo.HandleListPaths(repoCtrl)) r.Post("/path-details", handlerrepo.HandlePathsDetails(repoCtrl)) r.Route("/blame", func(r chi.Router) { diff --git a/audit/audit.go b/audit/audit.go index fce7ad0f6..ca1b4ebfe 100644 --- a/audit/audit.go +++ b/audit/audit.go @@ -17,6 +17,7 @@ package audit import ( "context" "errors" + "fmt" "github.com/harness/gitness/types" ) @@ -102,7 +103,7 @@ type Event struct { func (e *Event) Validate() error { if err := e.Action.Validate(); err != nil { - return err + return fmt.Errorf("invalid action: %w", err) } if e.User.UID == "" { return ErrUserIsRequired @@ -111,7 +112,7 @@ func (e *Event) Validate() error { return ErrSpacePathIsRequired } if err := e.Resource.Validate(); err != nil { - return err + return fmt.Errorf("invalid resource: %w", err) } return nil } diff --git a/git/api/branch.go b/git/api/branch.go index 7b4d7effb..65ae0c672 100644 --- a/git/api/branch.go +++ b/git/api/branch.go @@ -56,7 +56,7 @@ func (g *Git) GetBranch( } ref := GetReferenceFromBranchName(branchName) - commit, err := GetCommit(ctx, repoPath, ref+"^{commit}") + commit, err := GetCommit(ctx, repoPath, ref+"^{commit}") //nolint:goconst if err != nil { return nil, fmt.Errorf("failed to find the commit for the branch: %w", err) } diff --git a/git/api/commit.go b/git/api/commit.go index b8c8d358b..1e10412db 100644 --- a/git/api/commit.go +++ b/git/api/commit.go @@ -359,7 +359,7 @@ func gitGetRenameDetails( func gitLogNameStatus(ctx context.Context, repoPath string, sha sha.SHA) ([]string, error) { cmd := command.New("log", command.WithFlag("--name-status"), - command.WithFlag("--format="), + command.WithFlag("--format="), //nolint:goconst command.WithFlag("--max-count=1"), command.WithArg(sha.String()), ) @@ -378,7 +378,7 @@ func gitShowNumstat( ) ([]string, error) { cmd := command.New("show", command.WithFlag("--numstat"), - command.WithFlag("--format="), + command.WithFlag("--format="), //nolint:goconst command.WithArg(sha.String()), ) output := &bytes.Buffer{} @@ -682,7 +682,7 @@ func getCommit( cmd := command.New("log", command.WithFlag("--max-count", "1"), - command.WithFlag("--format="+format), + command.WithFlag("--format="+format), //nolint:goconst command.WithArg(rev), ) if path != "" { diff --git a/git/api/format.go b/git/api/format.go index c8f49ccc9..951e1c330 100644 --- a/git/api/format.go +++ b/git/api/format.go @@ -35,4 +35,7 @@ const ( fmtSubject = "%s" fmtBody = "%B" + + fmtFieldObjectType = "%(objecttype)" + fmtFieldPath = "%(path)" ) diff --git a/git/api/tree.go b/git/api/tree.go index 05204a944..0b720a47e 100644 --- a/git/api/tree.go +++ b/git/api/tree.go @@ -144,6 +144,9 @@ func lsTree( command.WithStdout(output), ) if err != nil { + if strings.Contains(err.Error(), "fatal: not a tree object") { + return nil, errors.InvalidArgument("revision %q does not point to a commit", rev) + } if strings.Contains(err.Error(), "fatal: Not a valid object name") { return nil, errors.NotFound("revision %q not found", rev) } @@ -234,40 +237,45 @@ func lsFile( // GetTreeNode returns the tree node at the given path as found for the provided reference. func (g *Git) GetTreeNode(ctx context.Context, repoPath, rev, treePath string) (*TreeNode, error) { - // root path (empty path) is a special case - if treePath == "" { - if repoPath == "" { - return nil, ErrRepositoryPathEmpty - } - cmd := command.New("show", - command.WithFlag("--no-patch"), - command.WithFlag("--format="+fmtTreeHash), - command.WithArg(rev+"^{commit}"), - ) - output := &bytes.Buffer{} - err := cmd.Run(ctx, command.WithDir(repoPath), command.WithStdout(output)) + if repoPath == "" { + return nil, ErrRepositoryPathEmpty + } + + // anything that's not the root path is a simple call + if treePath != "" { + treeNode, err := lsFile(ctx, repoPath, rev, treePath) if err != nil { - if strings.Contains(err.Error(), "ambiguous argument") { - return nil, errors.NotFound("could not resolve git revision: %s", rev) - } - return nil, fmt.Errorf("failed to get root tree node: %w", err) + return nil, fmt.Errorf("failed to get tree node: %w", err) } - return &TreeNode{ - NodeType: TreeNodeTypeTree, - Mode: TreeNodeModeTree, - SHA: sha.Must(output.String()), - Name: "", - Path: "", - }, err + return &treeNode, nil } - treeNode, err := lsFile(ctx, repoPath, rev, treePath) + // root path (empty path) is a special case + cmd := command.New("show", + command.WithFlag("--no-patch"), + command.WithFlag("--format="+fmtTreeHash), //nolint:goconst + command.WithArg(rev+"^{commit}"), //nolint:goconst + ) + output := &bytes.Buffer{} + err := cmd.Run(ctx, command.WithDir(repoPath), command.WithStdout(output)) if err != nil { - return nil, fmt.Errorf("failed to get tree node: %w", err) + if strings.Contains(err.Error(), "expected commit type") { + return nil, errors.InvalidArgument("revision %q does not point to a commit", rev) + } + if strings.Contains(err.Error(), "unknown revision") { + return nil, errors.NotFound("revision %q not found", rev) + } + return nil, fmt.Errorf("failed to get root tree node: %w", err) } - return &treeNode, nil + return &TreeNode{ + NodeType: TreeNodeTypeTree, + Mode: TreeNodeModeTree, + SHA: sha.Must(output.String()), + Name: "", + Path: "", + }, err } // ListTreeNodes lists the child nodes of a tree reachable from ref via the specified path. @@ -299,3 +307,75 @@ func (g *Git) ReadTree( } return nil } + +// ListPaths lists all the paths in a repo recursively (similar-ish to `ls -lR`). +// Note: Keep method simple for now to avoid unnecessary corner cases +// by always listing whole repo content (and not relative to any directory). +func (g *Git) ListPaths( + ctx context.Context, + repoPath string, + rev string, + includeDirs bool, +) (files []string, dirs []string, err error) { + if repoPath == "" { + return nil, nil, ErrRepositoryPathEmpty + } + + // use custom ls-tree for speed up (file listing is ~10% faster, with dirs roughly the same) + cmd := command.New("ls-tree", + command.WithConfig("core.quotePath", "false"), // force printing of path in custom format without quoting + command.WithFlag("-z"), + command.WithFlag("-r"), + command.WithFlag("--full-name"), + command.WithArg(rev+"^{commit}"), //nolint:goconst enforce commit revs for now (keep it simple) + ) + format := fmtFieldPath + if includeDirs { + cmd.Add(command.WithFlag("-t")) + format += fmtZero + fmtFieldObjectType + } + cmd.Add(command.WithFlag("--format=" + format)) //nolint:goconst + + output := &bytes.Buffer{} + err = cmd.Run(ctx, + command.WithDir(repoPath), + command.WithStdout(output), + ) + if err != nil { + if strings.Contains(err.Error(), "expected commit type") { + return nil, nil, errors.InvalidArgument("revision %q does not point to a commit", rev) + } + if strings.Contains(err.Error(), "fatal: Not a valid object name") { + return nil, nil, errors.NotFound("revision %q not found", rev) + } + return nil, nil, fmt.Errorf("failed to run git ls-tree: %w", err) + } + + scanner := bufio.NewScanner(output) + scanner.Split(parser.ScanZeroSeparated) + for scanner.Scan() { + path := scanner.Text() + + isDir := false + if includeDirs { + // custom format guarantees the object type in the next scan + if !scanner.Scan() { + return nil, nil, fmt.Errorf("unexpected output from ls-tree when getting object type: %w", scanner.Err()) + } + objectType := scanner.Text() + isDir = strings.EqualFold(objectType, string(GitObjectTypeTree)) + } + + if isDir { + dirs = append(dirs, path) + continue + } + + files = append(files, path) + } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("error reading ls-tree output: %w", err) + } + + return files, dirs, nil +} diff --git a/git/interface.go b/git/interface.go index 19836f855..ec76267c7 100644 --- a/git/interface.go +++ b/git/interface.go @@ -26,6 +26,7 @@ type Interface interface { DeleteRepository(ctx context.Context, params *DeleteRepositoryParams) error GetTreeNode(ctx context.Context, params *GetTreeNodeParams) (*GetTreeNodeOutput, error) ListTreeNodes(ctx context.Context, params *ListTreeNodeParams) (*ListTreeNodeOutput, error) + ListPaths(ctx context.Context, params *ListPathsParams) (*ListPathsOutput, error) GetSubmodule(ctx context.Context, params *GetSubmoduleParams) (*GetSubmoduleOutput, error) GetBlob(ctx context.Context, params *GetBlobParams) (*GetBlobOutput, error) CreateBranch(ctx context.Context, params *CreateBranchParams) (*CreateBranchOutput, error) diff --git a/git/tree.go b/git/tree.go index f766eea5c..d3b085fdd 100644 --- a/git/tree.go +++ b/git/tree.go @@ -150,6 +150,42 @@ func (s *Service) ListTreeNodes(ctx context.Context, params *ListTreeNodeParams) }, nil } +type ListPathsParams struct { + ReadParams + // GitREF is a git reference (branch / tag / commit SHA) + GitREF string + IncludeDirectories bool +} + +type ListPathsOutput struct { + Files []string + Directories []string +} + +func (s *Service) ListPaths(ctx context.Context, params *ListPathsParams) (*ListPathsOutput, error) { + if err := params.Validate(); err != nil { + return nil, err + } + + repoPath := getFullPathForRepo(s.reposRoot, params.RepoUID) + + files, dirs, err := s.git.ListPaths( + ctx, + repoPath, + params.GitREF, + params.IncludeDirectories, + ) + if err != nil { + return nil, fmt.Errorf("failed to list paths: %w", err) + } + + return &ListPathsOutput{ + Files: files, + Directories: dirs, + }, + nil +} + type PathsDetailsParams struct { ReadParams GitREF string