feat: [PIPE-21976]: add flattened directory listing (#2869)

* remove unused parameter
* updated open api to include flatten_driectories param
* add flattened directory listing

app/api/controller/repo/content_get.go

@@ -100,6 +100,7 @@ func (c *Controller) GetContent(ctx context.Context,
 	gitRef string,
 	repoPath string,
 	includeLatestCommit bool,
+	flattenDirectories bool,
 ) (*GetContentOutput, error) {
 	repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView)
 	if err != nil {
@@ -132,7 +133,7 @@ func (c *Controller) GetContent(ctx context.Context,
 	var content Content
 	switch info.Type {
 	case ContentTypeDir:
-		content, err = c.getDirContent(ctx, readParams, gitRef, repoPath, includeLatestCommit)
+		content, err = c.getDirContent(ctx, readParams, gitRef, repoPath, flattenDirectories)
 	case ContentTypeFile:
 		content, err = c.getFileContent(ctx, readParams, info.SHA)
 	case ContentTypeSymlink:
@@ -242,13 +243,13 @@ func (c *Controller) getDirContent(ctx context.Context,
 	readParams git.ReadParams,
 	gitRef string,
 	repoPath string,
-	includeLatestCommit bool,
+	flattenDirectories bool,
 ) (*DirContent, error) {
 	output, err := c.git.ListTreeNodes(ctx, &git.ListTreeNodeParams{
-		ReadParams:          readParams,
-		GitREF:              gitRef,
-		Path:                repoPath,
-		IncludeLatestCommit: includeLatestCommit,
+		ReadParams:         readParams,
+		GitREF:             gitRef,
+		Path:               repoPath,
+		FlattenDirectories: flattenDirectories,
 	})
 	if err != nil {
 		// TODO: handle not found error

app/api/handler/repo/content_get.go

@@ -41,9 +41,15 @@ func HandleGetContent(repoCtrl *repo.Controller) http.HandlerFunc {
 			return
 		}
 
+		flattenDirectories, err := request.GetFlattenDirectoriesFromQueryOrDefault(r, false)
+		if err != nil {
+			render.TranslatedUserError(ctx, w, err)
+			return
+		}
+
 		repoPath := request.GetOptionalRemainderFromPath(r)
 
-		resp, err := repoCtrl.GetContent(ctx, session, repoRef, gitRef, repoPath, includeCommit)
+		resp, err := repoCtrl.GetContent(ctx, session, repoRef, gitRef, repoPath, includeCommit, flattenDirectories)
 		if err != nil {
 			render.TranslatedUserError(ctx, w, err)
 			return

app/api/openapi/repo.go

@@ -297,6 +297,21 @@ var queryParameterUntil = openapi3.ParameterOrRef{
 	},
 }
 
+var queryParameterFlattenDirectories = openapi3.ParameterOrRef{
+	Parameter: &openapi3.Parameter{
+		Name:        request.QueryParamFlattenDirectories,
+		In:          openapi3.ParameterInQuery,
+		Description: ptr.String("Flatten directories that contain just one subdirectory."),
+		Required:    ptr.Bool(false),
+		Schema: &openapi3.SchemaOrRef{
+			Schema: &openapi3.Schema{
+				Type:    ptrSchemaType(openapi3.SchemaTypeBoolean),
+				Default: ptrptr(false),
+			},
+		},
+	},
+}
+
 var queryParameterIncludeCommit = openapi3.ParameterOrRef{
 	Parameter: &openapi3.Parameter{
 		Name:        request.QueryParamIncludeCommit,
@@ -865,7 +880,7 @@ func repoOperations(reflector *openapi3.Reflector) {
 	opGetContent := openapi3.Operation{}
 	opGetContent.WithTags("repository")
 	opGetContent.WithMapOfAnything(map[string]interface{}{"operationId": "getContent"})
-	opGetContent.WithParameters(queryParameterGitRef, queryParameterIncludeCommit)
+	opGetContent.WithParameters(queryParameterGitRef, queryParameterIncludeCommit, queryParameterFlattenDirectories)
 	_ = reflector.SetRequest(&opGetContent, new(getContentRequest), http.MethodGet)
 	_ = reflector.SetJSONResponse(&opGetContent, new(getContentOutput), http.StatusOK)
 	_ = reflector.SetJSONResponse(&opGetContent, new(usererror.Error), http.StatusInternalServerError)

app/api/request/git.go

@@ -34,6 +34,7 @@ const (
 	QueryParamGitRef             = "git_ref"
 	QueryParamIncludeCommit      = "include_commit"
 	QueryParamIncludeDirectories = "include_directories"
+	QueryParamFlattenDirectories = "flatten_directories"
 	QueryParamLineFrom           = "line_from"
 	QueryParamLineTo             = "line_to"
 	QueryParamPath               = "path"
@@ -79,6 +80,10 @@ func GetIncludeDirectoriesFromQueryOrDefault(r *http.Request, deflt bool) (bool,
 	return QueryParamAsBoolOrDefault(r, QueryParamIncludeDirectories, deflt)
 }
 
+func GetFlattenDirectoriesFromQueryOrDefault(r *http.Request, deflt bool) (bool, error) {
+	return QueryParamAsBoolOrDefault(r, QueryParamFlattenDirectories, deflt)
+}
+
 func GetCommitSHAFromPath(r *http.Request) (string, error) {
 	return PathParamOrError(r, PathParamCommitSHA)
 }

app/services/capabilities/list_files.go

@@ -101,10 +101,9 @@ func listFiles(
 	files := make([]string, 0)
 	directories := make([]string, 0)
 	tree, err := gitI.ListTreeNodes(ctx, &git.ListTreeNodeParams{
-		ReadParams:          git.CreateReadParams(repo),
-		GitREF:              gitRef,
-		Path:                path,
-		IncludeLatestCommit: false,
+		ReadParams: git.CreateReadParams(repo),
+		GitREF:     gitRef,
+		Path:       path,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to list tree nodes: %w", err)

git/api/fs.go

@@ -164,7 +164,7 @@ func (f *FS) ReadFile(path string) ([]byte, error) {
 // ReadDir returns all entries for a directory.
 // It is part of the fs.ReadDirFS interface.
 func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) {
-	treeNodes, err := ListTreeNodes(f.ctx, f.dir, f.rev, name, true)
+	treeNodes, err := ListTreeNodes(f.ctx, f.dir, f.rev, name, true, false)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read git directory: %w", err)
 	}
@@ -225,7 +225,7 @@ func (d *fsDir) Close() error { return nil }
 // ReadDir lists entries in the directory. The integer parameter can be used for pagination.
 // It is part of the fs.ReadDirFile interface.
 func (d *fsDir) ReadDir(n int) ([]fs.DirEntry, error) {
-	treeNodes, err := ListTreeNodes(d.ctx, d.dir, d.treeSHA.String(), "", true)
+	treeNodes, err := ListTreeNodes(d.ctx, d.dir, d.treeSHA.String(), "", true, false)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read git directory: %w", err)
 	}

git/api/match_files.go

@@ -35,7 +35,7 @@ func (g *Git) MatchFiles(
 	pattern string,
 	maxSize int,
 ) ([]FileContent, error) {
-	nodes, err := lsDirectory(ctx, repoPath, rev, treePath, false)
+	nodes, err := lsDirectory(ctx, repoPath, rev, treePath, false, false)
 	if err != nil {
 		return nil, fmt.Errorf("failed to list files in match files: %w", err)
 	}

git/api/tree.go

@@ -228,6 +228,7 @@ func lsDirectory(
 	rev string,
 	treePath string,
 	fetchSizes bool,
+	flattenDirectories bool,
 ) ([]TreeNode, error) {
 	treePath = path.Clean(treePath)
 	if treePath == "" {
@@ -236,7 +237,24 @@ func lsDirectory(
 		treePath += "/"
 	}
 
-	return lsTree(ctx, repoPath, rev, treePath, fetchSizes)
+	nodes, err := lsTree(ctx, repoPath, rev, treePath, fetchSizes)
+	if err != nil {
+		return nil, err
+	}
+
+	if flattenDirectories {
+		for i := range nodes {
+			if nodes[i].NodeType != TreeNodeTypeTree {
+				continue
+			}
+
+			if err := flattenDirectory(ctx, repoPath, rev, &nodes[i], fetchSizes); err != nil {
+				return nil, fmt.Errorf("failed to flatten directory: %w", err)
+			}
+		}
+	}
+
+	return nodes, nil
 }
 
 // lsFile returns one tree node entry.
@@ -260,6 +278,38 @@ func lsFile(
 	return list[0], nil
 }
 
+func flattenDirectory(
+	ctx context.Context,
+	repoPath string,
+	rev string,
+	node *TreeNode,
+	fetchSizes bool,
+) error {
+	nodes := []TreeNode{*node}
+	var pathPrefix string
+
+	// Go in depth for as long as there are subdirectories with just one subdirectory.
+	for len(nodes) == 1 && nodes[0].NodeType == TreeNodeTypeTree {
+		nodesTemp, err := lsTree(ctx, repoPath, rev, nodes[0].Path+"/", fetchSizes)
+		if err != nil {
+			return fmt.Errorf("failed to peek dir entries for flattening: %w", err)
+		}
+
+		// Abort when the subdirectory contains more than one entry or contains an entry which is not a directory.
+		// Git doesn't store empty directories. Every git tree must have at least one entry (except the sha.EmptyTree).
+		if len(nodesTemp) != 1 || (len(nodesTemp) == 1 && nodesTemp[0].NodeType != TreeNodeTypeTree) {
+			nodes[0].Name = path.Join(pathPrefix, nodes[0].Name)
+			*node = nodes[0]
+			break
+		}
+
+		pathPrefix = path.Join(pathPrefix, nodes[0].Name)
+		nodes = nodesTemp
+	}
+
+	return nil
+}
+
 // GetTreeNode returns the tree node at the given path as found for the provided reference.
 func (g *Git) GetTreeNode(ctx context.Context, repoPath, rev, treePath string) (*TreeNode, error) {
 	return GetTreeNode(ctx, repoPath, rev, treePath, false)
@@ -309,13 +359,21 @@ func GetTreeNode(ctx context.Context, repoPath, rev, treePath string, fetchSize
 }
 
 // ListTreeNodes lists the child nodes of a tree reachable from ref via the specified path.
-func (g *Git) ListTreeNodes(ctx context.Context, repoPath, rev, treePath string) ([]TreeNode, error) {
-	return ListTreeNodes(ctx, repoPath, rev, treePath, false)
+func (g *Git) ListTreeNodes(
+	ctx context.Context,
+	repoPath, rev, treePath string,
+	flattenDirectories bool,
+) ([]TreeNode, error) {
+	return ListTreeNodes(ctx, repoPath, rev, treePath, false, flattenDirectories)
 }
 
 // ListTreeNodes lists the child nodes of a tree reachable from ref via the specified path.
-func ListTreeNodes(ctx context.Context, repoPath, rev, treePath string, fetchSizes bool) ([]TreeNode, error) {
-	list, err := lsDirectory(ctx, repoPath, rev, treePath, fetchSizes)
+func ListTreeNodes(
+	ctx context.Context,
+	repoPath, rev, treePath string,
+	fetchSizes, flattenDirectories bool,
+) ([]TreeNode, error) {
+	list, err := lsDirectory(ctx, repoPath, rev, treePath, fetchSizes, flattenDirectories)
 	if err != nil {
 		return nil, fmt.Errorf("failed to list tree nodes: %w", err)
 	}

git/tree.go

@@ -52,9 +52,9 @@ type TreeNode struct {
 type ListTreeNodeParams struct {
 	ReadParams
 	// GitREF is a git reference (branch / tag / commit SHA)
-	GitREF              string
-	Path                string
-	IncludeLatestCommit bool
+	GitREF             string
+	Path               string
+	FlattenDirectories bool
 }
 
 type ListTreeNodeOutput struct {
@@ -126,11 +126,7 @@ func (s *Service) ListTreeNodes(ctx context.Context, params *ListTreeNodeParams)
 
 	repoPath := getFullPathForRepo(s.reposRoot, params.RepoUID)
 
-	res, err := s.git.ListTreeNodes(
-		ctx,
-		repoPath,
-		params.GitREF,
-		params.Path)
+	res, err := s.git.ListTreeNodes(ctx, repoPath, params.GitREF, params.Path, params.FlattenDirectories)
 	if err != nil {
 		return nil, fmt.Errorf("failed to list tree nodes: %w", err)
 	}