add option to import repositories in an existing space (#772)

app/api/controller/space/import.go

@@ -24,13 +24,17 @@ import (
 	"github.com/harness/gitness/types"
 )
 
-type ImportInput struct {
-	CreateInput
+type ProviderInput struct {
 	Provider      importer.Provider       `json:"provider"`
 	ProviderSpace string                  `json:"provider_space"`
 	Pipelines     importer.PipelineOption `json:"pipelines"`
 }
 
+type ImportInput struct {
+	CreateInput
+	ProviderInput
+}
+
 // Import creates new space and starts import of all repositories from the remote provider's space into it.
 func (c *Controller) Import(ctx context.Context, session *auth.Session, in *ImportInput) (*types.Space, error) {
 	parentSpaceID, err := c.getSpaceCheckAuthSpaceCreation(ctx, session, in.ParentRef)

app/api/controller/space/import_repositories.go

@@ -0,0 +1,131 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package space
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	apiauth "github.com/harness/gitness/app/api/auth"
+	"github.com/harness/gitness/app/api/usererror"
+	"github.com/harness/gitness/app/auth"
+	"github.com/harness/gitness/app/services/importer"
+	"github.com/harness/gitness/store"
+	"github.com/harness/gitness/types"
+	"github.com/harness/gitness/types/enum"
+
+	"github.com/rs/zerolog/log"
+)
+
+type ImportRepositoriesInput struct {
+	ProviderInput
+}
+
+type ImportRepositoriesOutput struct {
+	ImportingRepos []*types.Repository `json:"importing_repos"`
+	DuplicateRepos []*types.Repository `json:"duplicate_repos"` // repos which already exist in the space.
+}
+
+// getSpaceCheckAuthRepoCreation checks whether the user has permissions to create repos
+// in the given space.
+func (c *Controller) getSpaceCheckAuthRepoCreation(
+	ctx context.Context,
+	session *auth.Session,
+	spaceRef string,
+) (*types.Space, error) {
+	space, err := c.spaceStore.FindByRef(ctx, spaceRef)
+	if err != nil {
+		return nil, fmt.Errorf("parent space not found: %w", err)
+	}
+
+	// create is a special case - check permission without specific resource
+	scope := &types.Scope{SpacePath: space.Path}
+	resource := &types.Resource{
+		Type: enum.ResourceTypeRepo,
+		Name: "",
+	}
+
+	err = apiauth.Check(ctx, c.authorizer, session, scope, resource, enum.PermissionRepoEdit)
+	if err != nil {
+		return nil, fmt.Errorf("auth check failed: %w", err)
+	}
+
+	return space, nil
+}
+
+// ImportRepositories imports repositories into an existing space. It ignores and continues on
+// repo naming conflicts.
+func (c *Controller) ImportRepositories(
+	ctx context.Context,
+	session *auth.Session,
+	spaceRef string,
+	in *ImportRepositoriesInput,
+) (ImportRepositoriesOutput, error) {
+	space, err := c.getSpaceCheckAuthRepoCreation(ctx, session, spaceRef)
+	if err != nil {
+		return ImportRepositoriesOutput{}, err
+	}
+
+	remoteRepositories, provider, err :=
+		importer.LoadRepositoriesFromProviderSpace(ctx, in.Provider, in.ProviderSpace)
+	if err != nil {
+		return ImportRepositoriesOutput{}, err
+	}
+
+	if len(remoteRepositories) == 0 {
+		return ImportRepositoriesOutput{}, usererror.BadRequestf("found no repositories at %s", in.ProviderSpace)
+	}
+
+	repoIDs := make([]int64, 0, len(remoteRepositories))
+	cloneURLs := make([]string, 0, len(remoteRepositories))
+	repos := make([]*types.Repository, 0, len(remoteRepositories))
+	duplicateRepos := make([]*types.Repository, 0, len(remoteRepositories))
+
+	err = c.tx.WithTx(ctx, func(ctx context.Context) error {
+		for _, remoteRepository := range remoteRepositories {
+			repo := remoteRepository.ToRepo(
+				space.ID, remoteRepository.UID, "", &session.Principal)
+
+			err = c.repoStore.Create(ctx, repo)
+			if errors.Is(err, store.ErrDuplicate) {
+				log.Ctx(ctx).Warn().Err(err).Msg("skipping duplicate repo")
+				duplicateRepos = append(duplicateRepos, repo)
+				continue
+			} else if err != nil {
+				return fmt.Errorf("failed to create repository in storage: %w", err)
+			}
+			repos = append(repos, repo)
+			repoIDs = append(repoIDs, repo.ID)
+			cloneURLs = append(cloneURLs, remoteRepository.CloneURL)
+		}
+		if len(repoIDs) == 0 {
+			return nil
+		}
+
+		jobGroupID := fmt.Sprintf("space-import-%d", space.ID)
+		err = c.importer.RunMany(ctx, jobGroupID, provider, repoIDs, cloneURLs, in.Pipelines)
+		if err != nil {
+			return fmt.Errorf("failed to start import repository jobs: %w", err)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return ImportRepositoriesOutput{}, err
+	}
+
+	return ImportRepositoriesOutput{ImportingRepos: repos, DuplicateRepos: duplicateRepos}, nil
+}

app/api/handler/space/import_repositories.go

@@ -0,0 +1,52 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package space
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/harness/gitness/app/api/controller/space"
+	"github.com/harness/gitness/app/api/render"
+	"github.com/harness/gitness/app/api/request"
+)
+
+func HandleImportRepositories(spaceCtrl *space.Controller) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		session, _ := request.AuthSessionFrom(ctx)
+
+		spaceRef, err := request.GetSpaceRefFromPath(r)
+		if err != nil {
+			render.TranslatedUserError(w, err)
+			return
+		}
+
+		in := new(space.ImportRepositoriesInput)
+		err = json.NewDecoder(r.Body).Decode(in)
+		if err != nil {
+			render.BadRequestf(w, "Invalid Request Body: %s.", err)
+			return
+		}
+
+		repos, err := spaceCtrl.ImportRepositories(ctx, session, spaceRef, in)
+		if err != nil {
+			render.TranslatedUserError(w, err)
+			return
+		}
+
+		render.JSON(w, http.StatusOK, repos)
+	}
+}

app/api/openapi/space.go

@@ -172,6 +172,17 @@ func spaceOperations(reflector *openapi3.Reflector) {
 	_ = reflector.SetJSONResponse(&opImport, new(usererror.Error), http.StatusForbidden)
 	_ = reflector.Spec.AddOperation(http.MethodPost, "/spaces/import", opImport)
 
+	opImportRepositories := openapi3.Operation{}
+	opImportRepositories.WithTags("space")
+	opImportRepositories.WithMapOfAnything(map[string]interface{}{"operationId": "importSpaceRepositories"})
+	_ = reflector.SetRequest(&opImportRepositories, &struct{ space.ImportRepositoriesInput }{}, http.MethodPost)
+	_ = reflector.SetJSONResponse(&opImportRepositories, new(space.ImportRepositoriesOutput), http.StatusOK)
+	_ = reflector.SetJSONResponse(&opImportRepositories, new(usererror.Error), http.StatusBadRequest)
+	_ = reflector.SetJSONResponse(&opImportRepositories, new(usererror.Error), http.StatusInternalServerError)
+	_ = reflector.SetJSONResponse(&opImportRepositories, new(usererror.Error), http.StatusUnauthorized)
+	_ = reflector.SetJSONResponse(&opImportRepositories, new(usererror.Error), http.StatusForbidden)
+	_ = reflector.Spec.AddOperation(http.MethodPost, "/spaces/{space_ref}/import", opImportRepositories)
+
 	opExport := openapi3.Operation{}
 	opExport.WithTags("space")
 	opExport.WithMapOfAnything(map[string]interface{}{"operationId": "exportSpace"})

app/router/api.go

@@ -212,6 +212,7 @@ func setupSpaces(r chi.Router, appCtx context.Context, spaceCtrl *space.Controll
 
 			r.Get("/events", handlerspace.HandleEvents(appCtx, spaceCtrl))
 
+			r.Post("/import", handlerspace.HandleImportRepositories(spaceCtrl))
 			r.Post("/move", handlerspace.HandleMove(spaceCtrl))
 			r.Get("/spaces", handlerspace.HandleListSpaces(spaceCtrl))
 			r.Get("/repos", handlerspace.HandleListRepos(spaceCtrl))