USEFULL/drone
1
0
Fork 0
mirror of https://github.com/harness/drone.git synced 2025-05-01 21:21:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin' into abhinav/CODE-830

Browse Source
This commit is contained in:
Abhinav Singh 2023-09-11 15:59:23 -07:00
commit 742eea6321
189 changed files with 2586 additions and 1666 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
cli/server/config.go
View File

@ -13,6 +13,7 @@ import (
"github.com/harness/gitness/events"
"github.com/harness/gitness/gitrpc"
"github.com/harness/gitness/gitrpc/server"
"github.com/harness/gitness/internal/services/trigger"
"github.com/harness/gitness/internal/services/webhook"
"github.com/harness/gitness/lock"
"github.com/harness/gitness/store/database"
@ -147,23 +148,26 @@ func ProvideEventsConfig() (events.Config, error) {
return config, nil
}
// ProvideWebhookConfig loads the webhook config from the environment.
// It backfills certain config elements if required.
func ProvideWebhookConfig() (webhook.Config, error) {
config := webhook.Config{}
err := envconfig.Process("", &config)
if err != nil {
return webhook.Config{}, fmt.Errorf("failed to load events config: %w", err)
// ProvideWebhookConfig loads the webhook service config from the main config.
func ProvideWebhookConfig(config *types.Config) webhook.Config {
return webhook.Config{
UserAgentIdentity: config.Webhook.UserAgentIdentity,
HeaderIdentity: config.Webhook.HeaderIdentity,
EventReaderName: config.InstanceID,
Concurrency: config.Webhook.Concurrency,
MaxRetries: config.Webhook.MaxRetries,
AllowPrivateNetwork: config.Webhook.AllowPrivateNetwork,
AllowLoopback: config.Webhook.AllowLoopback,
}
}
if config.EventReaderName == "" {
config.EventReaderName, err = getSanitizedMachineName()
if err != nil {
return webhook.Config{}, fmt.Errorf("failed to get sanitized machine name: %w", err)
}
// ProvideTriggerConfig loads the trigger service config from the main config.
func ProvideTriggerConfig(config *types.Config) trigger.Config {
return trigger.Config{
EventReaderName: config.InstanceID,
Concurrency: config.Webhook.Concurrency,
MaxRetries: config.Webhook.MaxRetries,
}
return config, nil
}
// ProvideLockConfig generates the `lock` package config from the gitness config.

7
cmd/gitness/wire.go
View File

@ -33,7 +33,7 @@ import (
"github.com/harness/gitness/internal/api/controller/space"
"github.com/harness/gitness/internal/api/controller/system"
"github.com/harness/gitness/internal/api/controller/template"
"github.com/harness/gitness/internal/api/controller/trigger"
controllertrigger "github.com/harness/gitness/internal/api/controller/trigger"
"github.com/harness/gitness/internal/api/controller/user"
controllerwebhook "github.com/harness/gitness/internal/api/controller/webhook"
"github.com/harness/gitness/internal/auth/authn"
@ -55,6 +55,7 @@ import (
"github.com/harness/gitness/internal/services/importer"
"github.com/harness/gitness/internal/services/job"
pullreqservice "github.com/harness/gitness/internal/services/pullreq"
"github.com/harness/gitness/internal/services/trigger"
"github.com/harness/gitness/internal/services/webhook"
"github.com/harness/gitness/internal/store"
"github.com/harness/gitness/internal/store/cache"
@ -107,6 +108,8 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e
events.WireSet,
cliserver.ProvideWebhookConfig,
webhook.WireSet,
cliserver.ProvideTriggerConfig,
trigger.WireSet,
githook.WireSet,
cliserver.ProvideLockConfig,
lock.WireSet,
@ -130,7 +133,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e
eventsstream.WireSet,
scheduler.WireSet,
commit.WireSet,
trigger.WireSet,
controllertrigger.WireSet,
plugin.WireSet,
importer.WireSet,
exporter.WireSet,

17
cmd/gitness/wire_gen.go
View File

@ -52,6 +52,7 @@ import (
"github.com/harness/gitness/internal/services/importer"
"github.com/harness/gitness/internal/services/job"
pullreq2 "github.com/harness/gitness/internal/services/pullreq"
trigger2 "github.com/harness/gitness/internal/services/trigger"
"github.com/harness/gitness/internal/services/webhook"
"github.com/harness/gitness/internal/store"
"github.com/harness/gitness/internal/store/cache"
@ -119,7 +120,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
if err != nil {
return nil, err
}
repository, err := importer.ProvideRepoImporter(provider, gitrpcInterface, repoStore, jobScheduler, executor)
repository, err := importer.ProvideRepoImporter(config, provider, gitrpcInterface, repoStore, jobScheduler, executor)
if err != nil {
return nil, err
}
@ -142,7 +143,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
secretStore := database.ProvideSecretStore(db)
connectorStore := database.ProvideConnectorStore(db)
templateStore := database.ProvideTemplateStore(db)
spaceController := space.ProvideController(db, provider, eventsStreamer, pathUID, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoController, membershipStore)
spaceController := space.ProvideController(db, provider, eventsStreamer, pathUID, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoController, membershipStore, repository)
pipelineController := pipeline.ProvideController(db, pathUID, pathStore, repoStore, authorizer, pipelineStore)
encrypter, err := encrypt.ProvideEncrypter(config)
if err != nil {
@ -174,10 +175,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
}
migrator := codecomments.ProvideMigrator(gitrpcInterface)
pullreqController := pullreq.ProvideController(db, provider, authorizer, pullReqStore, pullReqActivityStore, codeCommentView, pullReqReviewStore, pullReqReviewerStore, repoStore, principalStore, gitrpcInterface, reporter, mutexManager, migrator)
webhookConfig, err := server.ProvideWebhookConfig()
if err != nil {
return nil, err
}
webhookConfig := server.ProvideWebhookConfig(config)
webhookStore := database.ProvideWebhookStore(db)
webhookExecutionStore := database.ProvideWebhookExecutionStore(db)
readerFactory, err := events4.ProvideReaderFactory(eventsSystem)
@ -236,7 +234,12 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
if err != nil {
return nil, err
}
servicesServices := services.ProvideServices(webhookService, pullreqService, executor, jobScheduler)
triggerConfig := server.ProvideTriggerConfig(config)
triggerService, err := trigger2.ProvideService(ctx, triggerConfig, readerFactory, eventsReaderFactory)
if err != nil {
return nil, err
}
servicesServices := services.ProvideServices(webhookService, pullreqService, triggerService, jobScheduler)
serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, poller, grpcServer, cronManager, servicesServices)
return serverSystem, nil
}

10
events/reader.go
View File

@ -43,9 +43,9 @@ func (f *ReaderFactory[R]) Launch(ctx context.Context,
// setup ctx with copied logger that has extra fields set
log := log.Ctx(ctx).With().
Str("events_category", f.category).
Str("events_group_name", groupName).
Str("events_reader_name", readerName).
Str("events.category", f.category).
Str("events.group_name", groupName).
Str("events.reader_name", readerName).
Logger()
// create new stream consumer using factory method
@ -191,8 +191,8 @@ func ReaderRegisterEvent[T interface{}](reader *GenericReader,
// update ctx with event type for proper logging
log := log.Ctx(ctx).With().
Str("events_type", string(eventType)).
Str("events_id", event.ID).
Str("events.type", string(eventType)).
Str("events.id", event.ID).
Logger()
ctx = log.WithContext(ctx)

5
gitrpc/internal/gitea/repo.go
View File

@ -69,6 +69,7 @@ func (g Adapter) GetDefaultBranch(ctx context.Context, repoPath string) (string,
// If the repo doesn't have a default branch, types.ErrNoDefaultBranch is returned.
func (g Adapter) GetRemoteDefaultBranch(ctx context.Context, remoteURL string) (string, error) {
args := []string{
"-c", "credential.helper=",
"ls-remote",
"--symref",
"-q",
@ -118,6 +119,7 @@ func (g Adapter) Clone(ctx context.Context, from, to string, opts types.CloneRep
func (g Adapter) Sync(ctx context.Context, repoPath string, remoteURL string) error {
args := []string{
"-c", "advice.fetchShowForcedUpdates=false",
"-c", "credential.helper=",
"fetch",
"--quiet",
"--prune",
@ -131,7 +133,8 @@ func (g Adapter) Sync(ctx context.Context, repoPath string, remoteURL string) er
cmd := gitea.NewCommand(ctx, args...)
_, _, err := cmd.RunStdString(&gitea.RunOpts{
Dir: repoPath,
Dir: repoPath,
UseContextTimeout: true,
})
if err != nil {
return processGiteaErrorf(err, "failed to sync repo")

5
gitrpc/internal/service/repo.go
View File

@ -344,6 +344,11 @@ func (s RepositoryService) SyncRepository(
// get remote default branch
defaultBranch, err := s.adapter.GetRemoteDefaultBranch(ctx, request.GetSource())
if errors.Is(err, types.ErrNoDefaultBranch) {
return &rpc.SyncRepositoryResponse{
DefaultBranch: "",
}, nil
}
if err != nil {
return nil, processGitErrorf(err, "failed to get default branch from repo")
}

10
internal/api/controller/repo/blame.go
View File

@ -9,14 +9,14 @@ import (
"strings"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
)
func (c *Controller) Blame(ctx context.Context, session *auth.Session,
func (c *Controller) Blame(ctx context.Context,
session *auth.Session,
repoRef, gitRef, path string,
lineFrom, lineTo int,
) (types.Stream[*gitrpc.BlamePart], error) {
@ -29,15 +29,11 @@ func (c *Controller) Blame(ctx context.Context, session *auth.Session,
return nil, usererror.BadRequest("Line range must be valid.")
}
repo, err := c.repoStore.FindByRef(ctx, repoRef)
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return nil, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, true); err != nil {
return nil, err
}
if gitRef == "" {
gitRef = repo.DefaultBranch
}

14
internal/api/controller/repo/commit.go
View File

@ -11,7 +11,6 @@ import (
"time"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/internal/bootstrap"
"github.com/harness/gitness/types/enum"
@ -40,17 +39,16 @@ type CommitFilesResponse struct {
CommitID string `json:"commit_id"`
}
func (c *Controller) CommitFiles(ctx context.Context, session *auth.Session,
repoRef string, in *CommitFilesOptions) (CommitFilesResponse, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
func (c *Controller) CommitFiles(ctx context.Context,
session *auth.Session,
repoRef string,
in *CommitFilesOptions,
) (CommitFilesResponse, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoPush, false)
if err != nil {
return CommitFilesResponse{}, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoPush, false); err != nil {
return CommitFilesResponse{}, err
}
actions := make([]gitrpc.CommitFileAction, len(in.Actions))
for i, action := range in.Actions {
var rawPayload []byte

7
internal/api/controller/repo/content_get.go
View File

@ -11,7 +11,6 @@ import (
"io"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/controller"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types"
@ -89,15 +88,11 @@ func (c *Controller) GetContent(ctx context.Context,
repoPath string,
includeLatestCommit bool,
) (*GetContentOutput, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return nil, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, true); err != nil {
return nil, err
}
// set gitRef to default branch in case an empty reference was provided
if gitRef == "" {
gitRef = repo.DefaultBranch

13
internal/api/controller/repo/content_paths_details.go
View File

@ -8,7 +8,6 @@ import (
"context"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types/enum"
@ -30,21 +29,19 @@ func (c *Controller) PathsDetails(ctx context.Context,
gitRef string,
input PathsDetailsInput,
) (PathsDetailsOutput, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return PathsDetailsOutput{}, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, true); err != nil {
return PathsDetailsOutput{}, err
}
if len(input.Paths) == 0 {
return PathsDetailsOutput{}, nil
}
if len(input.Paths) > 50 {
return PathsDetailsOutput{}, usererror.BadRequest("maximum number of elements in the Paths array is 25")
const maxInputPaths = 50
if len(input.Paths) > maxInputPaths {
return PathsDetailsOutput{},
usererror.BadRequestf("maximum number of elements in the Paths array is %d", maxInputPaths)
}
// set gitRef to default branch in case an empty reference was provided

32
internal/api/controller/repo/controller.go
View File

@ -12,6 +12,8 @@ import (
"strings"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/internal/auth/authz"
"github.com/harness/gitness/internal/githook"
@ -20,6 +22,7 @@ import (
"github.com/harness/gitness/internal/url"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/check"
"github.com/harness/gitness/types/enum"
"github.com/jmoiron/sqlx"
)
@ -70,6 +73,35 @@ func NewController(
}
}
// getRepoCheckAccess fetches an active repo (not one that is currently being imported)
// and checks if the current user has permission to access it.
func (c *Controller) getRepoCheckAccess(
ctx context.Context,
session *auth.Session,
repoRef string,
reqPermission enum.Permission,
orPublic bool,
) (*types.Repository, error) {
if repoRef == "" {
return nil, usererror.BadRequest("A valid repository reference must be provided.")
}
repo, err := c.repoStore.FindByRef(ctx, repoRef)
if err != nil {
return nil, fmt.Errorf("failed to find repository: %w", err)
}
if repo.Importing {
return nil, usererror.BadRequest("Repository import is in progress.")
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, reqPermission, orPublic); err != nil {
return nil, fmt.Errorf("access check failed: %w", err)
}
return repo, nil
}
// CreateRPCWriteParams creates base write parameters for gitrpc write operations.
// IMPORTANT: session & repo are assumed to be not nil!
func CreateRPCWriteParams(ctx context.Context, urlProvider *url.Provider,

14
internal/api/controller/repo/create_branch.go
View File

@ -9,7 +9,6 @@ import (
"fmt"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types/enum"
)
@ -24,17 +23,16 @@ type CreateBranchInput struct {
}
// CreateBranch creates a new branch for a repo.
func (c *Controller) CreateBranch(ctx context.Context, session *auth.Session,
repoRef string, in *CreateBranchInput) (*Branch, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
func (c *Controller) CreateBranch(ctx context.Context,
session *auth.Session,
repoRef string,
in *CreateBranchInput,
) (*Branch, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoPush, false)
if err != nil {
return nil, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoPush, false); err != nil {
return nil, err
}
// set target to default branch in case no target was provided
if in.Target == "" {
in.Target = repo.DefaultBranch

14
internal/api/controller/repo/create_commit_tag.go
View File

@ -10,7 +10,6 @@ import (
"time"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types/enum"
)
@ -28,17 +27,16 @@ type CreateCommitTagInput struct {
}
// CreateCommitTag creates a new tag for a repo.
func (c *Controller) CreateCommitTag(ctx context.Context, session *auth.Session,
repoRef string, in *CreateCommitTagInput) (*CommitTag, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
func (c *Controller) CreateCommitTag(ctx context.Context,
session *auth.Session,
repoRef string,
in *CreateCommitTagInput,
) (*CommitTag, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoPush, false)
if err != nil {
return nil, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoPush, false); err != nil {
return nil, err
}
// set target to default branch in case no branch or commit was provided
if in.Target == "" {
in.Target = repo.DefaultBranch

14
internal/api/controller/repo/create_path.go
View File

@ -10,7 +10,6 @@ import (
"strings"
"time"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types"
@ -24,17 +23,16 @@ type CreatePathInput struct {
}
// CreatePath creates a new path for a repo.
func (c *Controller) CreatePath(ctx context.Context, session *auth.Session,
repoRef string, in *CreatePathInput) (*types.Path, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
func (c *Controller) CreatePath(ctx context.Context,
session *auth.Session,
repoRef string,
in *CreatePathInput,
) (*types.Path, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoEdit, false)
if err != nil {
return nil, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoEdit, false); err != nil {
return nil, err
}
if err = c.sanitizeCreatePathInput(in, repo.Path); err != nil {
return nil, fmt.Errorf("failed to sanitize input: %w", err)
}

13
internal/api/controller/repo/delete.go
View File

@ -19,6 +19,7 @@ import (
// Delete deletes a repo.
func (c *Controller) Delete(ctx context.Context, session *auth.Session, repoRef string) error {
// note: can't use c.getRepoCheckAccess because import job for repositories being imported must be cancelled.
repo, err := c.repoStore.FindByRef(ctx, repoRef)
if err != nil {
return err
@ -27,9 +28,21 @@ func (c *Controller) Delete(ctx context.Context, session *auth.Session, repoRef
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoDelete, false); err != nil {
return err
}
if repo.Importing {
err = c.importer.Cancel(ctx, repo)
if err != nil {
return fmt.Errorf("failed to cancel repository import")
}
return c.DeleteNoAuth(ctx, session, repo)
}
log.Ctx(ctx).Info().Msgf("Delete request received for repo %s , id: %d", repo.Path, repo.ID)
// TODO: uncomment when soft delete is implemented
// return c.DeleteNoAuth(ctx, session, repo)
return nil
}

13
internal/api/controller/repo/delete_branch.go
View File

@ -9,23 +9,22 @@ import (
"fmt"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types/enum"
)
// DeleteBranch deletes a repo branch.
func (c *Controller) DeleteBranch(ctx context.Context, session *auth.Session, repoRef string, branchName string) error {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
func (c *Controller) DeleteBranch(ctx context.Context,
session *auth.Session,
repoRef string,
branchName string,
) error {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoPush, false)
if err != nil {
return err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoPush, false); err != nil {
return err
}
// make sure user isn't deleting the default branch
// ASSUMPTION: lower layer calls explicit branch api
// and 'refs/heads/branch1' would fail if 'branch1' exists.

13
internal/api/controller/repo/delete_path.go
View File

@ -8,7 +8,6 @@ import (
"context"
"fmt"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/store/database/dbtx"
@ -17,16 +16,16 @@ import (
)
// DeletePath deletes a repo path.
func (c *Controller) DeletePath(ctx context.Context, session *auth.Session, repoRef string, pathID int64) error {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
func (c *Controller) DeletePath(ctx context.Context,
session *auth.Session,
repoRef string,
pathID int64,
) error {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoEdit, false)
if err != nil {
return err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoEdit, false); err != nil {
return err
}
err = dbtx.New(c.db).WithTx(ctx, func(ctx context.Context) error {
var path *types.Path
path, err = c.pathStore.FindWithLock(ctx, pathID)

14
internal/api/controller/repo/delete_tag.go
View File

@ -9,23 +9,21 @@ import (
"fmt"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types/enum"
)
// DeleteTag deletes a tag from the repo.
func (c *Controller) DeleteTag(ctx context.Context, session *auth.Session,
repoRef, tagName string) error {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
func (c *Controller) DeleteTag(ctx context.Context,
session *auth.Session,
repoRef,
tagName string,
) error {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoPush, false)
if err != nil {
return err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoPush, false); err != nil {
return err
}
writeParams, err := CreateRPCWriteParams(ctx, c.urlProvider, session, repo)
if err != nil {
return fmt.Errorf("failed to create RPC write params: %w", err)

6
internal/api/controller/repo/diff.go
View File

@ -25,15 +25,11 @@ func (c *Controller) RawDiff(
path string,
w io.Writer,
) error {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil {
return err
}
info, err := parseDiffPath(path)
if err != nil {
return err

3
internal/api/controller/repo/find.go
View File

@ -15,6 +15,7 @@ import (
// Find finds a repo.
func (c *Controller) Find(ctx context.Context, session *auth.Session, repoRef string) (*types.Repository, error) {
// note: can't use c.getRepoCheckAccess because even repositories that are currently being imported can be fetched.
repo, err := c.repoStore.FindByRef(ctx, repoRef)
if err != nil {
return nil, err
@ -24,7 +25,7 @@ func (c *Controller) Find(ctx context.Context, session *auth.Session, repoRef st
return nil, err
}
// backfil clone url
// backfill clone url
repo.GitURL = c.urlProvider.GenerateRepoCloneURL(repo.Path)
return repo, nil

16
internal/api/controller/repo/get_branch.go
View File

@ -9,21 +9,19 @@ import (
"fmt"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types/enum"
)
// GetBranch gets a repo branch.
func (c *Controller) GetBranch(ctx context.Context, session *auth.Session,
repoRef string, branchName string) (*Branch, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
func (c *Controller) GetBranch(ctx context.Context,
session *auth.Session,
repoRef string,
branchName string,
) (*Branch, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return nil, fmt.Errorf("faild to find repo: %w", err)
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil {
return nil, fmt.Errorf("access check failed: %w", err)
return nil, err
}
rpcOut, err := c.gitRPCClient.GetBranch(ctx, &gitrpc.GetBranchParams{

16
internal/api/controller/repo/get_commit.go
View File

@ -9,7 +9,6 @@ import (
"fmt"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/controller"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types"
@ -17,15 +16,14 @@ import (
)
// GetCommit gets a repo commit.
func (c *Controller) GetCommit(ctx context.Context, session *auth.Session,
repoRef string, sha string) (*types.Commit, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
func (c *Controller) GetCommit(ctx context.Context,
session *auth.Session,
repoRef string,
sha string,
) (*types.Commit, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return nil, fmt.Errorf("faild to find repo: %w", err)
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil {
return nil, fmt.Errorf("access check failed: %w", err)
return nil, err
}
rpcOut, err := c.gitRPCClient.GetCommit(ctx, &gitrpc.GetCommitParams{

18
internal/api/controller/repo/get_commit_divergences.go
View File

@ -8,7 +8,6 @@ import (
"context"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/request"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/internal/auth"
@ -39,20 +38,17 @@ type CommitDivergence struct {
Behind int32 `json:"behind"`
}
/*
* GetCommitDivergences returns the commit divergences between reference pairs.
*/
func (c *Controller) GetCommitDivergences(ctx context.Context, session *auth.Session,
repoRef string, in *GetCommitDivergencesInput) ([]CommitDivergence, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
// GetCommitDivergences returns the commit divergences between reference pairs.
func (c *Controller) GetCommitDivergences(ctx context.Context,
session *auth.Session,
repoRef string,
in *GetCommitDivergencesInput,
) ([]CommitDivergence, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return nil, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil {
return nil, err
}
// if no requests were provided return an empty list
if in == nil || len(in.Requests) == 0 {
return []CommitDivergence{}, nil

121
internal/api/controller/repo/import.go
View File

@ -7,34 +7,23 @@ package repo
import (
"context"
"fmt"
"time"
"github.com/harness/gitness/gitrpc"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/internal/bootstrap"
"github.com/harness/gitness/internal/githook"
"github.com/harness/gitness/internal/paths"
"github.com/harness/gitness/internal/services/importer"
"github.com/harness/gitness/internal/services/job"
"github.com/harness/gitness/store/database/dbtx"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
"github.com/rs/zerolog/log"
)
type ImportInput struct {
ParentRef string `json:"parent_ref"`
UID string `json:"uid"`
Provider importer.ProviderType `json:"provider"`
ProviderURL string `json:"provider_url"`
RepoSlug string `json:"repo_slug"`
Username string `json:"username"`
Password string `json:"password"`
ParentRef string `json:"parent_ref"`
UID string `json:"uid"`
Description string `json:"description"`
Provider importer.Provider `json:"provider"`
ProviderRepo string `json:"provider_repo"`
}
// Import creates a new empty repository and starts git import to it from a remote repository.
@ -49,14 +38,7 @@ func (c *Controller) Import(ctx context.Context, session *auth.Session, in *Impo
return nil, fmt.Errorf("failed to sanitize input: %w", err)
}
providerInfo := importer.ProviderInfo{
Type: in.Provider,
Host: in.ProviderURL,
User: in.Username,
Pass: in.Password,
}
repoInfo, err := importer.Repo(ctx, providerInfo, in.RepoSlug)
remoteRepository, err := importer.LoadRepositoryFromProvider(ctx, in.Provider, in.ProviderRepo)
if err != nil {
return nil, err
}
@ -66,29 +48,7 @@ func (c *Controller) Import(ctx context.Context, session *auth.Session, in *Impo
return nil, fmt.Errorf("error creating job UID: %w", err)
}
gitRPCResp, err := c.createEmptyGitRepository(ctx, session)
if err != nil {
return nil, fmt.Errorf("error creating repository on GitRPC: %w", err)
}
now := time.Now().UnixMilli()
repo := &types.Repository{
Version: 0,
ParentID: parentSpace.ID,
UID: in.UID,
GitUID: gitRPCResp.UID,
Path: "", // the path is set in the DB transaction below
Description: in.Description,
IsPublic: repoInfo.IsPublic,
CreatedBy: session.Principal.ID,
Created: now,
Updated: now,
ForkID: 0,
DefaultBranch: repoInfo.DefaultBranch,
Importing: true,
ImportingJobUID: &jobUID,
}
var repo *types.Repository
err = dbtx.New(c.db).WithTx(ctx, func(ctx context.Context) error {
// lock parent space path to ensure it doesn't get updated while we setup new repo
spacePath, err := c.pathStore.FindPrimaryWithLock(ctx, enum.PathTargetTypeSpace, parentSpace.ID)
@ -96,7 +56,8 @@ func (c *Controller) Import(ctx context.Context, session *auth.Session, in *Impo
return usererror.BadRequest("Parent not found'")
}
repo.Path = paths.Concatinate(spacePath.Value, in.UID)
pathToRepo := paths.Concatinate(spacePath.Value, in.UID)
repo = remoteRepository.ToRepo(parentSpace.ID, pathToRepo, in.UID, in.Description, jobUID, &session.Principal)
err = c.repoStore.Create(ctx, repo)
if err != nil {
@ -119,26 +80,17 @@ func (c *Controller) Import(ctx context.Context, session *auth.Session, in *Impo
return fmt.Errorf("failed to create path: %w", err)
}
err = c.importer.Run(ctx, in.Provider, repo, remoteRepository.CloneURL)
if err != nil {
return fmt.Errorf("failed to start import repository job: %w", err)
}
return nil
})
if err != nil {
if err := c.DeleteRepositoryRPC(ctx, session, repo); err != nil {
log.Ctx(ctx).Warn().Err(err).Msg("gitrpc failed to delete repo for cleanup")
}
return nil, err
}
err = c.importer.Run(ctx, jobUID, importer.Input{
RepoID: repo.ID,
GitUser: in.Username,
GitPass: in.Password,
CloneURL: repoInfo.CloneURL,
})
if err != nil {
log.Ctx(ctx).Err(err).Msg("failed to start import repository job")
}
repo.GitURL = c.urlProvider.GenerateRepoCloneURL(repo.Path)
return repo, nil
@ -153,50 +105,5 @@ func (c *Controller) sanitizeImportInput(in *ImportInput) error {
return err
}
if in.Provider == "" {
return usererror.BadRequest("provider must be provided")
}
if in.RepoSlug == "" {
return usererror.BadRequest("repo slug must be provided")
}
return nil
}
func (c *Controller) createEmptyGitRepository(
ctx context.Context,
session *auth.Session,
) (*gitrpc.CreateRepositoryOutput, error) {
// generate envars (add everything githook CLI needs for execution)
envVars, err := githook.GenerateEnvironmentVariables(
ctx,
c.urlProvider.GetAPIBaseURLInternal(),
0,
session.Principal.ID,
true,
)
if err != nil {
return nil, fmt.Errorf("failed to generate git hook environment variables: %w", err)
}
actor := rpcIdentityFromPrincipal(session.Principal)
committer := rpcIdentityFromPrincipal(bootstrap.NewSystemServiceSession().Principal)
now := time.Now()
resp, err := c.gitRPCClient.CreateRepository(ctx, &gitrpc.CreateRepositoryParams{
Actor: *actor,
EnvVars: envVars,
DefaultBranch: c.defaultBranch,
Files: nil,
Author: actor,
AuthorDate: &now,
Committer: committer,
CommitterDate: &now,
})
if err != nil {
return nil, fmt.Errorf("failed to create repo on gitrpc: %w", err)
}
return resp, nil
}

41
internal/api/controller/repo/import_cancel.go Normal file
View File

@ -0,0 +1,41 @@
// Copyright 2022 Harness Inc. All rights reserved.
// Use of this source code is governed by the Polyform Free Trial License
// that can be found in the LICENSE.md file for this repository.
package repo
import (
"context"
"fmt"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types/enum"
)
// ImportCancel cancels a repository import.
func (c *Controller) ImportCancel(ctx context.Context,
session *auth.Session,
repoRef string,
) error {
// note: can't use c.getRepoCheckAccess because this needs to fetch a repo being imported.
repo, err := c.repoStore.FindByRef(ctx, repoRef)
if err != nil {
return err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoDelete, false); err != nil {
return err
}
if !repo.Importing {
return usererror.BadRequest("repository is not being imported")
}
if err = c.importer.Cancel(ctx, repo); err != nil {
return fmt.Errorf("failed to cancel repository import")
}
return c.DeleteNoAuth(ctx, session, repo)
}

1
internal/api/controller/repo/import_progress.go
View File

@ -18,6 +18,7 @@ func (c *Controller) ImportProgress(ctx context.Context,
session *auth.Session,
repoRef string,
) (types.JobProgress, error) {
// note: can't use c.getRepoCheckAccess because this needs to fetch a repo being imported.
repo, err := c.repoStore.FindByRef(ctx, repoRef)
if err != nil {
return types.JobProgress{}, err

19
internal/api/controller/repo/list_branches.go
View File

@ -9,7 +9,6 @@ import (
"fmt"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/controller"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types"
@ -22,20 +21,18 @@ type Branch struct {
Commit *types.Commit `json:"commit,omitempty"`
}
/*
* ListBranches lists the branches of a repo.
*/
func (c *Controller) ListBranches(ctx context.Context, session *auth.Session,
repoRef string, includeCommit bool, filter *types.BranchFilter) ([]Branch, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
// ListBranches lists the branches of a repo.
func (c *Controller) ListBranches(ctx context.Context,
session *auth.Session,
repoRef string,
includeCommit bool,
filter *types.BranchFilter,
) ([]Branch, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return nil, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil {
return nil, err
}
rpcOut, err := c.gitRPCClient.ListBranches(ctx, &gitrpc.ListBranchesParams{
ReadParams: CreateRPCReadParams(repo),
IncludeCommit: includeCommit,

19
internal/api/controller/repo/list_commit_tags.go
View File

@ -9,7 +9,6 @@ import (
"fmt"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/controller"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types"
@ -26,20 +25,18 @@ type CommitTag struct {
Commit *types.Commit `json:"commit,omitempty"`
}
/*
* ListCommitTags lists the commit tags of a repo.
*/
func (c *Controller) ListCommitTags(ctx context.Context, session *auth.Session,
repoRef string, includeCommit bool, filter *types.TagFilter) ([]CommitTag, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
// ListCommitTags lists the commit tags of a repo.
func (c *Controller) ListCommitTags(ctx context.Context,
session *auth.Session,
repoRef string,
includeCommit bool,
filter *types.TagFilter,
) ([]CommitTag, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return nil, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil {
return nil, err
}
rpcOut, err := c.gitRPCClient.ListCommitTags(ctx, &gitrpc.ListCommitTagsParams{
ReadParams: CreateRPCReadParams(repo),
IncludeCommit: includeCommit,

19
internal/api/controller/repo/list_commits.go
View File

@ -9,27 +9,24 @@ import (
"fmt"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/controller"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
)
/*
* ListCommits lists the commits of a repo.
*/
func (c *Controller) ListCommits(ctx context.Context, session *auth.Session,
repoRef string, gitRef string, filter *types.CommitFilter) (types.ListCommitResponse, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
// ListCommits lists the commits of a repo.
func (c *Controller) ListCommits(ctx context.Context,
session *auth.Session,
repoRef string,
gitRef string,
filter *types.CommitFilter,
) (types.ListCommitResponse, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return types.ListCommitResponse{}, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil {
return types.ListCommitResponse{}, err
}
// set gitRef to default branch in case an empty reference was provided
if gitRef == "" {
gitRef = repo.DefaultBranch

18
internal/api/controller/repo/list_paths.go
View File

@ -8,27 +8,23 @@ import (
"context"
"fmt"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/store/database/dbtx"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
)
/*
* ListPaths lists all paths of a repo.
*/
func (c *Controller) ListPaths(ctx context.Context, session *auth.Session,
repoRef string, filter *types.PathFilter) ([]*types.Path, int64, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
// ListPaths lists all paths of a repo.
func (c *Controller) ListPaths(ctx context.Context,
session *auth.Session,
repoRef string,
filter *types.PathFilter,
) ([]*types.Path, int64, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return nil, 0, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil {
return nil, 0, err
}
var (
paths []*types.Path
count int64

11
internal/api/controller/repo/list_pipelines.go
View File

@ -1,13 +1,13 @@
// Copyright 2022 Harness Inc. All rights reserved.
// Use of this source code is governed by the Polyform Free Trial License
// that can be found in the LICENSE.md file for this repository.
package repo
import (
"context"
"fmt"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/store/database/dbtx"
"github.com/harness/gitness/types"
@ -22,14 +22,9 @@ func (c *Controller) ListPipelines(
latest bool,
filter types.ListQueryFilter,
) ([]*types.Pipeline, int64, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return nil, 0, fmt.Errorf("failed to find repo: %w", err)
}
err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionPipelineView, false)
if err != nil {
return nil, 0, fmt.Errorf("failed to authorize: %w", err)
return nil, 0, err
}
var count int64

18
internal/api/controller/repo/list_service_accounts.go
View File

@ -7,23 +7,19 @@ package repo
import (
"context"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
)
/*
* ListServiceAccounts lists the service accounts of a repo.
*/
func (c *Controller) ListServiceAccounts(ctx context.Context, session *auth.Session,
repoRef string) ([]*types.ServiceAccount, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
if err != nil {
return nil, err
}
// ListServiceAccounts lists the service accounts of a repo.
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil {
func (c *Controller) ListServiceAccounts(ctx context.Context,
session *auth.Session,
repoRef string,
) ([]*types.ServiceAccount, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, false)
if err != nil {
return nil, err
}

7
internal/api/controller/repo/merge_check.go
View File

@ -9,7 +9,6 @@ import (
"fmt"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types/enum"
)
@ -25,15 +24,11 @@ func (c *Controller) MergeCheck(
repoRef string,
diffPath string,
) (MergeCheck, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, false)
if err != nil {
return MergeCheck{}, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil {
return MergeCheck{}, fmt.Errorf("access check failed: %w", err)
}
info, err := parseDiffPath(diffPath)
if err != nil {
return MergeCheck{}, err

12
internal/api/controller/repo/move.go
View File

@ -7,6 +7,7 @@ package repo
import (
"context"
"fmt"
"github.com/harness/gitness/internal/api/usererror"
"strconv"
"strings"
"time"
@ -55,13 +56,20 @@ func (i *MoveInput) hasChanges(repo *types.Repository) bool {
// Move moves a repository to a new space and/or uid.
//
//nolint:gocognit // refactor if needed
func (c *Controller) Move(ctx context.Context, session *auth.Session,
repoRef string, in *MoveInput) (*types.Repository, error) {
func (c *Controller) Move(ctx context.Context,
session *auth.Session,
repoRef string,
in *MoveInput,
) (*types.Repository, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
if err != nil {
return nil, err
}
if repo.Importing {
return nil, usererror.BadRequest("can't move a repo that is being imported")
}
permission := enum.PermissionRepoEdit
var inParentSpaceID *int64
if in.ParentRef != nil {

21
internal/api/controller/repo/raw.go
View File

@ -10,27 +10,24 @@ import (
"io"
"github.com/harness/gitness/gitrpc"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types/enum"
)
/*
* Raw finds the file of the repo at the given path and returns its raw content.
* If no gitRef is provided, the content is retrieved from the default branch.
*/
func (c *Controller) Raw(ctx context.Context, session *auth.Session, repoRef string,
gitRef string, repoPath string) (io.Reader, int64, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
// Raw finds the file of the repo at the given path and returns its raw content.
// If no gitRef is provided, the content is retrieved from the default branch.
func (c *Controller) Raw(ctx context.Context,
session *auth.Session,
repoRef string,
gitRef string,
repoPath string,
) (io.Reader, int64, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
if err != nil {
return nil, 0, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, true); err != nil {
return nil, 0, err
}
// set gitRef to default branch in case an empty reference was provided
if gitRef == "" {
gitRef = repo.DefaultBranch

14
internal/api/controller/repo/update.go
View File

@ -9,7 +9,6 @@ import (
"fmt"
"strings"
apiauth "github.com/harness/gitness/internal/api/auth"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/check"
@ -28,17 +27,16 @@ func (in *UpdateInput) hasChanges(repo *types.Repository) bool {
}
// Update updates a repository.
func (c *Controller) Update(ctx context.Context, session *auth.Session,
repoRef string, in *UpdateInput) (*types.Repository, error) {
repo, err := c.repoStore.FindByRef(ctx, repoRef)
func (c *Controller) Update(ctx context.Context,
session *auth.Session,
repoRef string,
in *UpdateInput,
) (*types.Repository, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoEdit, false)
if err != nil {
return nil, err
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoEdit, false); err != nil {
return nil, err
}
if !in.hasChanges(repo) {
return repo, nil
}

5
internal/api/controller/space/controller.go
View File

@ -8,6 +8,7 @@ import (
"github.com/harness/gitness/internal/api/controller/repo"
"github.com/harness/gitness/internal/auth/authz"
"github.com/harness/gitness/internal/pipeline/events"
"github.com/harness/gitness/internal/services/importer"
"github.com/harness/gitness/internal/store"
"github.com/harness/gitness/internal/url"
"github.com/harness/gitness/types/check"
@ -31,6 +32,7 @@ type Controller struct {
principalStore store.PrincipalStore
repoCtrl *repo.Controller
membershipStore store.MembershipStore
importer *importer.Repository
}
func NewController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.EventsStreamer,
@ -38,7 +40,7 @@ func NewController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.E
pathStore store.PathStore, pipelineStore store.PipelineStore, secretStore store.SecretStore,
connectorStore store.ConnectorStore, templateStore store.TemplateStore, spaceStore store.SpaceStore,
repoStore store.RepoStore, principalStore store.PrincipalStore, repoCtrl *repo.Controller,
membershipStore store.MembershipStore,
membershipStore store.MembershipStore, importer *importer.Repository,
) *Controller {
return &Controller{
db: db,
@ -56,5 +58,6 @@ func NewController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.E
principalStore: principalStore,
repoCtrl: repoCtrl,
membershipStore: membershipStore,
importer: importer,
}
}

183
internal/api/controller/space/import.go Normal file
View File

@ -0,0 +1,183 @@
// Copyright 2022 Harness Inc. All rights reserved.
// Use of this source code is governed by the Polyform Free Trial License
// that can be found in the LICENSE.md file for this repository.
package space
import (
"context"
"fmt"
"strconv"
"strings"
"time"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/internal/auth"
"github.com/harness/gitness/internal/bootstrap"
"github.com/harness/gitness/internal/paths"
"github.com/harness/gitness/internal/services/importer"
"github.com/harness/gitness/internal/services/job"
"github.com/harness/gitness/store/database/dbtx"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/check"
"github.com/harness/gitness/types/enum"
)
type ImportInput struct {
CreateInput
Provider importer.Provider `json:"provider"`
ProviderSpace string `json:"provider_space"`
}
// Import creates new space and starts import of all repositories from the remote provider's space into it.
func (c *Controller) Import(ctx context.Context, session *auth.Session, in *ImportInput) (*types.Space, error) {
parentSpace, err := c.getSpaceCheckAuthSpaceCreation(ctx, session, in.ParentRef)
if err != nil {
return nil, err
}
if in.UID == "" {
in.UID = in.ProviderSpace
}
err = c.sanitizeCreateInput(&in.CreateInput)
if err != nil {
return nil, fmt.Errorf("failed to sanitize input: %w", err)
}
remoteRepositories, err := importer.LoadRepositoriesFromProviderSpace(ctx, in.Provider, in.ProviderSpace)
if err != nil {
return nil, err
}
if len(remoteRepositories) == 0 {
return nil, usererror.BadRequestf("found no repositories at %s", in.ProviderSpace)
}
localRepositories := make([]*types.Repository, len(remoteRepositories))
cloneURLs := make([]string, len(remoteRepositories))
var space *types.Space
err = dbtx.New(c.db).WithTx(ctx, func(ctx context.Context) error {
spacePath := in.UID
parentSpaceID := int64(0)
if parentSpace != nil {
parentSpaceID = parentSpace.ID
// lock parent space path to ensure it doesn't get updated while we setup new space
parentPath, err := c.pathStore.FindPrimaryWithLock(ctx, enum.PathTargetTypeSpace, parentSpaceID)
if err != nil {
return usererror.BadRequest("Parent not found")
}
spacePath = paths.Concatinate(parentPath.Value, in.UID)
// ensure path is within accepted depth!
err = check.PathDepth(spacePath, true)
if err != nil {
return fmt.Errorf("path is invalid: %w", err)
}
}
now := time.Now().UnixMilli()
space = &types.Space{
Version: 0,
ParentID: parentSpaceID,
UID: in.UID,
Path: spacePath,
Description: in.Description,
IsPublic: in.IsPublic,
CreatedBy: session.Principal.ID,
Created: now,
Updated: now,
}
err = c.spaceStore.Create(ctx, space)
if err != nil {
return fmt.Errorf("space creation failed: %w", err)
}
path := &types.Path{
Version: 0,
Value: space.Path,
IsPrimary: true,
TargetType: enum.PathTargetTypeSpace,
TargetID: space.ID,
CreatedBy: space.CreatedBy,
Created: now,
Updated: now,
}
err = c.pathStore.Create(ctx, path)
if err != nil {
return fmt.Errorf("failed to create path: %w", err)
}
// add space membership to top level space only (as the user doesn't have inherited permissions already)
parentRefAsID, err := strconv.ParseInt(in.ParentRef, 10, 64)
if (err == nil && parentRefAsID == 0) || (len(strings.TrimSpace(in.ParentRef)) == 0) {
membership := &types.Membership{
MembershipKey: types.MembershipKey{
SpaceID: space.ID,
PrincipalID: session.Principal.ID,
},
Role: enum.MembershipRoleSpaceOwner,
// membership has been created by the system
CreatedBy: bootstrap.NewSystemServiceSession().Principal.ID,
Created: now,
Updated: now,
}
err = c.membershipStore.Create(ctx, membership)
if err != nil {
return fmt.Errorf("failed to make user owner of the space: %w", err)
}
}
for i, remoteRepository := range remoteRepositories {
var jobUID string
jobUID, err = job.UID()
if err != nil {
return fmt.Errorf("error creating job UID: %w", err)
}
pathToRepo := paths.Concatinate(path.Value, remoteRepository.UID)
repo := remoteRepository.ToRepo(
space.ID, pathToRepo, remoteRepository.UID, "", jobUID, &session.Principal)
err = c.repoStore.Create(ctx, repo)
if err != nil {
return fmt.Errorf("failed to create repository in storage: %w", err)
}
repoPath := &types.Path{
Version: 0,
Value: repo.Path,
IsPrimary: true,
TargetType: enum.PathTargetTypeRepo,
TargetID: repo.ID,
CreatedBy: repo.CreatedBy,
Created: repo.Created,
Updated: repo.Updated,
}
err = c.pathStore.Create(ctx, repoPath)
if err != nil {
return fmt.Errorf("failed to create path: %w", err)
}
localRepositories[i] = repo
cloneURLs[i] = remoteRepository.CloneURL
}
jobGroupID := fmt.Sprintf("space-import-%d", space.ID)
err = c.importer.RunMany(ctx, jobGroupID, in.Provider, localRepositories, cloneURLs)
if err != nil {
return fmt.Errorf("failed to start import repository jobs: %w", err)
}
return nil
})
if err != nil {
return nil, err
}
return space, nil
}

9
internal/api/controller/space/wire.go
View File

@ -8,6 +8,7 @@ import (
"github.com/harness/gitness/internal/api/controller/repo"
"github.com/harness/gitness/internal/auth/authz"
"github.com/harness/gitness/internal/pipeline/events"
"github.com/harness/gitness/internal/services/importer"
"github.com/harness/gitness/internal/store"
"github.com/harness/gitness/internal/url"
"github.com/harness/gitness/types/check"
@ -26,9 +27,11 @@ func ProvideController(db *sqlx.DB, urlProvider *url.Provider, eventsStream even
pipelineStore store.PipelineStore, secretStore store.SecretStore,
connectorStore store.ConnectorStore, templateStore store.TemplateStore,
spaceStore store.SpaceStore, repoStore store.RepoStore, principalStore store.PrincipalStore,
repoCtrl *repo.Controller, membershipStore store.MembershipStore,
repoCtrl *repo.Controller, membershipStore store.MembershipStore, importer *importer.Repository,
) *Controller {
return NewController(db, urlProvider, eventsStream, uidCheck, authorizer,
pathStore, pipelineStore, secretStore, connectorStore, templateStore,
spaceStore, repoStore, principalStore, repoCtrl, membershipStore)
pathStore, pipelineStore, secretStore,
connectorStore, templateStore,
spaceStore, repoStore, principalStore,
repoCtrl, membershipStore, importer)
}

33
internal/api/handler/repo/import_cancel.go Normal file
View File

@ -0,0 +1,33 @@
// Copyright 2022 Harness Inc. All rights reserved.
// Use of this source code is governed by the Polyform Free Trial License
// that can be found in the LICENSE.md file for this repository.
package repo
import (
"net/http"
"github.com/harness/gitness/internal/api/controller/repo"
"github.com/harness/gitness/internal/api/render"
"github.com/harness/gitness/internal/api/request"
)
func HandleImportCancel(repoCtrl *repo.Controller) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
session, _ := request.AuthSessionFrom(ctx)
repoRef, err := request.GetRepoRefFromPath(r)
if err != nil {
render.TranslatedUserError(w, err)
return
}
err = repoCtrl.ImportCancel(ctx, session, repoRef)
if err != nil {
render.TranslatedUserError(w, err)
return
}
render.DeleteSuccessful(w)
}
}

36
internal/api/handler/space/import.go Normal file
View File

@ -0,0 +1,36 @@
// Copyright 2022 Harness Inc. All rights reserved.
// Use of this source code is governed by the Polyform Free Trial License
// that can be found in the LICENSE.md file for this repository.
package space
import (
"encoding/json"
"net/http"
"github.com/harness/gitness/internal/api/controller/space"
"github.com/harness/gitness/internal/api/render"
"github.com/harness/gitness/internal/api/request"
)
func HandleImport(spaceCtrl *space.Controller) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
session, _ := request.AuthSessionFrom(ctx)
in := new(space.ImportInput)
err := json.NewDecoder(r.Body).Decode(in)
if err != nil {
render.BadRequestf(w, "Invalid Request Body: %s.", err)
return
}
space, err := spaceCtrl.Import(ctx, session, in)
if err != nil {
render.TranslatedUserError(w, err)
return
}
render.JSON(w, http.StatusCreated, space)
}
}

12
internal/api/openapi/repo.go
View File

@ -384,6 +384,18 @@ func repoOperations(reflector *openapi3.Reflector) {
_ = reflector.SetJSONResponse(&createRepository, new(usererror.Error), http.StatusForbidden)
_ = reflector.Spec.AddOperation(http.MethodPost, "/repos", createRepository)
importRepository := openapi3.Operation{}
importRepository.WithTags("repository")
importRepository.WithMapOfAnything(map[string]interface{}{"operationId": "importRepository"})
importRepository.WithParameters(queryParameterSpacePath)
_ = reflector.SetRequest(&importRepository, &struct{ repo.ImportInput }{}, http.MethodPost)
_ = reflector.SetJSONResponse(&importRepository, new(types.Repository), http.StatusCreated)
_ = reflector.SetJSONResponse(&importRepository, new(usererror.Error), http.StatusBadRequest)
_ = reflector.SetJSONResponse(&importRepository, new(usererror.Error), http.StatusInternalServerError)
_ = reflector.SetJSONResponse(&importRepository, new(usererror.Error), http.StatusUnauthorized)
_ = reflector.SetJSONResponse(&importRepository, new(usererror.Error), http.StatusForbidden)
_ = reflector.Spec.AddOperation(http.MethodPost, "/repos/import", importRepository)
opFind := openapi3.Operation{}
opFind.WithTags("repository")
opFind.WithMapOfAnything(map[string]interface{}{"operationId": "findRepository"})

11
internal/api/openapi/space.go
View File

@ -158,6 +158,17 @@ func spaceOperations(reflector *openapi3.Reflector) {
_ = reflector.SetJSONResponse(&opCreate, new(usererror.Error), http.StatusForbidden)
_ = reflector.Spec.AddOperation(http.MethodPost, "/spaces", opCreate)
opImport := openapi3.Operation{}
opImport.WithTags("space")
opImport.WithMapOfAnything(map[string]interface{}{"operationId": "importSpace"})
_ = reflector.SetRequest(&opImport, &struct{ space.ImportInput }{}, http.MethodPost)
_ = reflector.SetJSONResponse(&opImport, new(types.Space), http.StatusCreated)
_ = reflector.SetJSONResponse(&opImport, new(usererror.Error), http.StatusBadRequest)
_ = reflector.SetJSONResponse(&opImport, new(usererror.Error), http.StatusInternalServerError)
_ = reflector.SetJSONResponse(&opImport, new(usererror.Error), http.StatusUnauthorized)
_ = reflector.SetJSONResponse(&opImport, new(usererror.Error), http.StatusForbidden)
_ = reflector.Spec.AddOperation(http.MethodPost, "/spaces/import", opImport)
opGet := openapi3.Operation{}
opGet.WithTags("space")
opGet.WithMapOfAnything(map[string]interface{}{"operationId": "getSpace"})

1
internal/router/api.go
View File

@ -181,6 +181,7 @@ func setupSpaces(r chi.Router, spaceCtrl *space.Controller) {
r.Route("/spaces", func(r chi.Router) {
// Create takes path and parentId via body, not uri
r.Post("/", handlerspace.HandleCreate(spaceCtrl))
r.Post("/import", handlerspace.HandleImport(spaceCtrl))
r.Route(fmt.Sprintf("/{%s}", request.PathParamSpaceRef), func(r chi.Router) {
// space operations

155
internal/services/importer/provider.go
View File

@ -9,25 +9,40 @@ import (
"errors"
"fmt"
"net/http"
"time"
"github.com/harness/gitness/internal/api/usererror"
"github.com/harness/gitness/types"
"github.com/drone/go-scm/scm"
"github.com/drone/go-scm/scm/driver/github"
"github.com/drone/go-scm/scm/driver/gitlab"
"github.com/drone/go-scm/scm/transport/oauth2"
)
type ProviderType string
const (
ProviderTypeGitHub ProviderType = "github"
ProviderTypeGitHub ProviderType = "github"
ProviderTypeGitHubEnterprise ProviderType = "github-enterprise"
ProviderTypeGitLab ProviderType = "gitlab"
ProviderTypeGitLabEnterprise ProviderType = "gitlab-enterprise"
)
type ProviderInfo struct {
Type ProviderType
Host string
User string
Pass string
func (p ProviderType) Enum() []any {
return []any{
ProviderTypeGitHub,
ProviderTypeGitHubEnterprise,
ProviderTypeGitLab,
ProviderTypeGitLabEnterprise,
}
}
type Provider struct {
Type ProviderType `json:"type"`
Host string `json:"host"`
Username string `json:"username"`
Password string `json:"password"`
}
type RepositoryInfo struct {
@ -38,42 +53,120 @@ type RepositoryInfo struct {
DefaultBranch string
}
func getClient(provider ProviderInfo) (*scm.Client, error) {
var scmClient *scm.Client
// ToRepo converts the RepositoryInfo into the types.Repository object marked as being imported.
func (r *RepositoryInfo) ToRepo(
spaceID int64,
path string,
uid string,
description string,
jobUID string,
principal *types.Principal,
) *types.Repository {
now := time.Now().UnixMilli()
gitTempUID := "importing-" + jobUID
return &types.Repository{
Version: 0,
ParentID: spaceID,
UID: uid,
GitUID: gitTempUID, // the correct git UID will be set by the job handler
Path: path,
Description: description,
IsPublic: r.IsPublic,
CreatedBy: principal.ID,
Created: now,
Updated: now,
ForkID: 0,
DefaultBranch: r.DefaultBranch,
Importing: true,
ImportingJobUID: &jobUID,
}
}
func getClient(provider Provider) (*scm.Client, error) {
switch provider.Type {
case "":
return nil, usererror.BadRequest("provider can not be empty")
case ProviderTypeGitHub:
scmClient = github.NewDefault()
if provider.Pass != "" {
scmClient.Client = &http.Client{
c := github.NewDefault()
if provider.Password != "" {
c.Client = &http.Client{
Transport: &oauth2.Transport{
Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Pass}),
Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
},
}
}
return c, nil
case ProviderTypeGitHubEnterprise:
c, err := github.New(provider.Host)
if err != nil {
return nil, usererror.BadRequestf("provider Host invalid: %s", err.Error())
}
if provider.Password != "" {
c.Client = &http.Client{
Transport: &oauth2.Transport{
Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
},
}
}
return c, nil
case ProviderTypeGitLab:
c := gitlab.NewDefault()
if provider.Password != "" {
c.Client = &http.Client{
Transport: &oauth2.Transport{
Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
},
}
}
return c, nil
case ProviderTypeGitLabEnterprise:
c, err := gitlab.New(provider.Host)
if err != nil {
return nil, usererror.BadRequestf("provider Host invalid: %s", err.Error())
}
if provider.Password != "" {
c.Client = &http.Client{
Transport: &oauth2.Transport{
Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
},
}
}
return c, nil
default:
return nil, usererror.BadRequestf("unsupported provider: %s", provider)
}
return scmClient, nil
}
func Repo(ctx context.Context, provider ProviderInfo, repoSlug string) (RepositoryInfo, error) {
func LoadRepositoryFromProvider(ctx context.Context, provider Provider, repoSlug string) (RepositoryInfo, error) {
scmClient, err := getClient(provider)
if err != nil {
return RepositoryInfo{}, err
}
if repoSlug == "" {
return RepositoryInfo{}, usererror.BadRequest("provider repository identifier is missing")
}
scmRepo, _, err := scmClient.Repositories.Find(ctx, repoSlug)
if errors.Is(err, scm.ErrNotFound) {
return RepositoryInfo{}, usererror.BadRequestf("repository %s not found at %s", repoSlug, provider)
return RepositoryInfo{},
usererror.BadRequestf("repository %s not found at %s", repoSlug, provider.Type)
}
if errors.Is(err, scm.ErrNotAuthorized) {
return RepositoryInfo{}, usererror.BadRequestf("bad credentials provided for %s at %s", repoSlug, provider)
return RepositoryInfo{},
usererror.BadRequestf("bad credentials provided for %s at %s", repoSlug, provider.Type)
}
if err != nil {
return RepositoryInfo{}, fmt.Errorf("failed to fetch repository %s from %s: %w", repoSlug, provider, err)
return RepositoryInfo{},
fmt.Errorf("failed to fetch repository %s from %s: %w", repoSlug, provider.Type, err)
}
return RepositoryInfo{
@ -85,13 +178,17 @@ func Repo(ctx context.Context, provider ProviderInfo, repoSlug string) (Reposito
}, nil
}
func Space(ctx context.Context, provider ProviderInfo, space string) (map[string]RepositoryInfo, error) {
func LoadRepositoriesFromProviderSpace(ctx context.Context, provider Provider, spaceSlug string) ([]RepositoryInfo, error) {
scmClient, err := getClient(provider)
if err != nil {
return nil, err
}
repoMap := make(map[string]RepositoryInfo)
if spaceSlug == "" {
return nil, usererror.BadRequest("provider space identifier is missing")
}
repos := make([]RepositoryInfo, 0)
const pageSize = 50
page := 1
@ -104,31 +201,27 @@ func Space(ctx context.Context, provider ProviderInfo, space string) (map[string
},
RepoSearchTerm: scm.RepoSearchTerm{
RepoName: "",
User: space,
User: spaceSlug,
},
})
if errors.Is(err, scm.ErrNotFound) {
return nil, usererror.BadRequestf("space %s not found at %s", space, provider)
return nil, usererror.BadRequestf("space %s not found at %s", spaceSlug, provider.Type)
}
if errors.Is(err, scm.ErrNotAuthorized) {
return nil, usererror.BadRequestf("bad credentials provided for %s at %s", space, provider)
return nil, usererror.BadRequestf("bad credentials provided for %s at %s", spaceSlug, provider.Type)
}
if err != nil {
return nil, fmt.Errorf("failed to fetch space %s from %s: %w", space, provider, err)
return nil, fmt.Errorf("failed to fetch space %s from %s: %w", spaceSlug, provider.Type, err)
}
for _, scmRepo := range scmRepos {
if !scmRepo.Perm.Pull {
continue
}
repoMap[scmRepo.Name] = RepositoryInfo{
repos = append(repos, RepositoryInfo{
Space: scmRepo.Namespace,
UID: scmRepo.Name,
CloneURL: scmRepo.Clone,
IsPublic: !scmRepo.Private,
DefaultBranch: scmRepo.Branch,
}
})
}
if len(scmRepos) == 0 || page == scmResponse.Page.Last {
@ -138,5 +231,5 @@ func Space(ctx context.Context, provider ProviderInfo, space string) (map[string
page++
}
return repoMap, nil
return repos, nil
}

288
internal/services/importer/repository.go
View File

@ -21,13 +21,21 @@ import (
gitnessurl "github.com/harness/gitness/internal/url"
gitness_store "github.com/harness/gitness/store"
"github.com/harness/gitness/types"
"github.com/rs/zerolog/log"
)
const (
importJobMaxRetries = 0
importJobMaxDuration = 45 * time.Minute
)
type Repository struct {
urlProvider *gitnessurl.Provider
git gitrpc.Interface
repoStore store.RepoStore
scheduler *job.Scheduler
defaultBranch string
urlProvider *gitnessurl.Provider
git gitrpc.Interface
repoStore store.RepoStore
scheduler *job.Scheduler
}
var _ job.Handler = (*Repository)(nil)
@ -41,29 +49,90 @@ type Input struct {
const jobType = "repository_import"
func (i *Repository) Register(executor *job.Executor) error {
return executor.Register(jobType, i)
func (r *Repository) Register(executor *job.Executor) error {
return executor.Register(jobType, r)
}
func (i *Repository) Run(ctx context.Context, jobUID string, input Input) error {
// Run starts a background job that imports the provided repository from the provided clone URL.
func (r *Repository) Run(ctx context.Context, provider Provider, repo *types.Repository, cloneURL string) error {
input := Input{
RepoID: repo.ID,
GitUser: provider.Username,
GitPass: provider.Password,
CloneURL: cloneURL,
}
data, err := json.Marshal(input)
if err != nil {
return err
return fmt.Errorf("failed to marshal job input json: %w", err)
}
strData := strings.TrimSpace(string(data))
return i.scheduler.RunJob(ctx, job.Definition{
UID: jobUID,
return r.scheduler.RunJob(ctx, job.Definition{
UID: *repo.ImportingJobUID,
Type: jobType,
MaxRetries: 1,
Timeout: 30 * time.Minute,
MaxRetries: importJobMaxRetries,
Timeout: importJobMaxDuration,
Data: strData,
})
}
// RunMany starts background jobs that import the provided repositories from the provided clone URLs.
func (r *Repository) RunMany(ctx context.Context,
groupID string,
provider Provider,
repos []*types.Repository,
cloneURLs []string,
) error {
if len(repos) != len(cloneURLs) {
return fmt.Errorf("slice length mismatch: have %d repositories and %d clone URLs",
len(repos), len(cloneURLs))
}
n := len(repos)
defs := make([]job.Definition, n)
for k := 0; k < n; k++ {
repo := repos[k]
cloneURL := cloneURLs[k]
input := Input{
RepoID: repo.ID,
GitUser: provider.Username,
GitPass: provider.Password,
CloneURL: cloneURL,
}
data, err := json.Marshal(input)
if err != nil {
return fmt.Errorf("failed to marshal job input json: %w", err)
}
strData := strings.TrimSpace(string(data))
defs[k] = job.Definition{
UID: *repo.ImportingJobUID,
Type: jobType,
MaxRetries: importJobMaxRetries,
Timeout: importJobMaxDuration,
Data: strData,
}
}
err := r.scheduler.RunJobs(ctx, groupID, defs)
if err != nil {
return fmt.Errorf("failed to run jobs: %w", err)
}
return nil
}
// Handle is repository import background job handler.
func (i *Repository) Handle(ctx context.Context, data string, _ job.ProgressReporter) (string, error) {
func (r *Repository) Handle(ctx context.Context, data string, _ job.ProgressReporter) (string, error) {
systemPrincipal := bootstrap.NewSystemServiceSession().Principal
var input Input
if err := json.NewDecoder(strings.NewReader(data)).Decode(&input); err != nil {
return "", fmt.Errorf("failed to unmarshal job input: %w", err)
@ -83,7 +152,7 @@ func (i *Repository) Handle(ctx context.Context, data string, _ job.ProgressRepo
input.CloneURL = repoURL.String()
}
repo, err := i.repoStore.Find(ctx, input.RepoID)
repo, err := r.repoStore.Find(ctx, input.RepoID)
if err != nil {
return "", fmt.Errorf("failed to find repo by id: %w", err)
}
@ -92,38 +161,60 @@ func (i *Repository) Handle(ctx context.Context, data string, _ job.ProgressRepo
return "", fmt.Errorf("repository %s is not being imported", repo.UID)
}
writeParams, err := createRPCWriteParams(ctx, i.urlProvider, repo)
gitUID, err := r.createGitRepository(ctx, &systemPrincipal, repo.ID)
if err != nil {
return "", fmt.Errorf("failed to create write params: %w", err)
return "", fmt.Errorf("failed to create empty git repository: %w", err)
}
syncOut, err := i.git.SyncRepository(ctx, &gitrpc.SyncRepositoryParams{
WriteParams: writeParams,
Source: input.CloneURL,
CreateIfNotExists: false,
})
err = func() error {
repo.GitUID = gitUID
defaultBranch, err := r.syncGitRepository(ctx, &systemPrincipal, repo, input.CloneURL)
if err != nil {
return fmt.Errorf("failed to sync git repository from '%s': %w", input.CloneURL, err)
}
if defaultBranch == "" {
defaultBranch = r.defaultBranch
}
repo, err = r.repoStore.UpdateOptLock(ctx, repo, func(repo *types.Repository) error {
if !repo.Importing {
return errors.New("repository has already finished importing")
}
repo.GitUID = gitUID
repo.DefaultBranch = defaultBranch
repo.Importing = false
return nil
})
if err != nil {
return fmt.Errorf("failed to update repository after import: %w", err)
}
return nil
}()
if err != nil {
return "", fmt.Errorf("failed to sync repositories: %w", err)
if errDel := r.deleteGitRepository(ctx, &systemPrincipal, repo); errDel != nil {
log.Ctx(ctx).Err(err).
Str("gitUID", gitUID).
Msg("failed to delete git repository after failed import")
}
return "", fmt.Errorf("failed to import repository: %w", err)
}
repo.Importing = false
repo.DefaultBranch = syncOut.DefaultBranch
err = i.repoStore.Update(ctx, repo)
if err != nil {
return "", fmt.Errorf("failed to update repository after import: %w", err)
}
return "", err
return "", nil
}
func (i *Repository) GetProgress(ctx context.Context, repo *types.Repository) (types.JobProgress, error) {
func (r *Repository) GetProgress(ctx context.Context, repo *types.Repository) (types.JobProgress, error) {
if !repo.Importing || repo.ImportingJobUID == nil || *repo.ImportingJobUID == "" {
// if the repo is not being imported, or it's job ID has been cleared (or never existed) return state=finished
return job.DoneProgress(), nil
}
progress, err := i.scheduler.GetJobProgress(ctx, *repo.ImportingJobUID)
progress, err := r.scheduler.GetJobProgress(ctx, *repo.ImportingJobUID)
if errors.Is(err, gitness_store.ErrResourceNotFound) {
// if the job is not found return state=failed
return job.FailProgress(), nil
@ -135,31 +226,130 @@ func (i *Repository) GetProgress(ctx context.Context, repo *types.Repository) (t
return progress, nil
}
// CreateRPCWriteParams creates base write parameters for gitrpc write operations.
func createRPCWriteParams(ctx context.Context,
urlProvider *gitnessurl.Provider,
func (r *Repository) Cancel(ctx context.Context, repo *types.Repository) error {
if repo.ImportingJobUID == nil || *repo.ImportingJobUID == "" {
return nil
}
err := r.scheduler.CancelJob(ctx, *repo.ImportingJobUID)
if err != nil {
return fmt.Errorf("failed to cancel job: %w", err)
}
return nil
}
func (r *Repository) createGitRepository(ctx context.Context,
principal *types.Principal,
repoID int64,
) (string, error) {
now := time.Now()
envVars, err := r.createEnvVars(ctx, principal, repoID)
if err != nil {
return "", err
}
resp, err := r.git.CreateRepository(ctx, &gitrpc.CreateRepositoryParams{
Actor: gitrpc.Identity{
Name: principal.DisplayName,
Email: principal.Email,
},
EnvVars: envVars,
DefaultBranch: r.defaultBranch,
Files: nil,
Author: &gitrpc.Identity{
Name: principal.DisplayName,
Email: principal.Email,
},
AuthorDate: &now,
Committer: &gitrpc.Identity{
Name: principal.DisplayName,
Email: principal.Email,
},
CommitterDate: &now,
})
if err != nil {
return "", fmt.Errorf("failed to create empty git repository: %w", err)
}
return resp.UID, nil
}
func (r *Repository) syncGitRepository(ctx context.Context,
principal *types.Principal,
repo *types.Repository,
sourceCloneURL string,
) (string, error) {
writeParams, err := r.createRPCWriteParams(ctx, principal, repo)
if err != nil {
return "", err
}
syncOut, err := r.git.SyncRepository(ctx, &gitrpc.SyncRepositoryParams{
WriteParams: writeParams,
Source: sourceCloneURL,
CreateIfNotExists: false,
})
if err != nil {
return "", fmt.Errorf("failed to sync repository: %w", err)
}
return syncOut.DefaultBranch, nil
}
func (r *Repository) deleteGitRepository(ctx context.Context,
principal *types.Principal,
repo *types.Repository,
) error {
writeParams, err := r.createRPCWriteParams(ctx, principal, repo)
if err != nil {
return err
}
err = r.git.DeleteRepository(ctx, &gitrpc.DeleteRepositoryParams{
WriteParams: writeParams,
})
if err != nil {
return fmt.Errorf("failed to delete git repository: %w", err)
}
return nil
}
func (r *Repository) createRPCWriteParams(ctx context.Context,
principal *types.Principal,
repo *types.Repository,
) (gitrpc.WriteParams, error) {
gitnessSession := bootstrap.NewSystemServiceSession()
// generate envars (add everything githook CLI needs for execution)
envVars, err := githook.GenerateEnvironmentVariables(
ctx,
urlProvider.GetAPIBaseURLInternal(),
repo.ID,
gitnessSession.Principal.ID,
false,
)
envVars, err := r.createEnvVars(ctx, principal, repo.ID)
if err != nil {
return gitrpc.WriteParams{}, fmt.Errorf("failed to generate git hook environment variables: %w", err)
return gitrpc.WriteParams{}, err
}
return gitrpc.WriteParams{
Actor: gitrpc.Identity{
Name: gitnessSession.Principal.DisplayName,
Email: gitnessSession.Principal.Email,
Name: principal.DisplayName,
Email: principal.Email,
},
RepoUID: repo.GitUID,
EnvVars: envVars,
}, nil
}
func (r *Repository) createEnvVars(ctx context.Context,
principal *types.Principal,
repoID int64,
) (map[string]string, error) {
envVars, err := githook.GenerateEnvironmentVariables(
ctx,
r.urlProvider.GetAPIBaseURLInternal(),
repoID,
principal.ID,
false,
)
if err != nil {
return nil, fmt.Errorf("failed to generate git hook environment variables: %w", err)
}
return envVars, nil
}

11
internal/services/importer/wire.go
View File

@ -9,6 +9,7 @@ import (
"github.com/harness/gitness/internal/services/job"
"github.com/harness/gitness/internal/store"
"github.com/harness/gitness/internal/url"
"github.com/harness/gitness/types"
"github.com/google/wire"
)
@ -18,6 +19,7 @@ var WireSet = wire.NewSet(
)
func ProvideRepoImporter(
config *types.Config,
urlProvider *url.Provider,
git gitrpc.Interface,
repoStore store.RepoStore,
@ -25,10 +27,11 @@ func ProvideRepoImporter(
executor *job.Executor,
) (*Repository, error) {
importer := &Repository{
urlProvider: urlProvider,
git: git,
repoStore: repoStore,
scheduler: scheduler,
defaultBranch: config.Git.DefaultBranch,
urlProvider: urlProvider,
git: git,
repoStore: repoStore,
scheduler: scheduler,
}
err := executor.Register(jobType, importer)

72
internal/services/job/scheduler.go
View File

@ -36,7 +36,6 @@ type Scheduler struct {
purgeMinOldAge time.Duration
// synchronization stuff
globalCtx context.Context
signal chan time.Time
done chan struct{}
wgRunning sync.WaitGroup
@ -102,7 +101,6 @@ func (s *Scheduler) Run(ctx context.Context) error {
defer close(s.done)
s.signal = make(chan time.Time, 1)
s.globalCtx = ctx
timer := newSchedulerTimer()
defer timer.Stop()
@ -271,75 +269,41 @@ func (s *Scheduler) RunJob(ctx context.Context, def Definition) error {
return err
}
return s.startNewJobs(ctx, []*types.Job{def.toNewJob()})
job := def.toNewJob()
if err := s.store.Create(ctx, job); err != nil {
return fmt.Errorf("failed to add new job to the database: %w", err)
}
s.scheduleProcessing(time.UnixMilli(job.Scheduled))
return nil
}
// RunJobs runs a several jobs. It's more efficient than calling RunJob several times
// because it locks the DB only once.
// TODO: Add groupID parameter and use it for all jobs.
func (s *Scheduler) RunJobs(ctx context.Context, defs []Definition) error {
func (s *Scheduler) RunJobs(ctx context.Context, groupID string, defs []Definition) error {
if len(defs) == 0 {
return nil
}
jobs := make([]*types.Job, len(defs))
for i, def := range defs {
if err := def.Validate(); err != nil {
return err
}
jobs[i] = def.toNewJob()
}
return s.startNewJobs(ctx, jobs)
}
func (s *Scheduler) startNewJobs(ctx context.Context, jobs []*types.Job) error {
mx, err := globalLock(ctx, s.mxManager)
if err != nil {
return fmt.Errorf("failed to obtain global lock to start new jobs: %w", err)
}
defer func() {
if err := mx.Unlock(ctx); err != nil {
log.Ctx(ctx).Err(err).Msg("failed to release global lock after starting new jobs")
}
}()
return s.startNewJobsNoLock(ctx, jobs)
}
func (s *Scheduler) startNewJobsNoLock(ctx context.Context, jobs []*types.Job) error {
available, err := s.availableSlots(ctx)
if err != nil {
return fmt.Errorf("failed to count available slots for job execution: %w", err)
jobs[i].GroupID = groupID
}
for _, job := range jobs {
if available > 0 {
available--
s.preExec(job) // Update the job fields for the new execution: It will be added to the DB as "running".
}
err = s.store.Create(ctx, job)
if err != nil {
if err := s.store.Create(ctx, job); err != nil {
return fmt.Errorf("failed to add new job to the database: %w", err)
}
if job.State != enum.JobStateRunning {
continue
}
func(ctx context.Context) {
ctx = log.Ctx(ctx).With().
Str("job.UID", job.UID).
Str("job.Type", job.Type).
Logger().WithContext(ctx)
// tell everybody that a job has started
if err := publishStateChange(ctx, s.pubsubService, job); err != nil {
log.Err(err).Msg("failed to publish job state change")
}
s.runJob(ctx, job)
}(s.globalCtx)
}
s.scheduleProcessing(time.Now())
return nil
}

0
internal/services/pullreq/pullreq.go → internal/services/pullreq/service.go
View File

22
internal/services/trigger/handler_branch.go Normal file
View File

@ -0,0 +1,22 @@
// Copyright 2022 Harness Inc. All rights reserved.
// Use of this source code is governed by the Polyform Free Trial License
// that can be found in the LICENSE.md file for this repository.
package trigger
import (
"context"
"github.com/harness/gitness/events"
gitevents "github.com/harness/gitness/internal/events/git"
)
func (s *Service) handleEventBranchCreated(ctx context.Context,
event *events.Event[*gitevents.BranchCreatedPayload]) error {
return events.NewDiscardEventErrorf("not implemented")
}
func (s *Service) handleEventBranchUpdated(ctx context.Context,
event *events.Event[*gitevents.BranchUpdatedPayload]) error {
return events.NewDiscardEventErrorf("not implemented")
}

27
internal/services/trigger/handler_pullreq.go Normal file
View File

@ -0,0 +1,27 @@
// Copyright 2022 Harness Inc. All rights reserved.
// Use of this source code is governed by the Polyform Free Trial License
// that can be found in the LICENSE.md file for this repository.
package trigger
import (
"context"
"github.com/harness/gitness/events"
pullreqevents "github.com/harness/gitness/internal/events/pullreq"
)
func (s *Service) handleEventPullReqCreated(ctx context.Context,
event *events.Event[*pullreqevents.CreatedPayload]) error {
return events.NewDiscardEventErrorf("not implemented")
}
func (s *Service) handleEventPullReqReopened(ctx context.Context,
event *events.Event[*pullreqevents.ReopenedPayload]) error {
return events.NewDiscardEventErrorf("not implemented")
}
func (s *Service) handleEventPullReqBranchUpdated(ctx context.Context,
event *events.Event[*pullreqevents.BranchUpdatedPayload]) error {
return events.NewDiscardEventErrorf("not implemented")
}

22
internal/services/trigger/handler_tag.go Normal file
View File

@ -0,0 +1,22 @@
// Copyright 2022 Harness Inc. All rights reserved.
// Use of this source code is governed by the Polyform Free Trial License
// that can be found in the LICENSE.md file for this repository.
package trigger
import (
"context"
"github.com/harness/gitness/events"
gitevents "github.com/harness/gitness/internal/events/git"
)
func (s *Service) handleEventTagCreated(ctx context.Context,
event *events.Event[*gitevents.TagCreatedPayload]) error {
return events.NewDiscardEventErrorf("not implemented")
}
func (s *Service) handleEventTagUpdated(ctx context.Context,
event *events.Event[*gitevents.TagUpdatedPayload]) error {
return events.NewDiscardEventErrorf("not implemented")
}

103
internal/services/trigger/service.go Normal file
View File

@ -0,0 +1,103 @@
// Copyright 2022 Harness Inc. All rights reserved.
// Use of this source code is governed by the Polyform Free Trial License
// that can be found in the LICENSE.md file for this repository.
package trigger
import (
"context"
"errors"
"fmt"
"time"
"github.com/harness/gitness/events"
gitevents "github.com/harness/gitness/internal/events/git"
pullreqevents "github.com/harness/gitness/internal/events/pullreq"
"github.com/harness/gitness/stream"
)
const (
eventsReaderGroupName = "gitness:trigger"
)
type Config struct {
EventReaderName string
Concurrency int
MaxRetries int
}
func (c *Config) Prepare() error {
if c == nil {
return errors.New("config is required")
}
if c.EventReaderName == "" {
return errors.New("config.EventReaderName is required")
}
if c.Concurrency < 1 {
return errors.New("config.Concurrency has to be a positive number")
}
if c.MaxRetries < 0 {
return errors.New("config.MaxRetries can't be negative")
}
return nil
}
type Service struct{}
func New(
ctx context.Context,
config Config,
gitReaderFactory *events.ReaderFactory[*gitevents.Reader],
pullreqEvReaderFactory *events.ReaderFactory[*pullreqevents.Reader],
) (*Service, error) {
if err := config.Prepare(); err != nil {
return nil, fmt.Errorf("provided trigger service config is invalid: %w", err)
}
service := &Service{}
_, err := gitReaderFactory.Launch(ctx, eventsReaderGroupName, config.EventReaderName,
func(r *gitevents.Reader) error {
const idleTimeout = 1 * time.Minute
r.Configure(
stream.WithConcurrency(config.Concurrency),
stream.WithHandlerOptions(
stream.WithIdleTimeout(idleTimeout),
stream.WithMaxRetries(config.MaxRetries),
))
_ = r.RegisterBranchCreated(service.handleEventBranchCreated)
_ = r.RegisterBranchUpdated(service.handleEventBranchUpdated)
_ = r.RegisterTagCreated(service.handleEventTagCreated)
_ = r.RegisterTagUpdated(service.handleEventTagUpdated)
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to launch git events reader: %w", err)
}
_, err = pullreqEvReaderFactory.Launch(ctx, eventsReaderGroupName, config.EventReaderName,
func(r *pullreqevents.Reader) error {
const idleTimeout = 1 * time.Minute
r.Configure(
stream.WithConcurrency(config.Concurrency),
stream.WithHandlerOptions(
stream.WithIdleTimeout(idleTimeout),
stream.WithMaxRetries(config.MaxRetries),
))
_ = r.RegisterCreated(service.handleEventPullReqCreated)
_ = r.RegisterBranchUpdated(service.handleEventPullReqBranchUpdated)
_ = r.RegisterReopened(service.handleEventPullReqReopened)
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to launch pr events reader: %w", err)
}
return service, nil
}

28
internal/services/trigger/wire.go Normal file
View File

@ -0,0 +1,28 @@
// Copyright 2022 Harness Inc. All rights reserved.
// Use of this source code is governed by the Polyform Free Trial License
// that can be found in the LICENSE.md file for this repository.
package trigger
import (
"context"
"github.com/harness/gitness/events"
gitevents "github.com/harness/gitness/internal/events/git"
pullreqevents "github.com/harness/gitness/internal/events/pullreq"
"github.com/google/wire"
)
var WireSet = wire.NewSet(
ProvideService,
)
func ProvideService(
ctx context.Context,
config Config,
gitReaderFactory *events.ReaderFactory[*gitevents.Reader],
pullReqEvFactory *events.ReaderFactory[*pullreqevents.Reader],
) (*Service, error) {
return New(ctx, config, gitReaderFactory, pullReqEvFactory)
}

0
internal/services/webhook/branch.go → internal/services/webhook/handler_branch.go
View File

0
internal/services/webhook/pullreq.go → internal/services/webhook/handler_pullreq.go
View File

0
internal/services/webhook/tag.go → internal/services/webhook/handler_tag.go
View File

18
internal/services/webhook/service.go
View File

@ -27,17 +27,15 @@ const (
type Config struct {
// UserAgentIdentity specifies the identity used for the user agent header
// IMPORTANT: do not include version.
UserAgentIdentity string `envconfig:"GITNESS_WEBHOOK_USER_AGENT_IDENTITY" default:"Gitness"`
UserAgentIdentity string
// HeaderIdentity specifies the identity used for headers in webhook calls (e.g. X-Gitness-Trigger, ...).
// NOTE: If no value is provided, the UserAgentIdentity will be used.
HeaderIdentity string `envconfig:"GITNESS_WEBHOOK_HEADER_IDENTITY"`
// EventReaderName is the name used to read events from stream.
// Note: this should be different for every running instance.
EventReaderName string `envconfig:"GITNESS_WEBHOOK_EVENT_READER_NAME"`
Concurrency int `envconfig:"GITNESS_WEBHOOK_CONCURRENCY" default:"4"`
MaxRetries int `envconfig:"GITNESS_WEBHOOK_MAX_RETRIES" default:"3"`
AllowPrivateNetwork bool `envconfig:"GITNESS_WEBHOOK_ALLOW_PRIVATE_NETWORK" default:"false"`
AllowLoopback bool `envconfig:"GITNESS_WEBHOOK_ALLOW_LOOPBACK" default:"false"`
HeaderIdentity string
EventReaderName string
Concurrency int
MaxRetries int
AllowPrivateNetwork bool
AllowLoopback bool
}
func (c *Config) Prepare() error {
@ -91,7 +89,7 @@ func NewService(ctx context.Context, config Config,
repoStore store.RepoStore, pullreqStore store.PullReqStore, urlProvider *url.Provider,
principalStore store.PrincipalStore, gitRPCClient gitrpc.Interface) (*Service, error) {
if err := config.Prepare(); err != nil {
return nil, fmt.Errorf("provided config is invalid: %w", err)
return nil, fmt.Errorf("provided webhook service config is invalid: %w", err)
}
service := &Service{
webhookStore: webhookStore,

15
internal/services/services.go → internal/services/wire.go
View File

@ -7,6 +7,7 @@ package services
import (
"github.com/harness/gitness/internal/services/job"
"github.com/harness/gitness/internal/services/pullreq"
"github.com/harness/gitness/internal/services/trigger"
"github.com/harness/gitness/internal/services/webhook"
"github.com/google/wire"
@ -19,20 +20,20 @@ var WireSet = wire.NewSet(
type Services struct {
Webhook *webhook.Service
PullReq *pullreq.Service
JobExecutor *job.Executor
Trigger *trigger.Service
JobScheduler *job.Scheduler
}
func ProvideServices(
webhooksSrv *webhook.Service,
pullReqSrv *pullreq.Service,
jobExecutor *job.Executor,
webhooksSvc *webhook.Service,
pullReqSvc *pullreq.Service,
triggerSvc *trigger.Service,
jobScheduler *job.Scheduler,
) Services {
return Services{
Webhook: webhooksSrv,
PullReq: pullReqSrv,
JobExecutor: jobExecutor,
Webhook: webhooksSvc,
PullReq: pullReqSvc,
Trigger: triggerSvc,
JobScheduler: jobScheduler,
}
}

6
internal/store/database/job.go
View File

@ -54,7 +54,8 @@ const (
,job_is_recurring
,job_recurring_cron
,job_consecutive_failures
,job_last_failure_error`
,job_last_failure_error
,job_group_id`
jobSelectBase = `
SELECT` + jobColumns + `
@ -101,6 +102,7 @@ func (s *JobStore) Create(ctx context.Context, job *types.Job) error {
,:job_recurring_cron
,:job_consecutive_failures
,:job_last_failure_error
,:job_group_id
)`
db := dbtx.GetAccessor(ctx, s.db)
@ -143,6 +145,7 @@ func (s *JobStore) Upsert(ctx context.Context, job *types.Job) error {
,:job_recurring_cron
,:job_consecutive_failures
,:job_last_failure_error
,:job_group_id
)
ON CONFLICT (job_uid) DO
UPDATE SET
@ -196,6 +199,7 @@ func (s *JobStore) UpdateDefinition(ctx context.Context, job *types.Job) error {
,job_scheduled = :job_scheduled
,job_is_recurring = :job_is_recurring
,job_recurring_cron = :job_recurring_cron
,job_group_id = :job_group_id
WHERE job_uid = :job_uid`
db := dbtx.GetAccessor(ctx, s.db)

1
internal/store/database/migrate/postgres/0025_alter_table_job_add_group_id.down.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE jobs DROP COLUMN job_group_id;

1
internal/store/database/migrate/postgres/0025_alter_table_job_add_group_id.up.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE jobs ADD COLUMN job_group_id TEXT NOT NULL DEFAULT '';

1
internal/store/database/migrate/sqlite/0025_alter_table_job_add_group_id.down.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE jobs DROP COLUMN job_group_id;

1
internal/store/database/migrate/sqlite/0025_alter_table_job_add_group_id.up.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE jobs ADD COLUMN job_group_id TEXT NOT NULL DEFAULT '';

34
internal/store/database/repo.go
View File

@ -170,21 +170,22 @@ func (s *RepoStore) Update(ctx context.Context, repo *types.Repository) error {
const sqlQuery = `
UPDATE repositories
SET
repo_version = :repo_version
,repo_updated = :repo_updated
,repo_parent_id = :repo_parent_id
,repo_uid = :repo_uid
,repo_description = :repo_description
,repo_is_public = :repo_is_public
,repo_default_branch = :repo_default_branch
,repo_pullreq_seq = :repo_pullreq_seq
,repo_num_forks = :repo_num_forks
,repo_num_pulls = :repo_num_pulls
,repo_num_closed_pulls = :repo_num_closed_pulls
,repo_num_open_pulls = :repo_num_open_pulls
,repo_num_merged_pulls = :repo_num_merged_pulls
,repo_importing = :repo_importing
,repo_importing_job_uid = :repo_importing_job_uid
repo_version = :repo_version
,repo_updated = :repo_updated
,repo_parent_id = :repo_parent_id
,repo_uid = :repo_uid
,repo_git_uid = :repo_git_uid
,repo_description = :repo_description
,repo_is_public = :repo_is_public
,repo_default_branch = :repo_default_branch
,repo_pullreq_seq = :repo_pullreq_seq
,repo_num_forks = :repo_num_forks
,repo_num_pulls = :repo_num_pulls
,repo_num_closed_pulls = :repo_num_closed_pulls
,repo_num_open_pulls = :repo_num_open_pulls
,repo_num_merged_pulls = :repo_num_merged_pulls
,repo_importing = :repo_importing
,repo_importing_job_uid = :repo_importing_job_uid
WHERE repo_id = :repo_id AND repo_version = :repo_version - 1`
updatedAt := time.Now()
@ -219,7 +220,8 @@ func (s *RepoStore) Update(ctx context.Context, repo *types.Repository) error {
// UpdateOptLock updates the repository using the optimistic locking mechanism.
func (s *RepoStore) UpdateOptLock(ctx context.Context,
repo *types.Repository,
mutateFn func(repository *types.Repository) error) (*types.Repository, error) {
mutateFn func(repository *types.Repository) error,
) (*types.Repository, error) {
for {
dup := *repo

18
types/config.go
View File

@ -200,4 +200,22 @@ type Config struct {
// finished and failed jobs will be purged from the DB.
PurgeFinishedOlderThan time.Duration `envconfig:"GITNESS_JOBS_PURGE_FINISHED_OLDER_THAN" default:"120h"`
}
Webhook struct {
// UserAgentIdentity specifies the identity used for the user agent header
// IMPORTANT: do not include version.
UserAgentIdentity string `envconfig:"GITNESS_WEBHOOK_USER_AGENT_IDENTITY" default:"Gitness"`
// HeaderIdentity specifies the identity used for headers in webhook calls (e.g. X-Gitness-Trigger, ...).
// NOTE: If no value is provided, the UserAgentIdentity will be used.
HeaderIdentity string `envconfig:"GITNESS_WEBHOOK_HEADER_IDENTITY"`
Concurrency int `envconfig:"GITNESS_WEBHOOK_CONCURRENCY" default:"4"`
MaxRetries int `envconfig:"GITNESS_WEBHOOK_MAX_RETRIES" default:"3"`
AllowPrivateNetwork bool `envconfig:"GITNESS_WEBHOOK_ALLOW_PRIVATE_NETWORK" default:"false"`
AllowLoopback bool `envconfig:"GITNESS_WEBHOOK_ALLOW_LOOPBACK" default:"false"`
}
Trigger struct {
Concurrency int `envconfig:"GITNESS_TRIGGER_CONCURRENCY" default:"4"`
MaxRetries int `envconfig:"GITNESS_TRIGGER_MAX_RETRIES" default:"3"`
}
}

1
types/job.go
View File

@ -27,6 +27,7 @@ type Job struct {
RecurringCron string `db:"job_recurring_cron"`
ConsecutiveFailures int `db:"job_consecutive_failures"`
LastFailureError string `db:"job_last_failure_error"`
GroupID string `db:"job_group_id"`
}
type JobStateChange struct {

9
web/package.json
View File

@ -15,7 +15,9 @@
},
"keywords": [],
"scripts": {
"dev": "NODE_ENV=development webpack serve --config config/webpack.dev.js",
"webpack": "NODE_ENV=development webpack serve --config config/webpack.dev.js",
"typed-scss": "typed-scss-modules src --watch",
"dev": "run-p webpack typed-scss",
"test": "jest src --silent",
"test:watch": "jest --watch",
"build": "rm -rf dist && webpack --config config/webpack.prod.js",
@ -62,7 +64,7 @@
"masonry-layout": "^4.2.2",
"moment": "^2.25.3",
"monaco-editor": "^0.40.0",
"monaco-editor-webpack-plugin": "^7.0.1",
"monaco-editor-webpack-plugin": "^7.1.0",
"qs": "^6.9.4",
"react": "^17.0.2",
"react-complex-tree": "^1.1.11",
@ -104,7 +106,7 @@
"@types/qs": "^6.9.4",
"@types/react": "^17.0.3",
"@types/react-dom": "^17.0.3",
"@types/react-router-dom": "^5.1.7",
"@types/react-router-dom": "^5.2.1",
"@types/react-table": "^7.0.18",
"@types/react-timeago": "^4.1.1",
"@types/testing-library__react-hooks": "^3.2.0",
@ -143,6 +145,7 @@
"ts-jest": "^26.5.5",
"ts-loader": "^9.2.6",
"tsconfig-paths-webpack-plugin": "^3.5.1",
"typed-scss-modules": "^7.1.4",
"typescript": "^4.7.4",
"url-loader": "^4.1.1",
"webpack": "^5.58.0",

9
web/src/App.module.scss.d.ts vendored
View File

@ -1,7 +1,4 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly main: string
readonly fullPage: string
}
export default styles
// This is an auto-generated file
export declare const fullPage: string
export declare const main: string

3
web/src/bootstrap.scss.d.ts vendored Normal file
View File

@ -0,0 +1,3 @@
/* eslint-disable */
// This is an auto-generated file
export declare const reactRoot: string

27
web/src/components/AuthLayout/AuthLayout.module.scss.d.ts vendored
View File

@ -1,16 +1,13 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly layout: string
readonly cardColumn: string
readonly card: string
readonly cardChildren: string
readonly imageColumn: string
readonly image: string
readonly subtractContainer: string
readonly subtractImage: string
readonly imageContainer: string
readonly overlayImage: string
readonly disclaimer: string
}
export default styles
// This is an auto-generated file
export declare const card: string
export declare const cardChildren: string
export declare const cardColumn: string
export declare const disclaimer: string
export declare const image: string
export declare const imageColumn: string
export declare const imageContainer: string
export declare const layout: string
export declare const overlayImage: string
export declare const subtractContainer: string
export declare const subtractImage: string

23
web/src/components/BranchTagSelect/BranchTagSelect.module.scss.d.ts vendored
View File

@ -1,14 +1,11 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly button: string
readonly prefix: string
readonly main: string
readonly input: string
readonly tabContainer: string
readonly branchesOnly: string
readonly popover: string
readonly listContainer: string
readonly newBtnText: string
}
export default styles
// This is an auto-generated file
export declare const branchesOnly: string
export declare const button: string
export declare const input: string
export declare const listContainer: string
export declare const main: string
export declare const newBtnText: string
export declare const popover: string
export declare const prefix: string
export declare const tabContainer: string

41
web/src/components/Changes/Changes.module.scss.d.ts vendored
View File

@ -1,23 +1,20 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly wrapper: string
readonly header: string
readonly stickied: string
readonly diffStatsLabel: string
readonly main: string
readonly enableDiffLineBreaks: string
readonly container: string
readonly hideBtn: string
readonly refreshIcon: string
readonly repeatBtn: string
readonly popover: string
readonly menuItem: string
readonly menuReviewItem: string
readonly reviewIcon: string
readonly reviewButton: string
readonly hide: string
readonly disabled: string
readonly commitsDropdown: string
}
export default styles
// This is an auto-generated file
export declare const commitsDropdown: string
export declare const container: string
export declare const diffStatsLabel: string
export declare const disabled: string
export declare const enableDiffLineBreaks: string
export declare const header: string
export declare const hide: string
export declare const hideBtn: string
export declare const main: string
export declare const menuItem: string
export declare const menuReviewItem: string
export declare const popover: string
export declare const refreshIcon: string
export declare const repeatBtn: string
export declare const reviewButton: string
export declare const reviewIcon: string
export declare const stickied: string
export declare const wrapper: string

13
web/src/components/Changes/ChangesDropdown.module.scss.d.ts vendored
View File

@ -1,9 +1,6 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly link: string
readonly filesMenu: string
readonly menuItem: string
readonly popover: string
}
export default styles
// This is an auto-generated file
export declare const filesMenu: string
export declare const link: string
export declare const menuItem: string
export declare const popover: string

15
web/src/components/CloneButtonTooltip/CloneButtonTooltip.module.scss.d.ts vendored
View File

@ -1,10 +1,7 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly container: string
readonly label: string
readonly layout: string
readonly url: string
readonly cloneCopyButton: string
}
export default styles
// This is an auto-generated file
export declare const cloneCopyButton: string
export declare const container: string
export declare const label: string
export declare const layout: string
export declare const url: string

11
web/src/components/CloneCredentialDialog/CloneCredentialDialog.module.scss.d.ts vendored
View File

@ -1,8 +1,5 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly layout: string
readonly url: string
readonly cloneCopyButton: string
}
export default styles
// This is an auto-generated file
export declare const cloneCopyButton: string
export declare const layout: string
export declare const url: string

7
web/src/components/CodeCommentStatusSelect/CodeCommentStatusSelect.module.scss.d.ts vendored
View File

@ -1,6 +1,3 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly select: string
}
export default styles
// This is an auto-generated file
export declare const select: string

35
web/src/components/CommentBox/CommentBox.module.scss.d.ts vendored
View File

@ -1,20 +1,17 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly main: string
readonly fluid: string
readonly box: string
readonly viewer: string
readonly deleted: string
readonly outdated: string
readonly replyPlaceHolder: string
readonly newCommentContainer: string
readonly hasThread: string
readonly editCommentContainer: string
readonly outletTopOfFirstOfComment: string
readonly standalone: string
readonly optionMenuIcon: string
readonly bp3Icon: string
readonly deleteIcon: string
}
export default styles
// This is an auto-generated file
export declare const box: string
export declare const bp3Icon: string
export declare const deleted: string
export declare const deleteIcon: string
export declare const editCommentContainer: string
export declare const fluid: string
export declare const hasThread: string
export declare const main: string
export declare const newCommentContainer: string
export declare const optionMenuIcon: string
export declare const outdated: string
export declare const outletTopOfFirstOfComment: string
export declare const replyPlaceHolder: string
export declare const standalone: string
export declare const viewer: string

13
web/src/components/CommitActions/CommitActions.module.scss.d.ts vendored
View File

@ -1,9 +1,6 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly container: string
readonly layout: string
readonly noCopy: string
readonly commitCopyButton: string
}
export default styles
// This is an auto-generated file
export declare const commitCopyButton: string
export declare const container: string
export declare const layout: string
export declare const noCopy: string

15
web/src/components/CommitDivergence/CommitDivergence.module.scss.d.ts vendored
View File

@ -1,10 +1,7 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly container: string
readonly main: string
readonly ahead: string
readonly behind: string
readonly pipe: string
}
export default styles
// This is an auto-generated file
export declare const ahead: string
export declare const behind: string
export declare const container: string
export declare const main: string
export declare const pipe: string

17
web/src/components/CommitInfo/CommitInfo.module.scss.d.ts vendored
View File

@ -1,11 +1,8 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly commitInfoContainer: string
readonly commitTitleContainer: string
readonly infoContainer: string
readonly alignContent: string
readonly infoText: string
readonly titleText: string
}
export default styles
// This is an auto-generated file
export declare const alignContent: string
export declare const commitInfoContainer: string
export declare const commitTitleContainer: string
export declare const infoContainer: string
export declare const infoText: string
export declare const titleText: string

17
web/src/components/CommitModalButton/CommitModalButton.module.scss.d.ts vendored
View File

@ -1,11 +1,8 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly main: string
readonly extendedDescription: string
readonly radioGroup: string
readonly directly: string
readonly newBranch: string
readonly newBranchContainer: string
}
export default styles
// This is an auto-generated file
export declare const directly: string
export declare const extendedDescription: string
export declare const main: string
export declare const newBranch: string
export declare const newBranchContainer: string
export declare const radioGroup: string

27
web/src/components/CommitsView/CommitsView.module.scss.d.ts vendored
View File

@ -1,16 +1,13 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly container: string
readonly table: string
readonly row: string
readonly rowText: string
readonly label: string
readonly refreshIcon: string
readonly fileButton: string
readonly layout: string
readonly noCopy: string
readonly commitFileButton: string
readonly commitRepoButton: string
}
export default styles
// This is an auto-generated file
export declare const commitFileButton: string
export declare const commitRepoButton: string
export declare const container: string
export declare const fileButton: string
export declare const label: string
export declare const layout: string
export declare const noCopy: string
export declare const refreshIcon: string
export declare const row: string
export declare const rowText: string
export declare const table: string

15
web/src/components/Console/Console.module.scss.d.ts vendored
View File

@ -1,10 +1,7 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly container: string
readonly log: string
readonly header: string
readonly headerLayout: string
readonly steps: string
}
export default styles
// This is an auto-generated file
export declare const container: string
export declare const header: string
export declare const headerLayout: string
export declare const log: string
export declare const steps: string

11
web/src/components/ConsoleLogs/ConsoleLogs.module.scss.d.ts vendored
View File

@ -1,8 +1,5 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly logLayout: string
readonly lineNumber: string
readonly log: string
}
export default styles
// This is an auto-generated file
export declare const lineNumber: string
export declare const log: string
export declare const logLayout: string

11
web/src/components/ConsoleStep/ConsoleStep.module.scss.d.ts vendored
View File

@ -1,8 +1,5 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly spin: string
readonly loading: string
readonly stepLayout: string
}
export default styles
// This is an auto-generated file
export declare const loading: string
export declare const spin: string
export declare const stepLayout: string

5
web/src/components/ConsoleStep/ConsoleStep.tsx
View File

@ -58,6 +58,7 @@ const ConsoleStep: FC<ConsoleStepProps> = ({ step, stageNumber, repoPath, pipeli
}
}
return () => {
setStreamingLogs([])
if (eventSourceRef.current) eventSourceRef.current.close()
}
}, [executionNumber, pipelineName, repoPath, stageNumber, step?.number, step?.status])
@ -74,7 +75,9 @@ const ConsoleStep: FC<ConsoleStepProps> = ({ step, stageNumber, repoPath, pipeli
}
let content
if (loading) {
if (!isOpened) {
content = null
} else if (loading) {
content = <div className={css.loading}>{getString('loading')}</div>
} else if (error && step?.status !== ExecutionState.RUNNING) {
content = <div>Error: {error.message}</div>

21
web/src/components/CreateBranchModal/CreateBranchModal.module.scss.d.ts vendored
View File

@ -1,13 +1,10 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly main: string
readonly title: string
readonly label: string
readonly container: string
readonly maxContainer: string
readonly branchTagSelect: string
readonly selectContainer: string
readonly popoverContainer: string
}
export default styles
// This is an auto-generated file
export declare const branchTagSelect: string
export declare const container: string
export declare const label: string
export declare const main: string
export declare const maxContainer: string
export declare const popoverContainer: string
export declare const selectContainer: string
export declare const title: string

21
web/src/components/CreateTagModal/CreateTagModal.module.scss.d.ts vendored
View File

@ -1,13 +1,10 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly main: string
readonly title: string
readonly label: string
readonly container: string
readonly extendedDescription: string
readonly branchTagSelect: string
readonly selectContainer: string
readonly popoverContainer: string
}
export default styles
// This is an auto-generated file
export declare const branchTagSelect: string
export declare const container: string
export declare const extendedDescription: string
export declare const label: string
export declare const main: string
export declare const popoverContainer: string
export declare const selectContainer: string
export declare const title: string

21
web/src/components/DiffViewer/DiffViewer.module.scss.d.ts vendored
View File

@ -1,13 +1,10 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly main: string
readonly readOnly: string
readonly collapsed: string
readonly diffHeader: string
readonly diffContent: string
readonly fname: string
readonly viewLabel: string
readonly standalone: string
}
export default styles
// This is an auto-generated file
export declare const collapsed: string
export declare const diffContent: string
export declare const diffHeader: string
export declare const fname: string
export declare const main: string
export declare const readOnly: string
export declare const standalone: string
export declare const viewLabel: string

2
web/src/components/DiffViewer/DiffViewerUtils.tsx
View File

@ -143,7 +143,7 @@ export function getCommentLineInfo(
) {
const isSideBySideView = viewStyle === ViewStyle.SIDE_BY_SIDE
const { left, lineNumber, filePath } = commentEntry
const filePathBody = contentDOM?.querySelector(`[data="${filePath}"`)
const filePathBody = filePath ? contentDOM?.querySelector(`[data="${filePath}"`) : contentDOM
const diffBody = filePathBody?.querySelector(
`${isSideBySideView ? `.d2h-file-side-diff${left ? '.left' : '.right'} ` : ''}.d2h-diff-tbody`

7
web/src/components/Editor/Editor.module.scss.d.ts vendored
View File

@ -1,6 +1,3 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly editor: string
}
export default styles
// This is an auto-generated file
export declare const editor: string

16
web/src/components/ExecutionPageHeader/ExecutionPageHeader.module.scss.d.ts vendored
View File

@ -1,11 +1,7 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly pageHeader: string
readonly header: string
readonly breadcrumb: string
readonly hash: string
readonly timer: string
readonly executionInfo: string
}
export default styles
// This is an auto-generated file
export declare const breadcrumb: string
export declare const executionInfo: string
export declare const hash: string
export declare const pageHeader: string
export declare const timer: string

19
web/src/components/ExecutionStageList/ExecutionStageList.module.scss.d.ts vendored
View File

@ -1,12 +1,9 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly container: string
readonly menu: string
readonly menuItem: string
readonly layout: string
readonly selected: string
readonly uid: string
readonly statusIcon: string
}
export default styles
// This is an auto-generated file
export declare const container: string
export declare const layout: string
export declare const menu: string
export declare const menuItem: string
export declare const selected: string
export declare const statusIcon: string
export declare const uid: string

21
web/src/components/ExecutionStatus/ExecutionStatus.module.scss.d.ts vendored
View File

@ -1,13 +1,10 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly main: string
readonly iconOnly: string
readonly noBackground: string
readonly pending: string
readonly running: string
readonly success: string
readonly failure: string
readonly error: string
}
export default styles
// This is an auto-generated file
export declare const error: string
export declare const failure: string
export declare const iconOnly: string
export declare const main: string
export declare const noBackground: string
export declare const pending: string
export declare const running: string
export declare const success: string

15
web/src/components/ExecutionStatusLabel/ExecutionStatusLabel.module.scss.d.ts vendored
View File

@ -1,10 +1,7 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly executionStatus: string
readonly iconOnly: string
readonly success: string
readonly failure: string
readonly open: string
}
export default styles
// This is an auto-generated file
export declare const executionStatus: string
export declare const failure: string
export declare const iconOnly: string
export declare const open: string
export declare const success: string

11
web/src/components/GitRefLink/GitRefLink.module.scss.d.ts vendored
View File

@ -1,8 +1,5 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly link: string
readonly copyContainer: string
readonly linkText: string
}
export default styles
// This is an auto-generated file
export declare const copyContainer: string
export declare const link: string
export declare const linkText: string

11
web/src/components/GitnessLogo/GitnessLogo.module.scss.d.ts vendored
View File

@ -1,8 +1,5 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly main: string
readonly layout: string
readonly text: string
}
export default styles
// This is an auto-generated file
export declare const layout: string
export declare const main: string
export declare const text: string

15
web/src/components/ImageCarousel/ImageCarousel.module.scss.d.ts vendored
View File

@ -1,10 +1,7 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly imageModal: string
readonly content: string
readonly image: string
readonly vertical: string
readonly portalContainer: string
}
export default styles
// This is an auto-generated file
export declare const content: string
export declare const image: string
export declare const imageModal: string
export declare const portalContainer: string
export declare const vertical: string

17
web/src/components/LatestCommit/LatestCommit.module.scss.d.ts vendored
View File

@ -1,11 +1,8 @@
/* eslint-disable */
// this is an auto-generated file
declare const styles: {
readonly latestCommit: string
readonly standalone: string
readonly forFile: string
readonly shaBtn: string
readonly commitLink: string
readonly time: string
}
export default styles
// This is an auto-generated file
export declare const commitLink: string
export declare const forFile: string
export declare const latestCommit: string
export declare const shaBtn: string
export declare const standalone: string
export declare const time: string

Some files were not shown because too many files have changed in this diff Show More