diff --git a/app/api/controller/githook/controller.go b/app/api/controller/githook/controller.go index 6ccf7ca4a..9fcc959be 100644 --- a/app/api/controller/githook/controller.go +++ b/app/api/controller/githook/controller.go @@ -56,6 +56,7 @@ type Controller struct { updateExtender UpdateExtender postReceiveExtender PostReceiveExtender sseStreamer sse.Streamer + lfsStore store.LFSObjectStore } func NewController( @@ -75,6 +76,7 @@ func NewController( updateExtender UpdateExtender, postReceiveExtender PostReceiveExtender, sseStreamer sse.Streamer, + lfsStore store.LFSObjectStore, ) *Controller { return &Controller{ authorizer: authorizer, @@ -93,6 +95,7 @@ func NewController( updateExtender: updateExtender, postReceiveExtender: postReceiveExtender, sseStreamer: sseStreamer, + lfsStore: lfsStore, } } diff --git a/app/api/controller/githook/git.go b/app/api/controller/githook/git.go index feb05e037..245252bf0 100644 --- a/app/api/controller/githook/git.go +++ b/app/api/controller/githook/git.go @@ -36,4 +36,5 @@ type RestrictedGIT interface { ctx context.Context, params *git.FindOversizeFilesParams, ) (*git.FindOversizeFilesOutput, error) + ListLFSPointers(ctx context.Context, params *git.ListLFSPointersParams) (*git.ListLFSPointersOutput, error) } diff --git a/app/api/controller/githook/pre_receive.go b/app/api/controller/githook/pre_receive.go index 1bb2501cb..71e17bebd 100644 --- a/app/api/controller/githook/pre_receive.go +++ b/app/api/controller/githook/pre_receive.go @@ -128,6 +128,14 @@ func (c *Controller) PreReceive( return hook.Output{}, err } + err = c.checkLFSObjects(ctx, rgit, repo, in, &output) + if output.Error != nil { + return output, nil + } + if err != nil { + return hook.Output{}, err + } + return output, nil } diff --git a/app/api/controller/githook/pre_receive_file_size_limit.go b/app/api/controller/githook/pre_receive_file_size_limit.go index 5a47e1dad..7f9474563 100644 --- a/app/api/controller/githook/pre_receive_file_size_limit.go +++ b/app/api/controller/githook/pre_receive_file_size_limit.go @@ -34,15 +34,7 @@ func (c *Controller) checkFileSizeLimit( output *hook.Output, ) error { // return if all new refs are nil refs - allNilRefs := true - for _, refUpdate := range in.RefUpdates { - if refUpdate.New.IsNil() { - continue - } - allNilRefs = false - break - } - if allNilRefs { + if isAllRefDeletions(in.RefUpdates) { return nil } diff --git a/app/api/controller/githook/pre_receive_lfs_objects.go b/app/api/controller/githook/pre_receive_lfs_objects.go new file mode 100644 index 000000000..77eecf33e --- /dev/null +++ b/app/api/controller/githook/pre_receive_lfs_objects.go @@ -0,0 +1,84 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package githook + +import ( + "context" + "fmt" + + "github.com/harness/gitness/git" + "github.com/harness/gitness/git/hook" + "github.com/harness/gitness/types" + + "github.com/gotidy/ptr" +) + +func (c *Controller) checkLFSObjects( + ctx context.Context, + rgit RestrictedGIT, + repo *types.RepositoryCore, + in types.GithookPreReceiveInput, + output *hook.Output, +) error { + // return if all new refs are nil refs + if isAllRefDeletions(in.RefUpdates) { + return nil + } + + res, err := rgit.ListLFSPointers(ctx, + &git.ListLFSPointersParams{ + ReadParams: git.ReadParams{ + RepoUID: repo.GitUID, + AlternateObjectDirs: in.Environment.AlternateObjectDirs, + }, + }, + ) + if err != nil { + return fmt.Errorf("failed to list lfs pointers: %w", err) + } + + if len(res.LFSInfos) == 0 { + return nil + } + + oids := make([]string, len(res.LFSInfos)) + for i := range res.LFSInfos { + oids[i] = res.LFSInfos[i].OID + } + + existingObjs, err := c.lfsStore.FindMany(ctx, in.RepoID, oids) + if err != nil { + return fmt.Errorf("failed to find lfs objects: %w", err) + } + + //nolint:lll + if len(existingObjs) != len(oids) { + output.Error = ptr.String( + "Changes blocked by missing lfs objects. Please try `git lfs push --all` or check if LFS is setup properly.") + return nil + } + + return nil +} + +func isAllRefDeletions(refUpdates []hook.ReferenceUpdate) bool { + for _, refUpdate := range refUpdates { + if !refUpdate.New.IsNil() { + return false + } + } + + return true +} diff --git a/app/api/controller/githook/wire.go b/app/api/controller/githook/wire.go index 1bf0e5a56..902029c00 100644 --- a/app/api/controller/githook/wire.go +++ b/app/api/controller/githook/wire.go @@ -60,6 +60,7 @@ func ProvideController( updateExtender UpdateExtender, postReceiveExtender PostReceiveExtender, sseStreamer sse.Streamer, + lfsStore store.LFSObjectStore, ) *Controller { ctrl := NewController( authorizer, @@ -78,6 +79,7 @@ func ProvideController( updateExtender, postReceiveExtender, sseStreamer, + lfsStore, ) // TODO: improve wiring if possible diff --git a/app/api/controller/lfs/authenticate.go b/app/api/controller/lfs/authenticate.go new file mode 100644 index 000000000..566e72452 --- /dev/null +++ b/app/api/controller/lfs/authenticate.go @@ -0,0 +1,43 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lfs + +import ( + "context" + "fmt" + + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/app/auth/authn" + "github.com/harness/gitness/app/token" +) + +func (c *Controller) Authenticate( + ctx context.Context, + session *auth.Session, + repoRef string, +) (*AuthenticateResponse, error) { + jwt, err := c.remoteAuth.GenerateToken(ctx, session.Principal.ID, session.Principal.Type, repoRef) + if err != nil { + return nil, fmt.Errorf("failed to generate auth token: %w", err) + } + + return &AuthenticateResponse{ + Header: map[string]string{ + "Authorization": authn.HeaderTokenPrefixRemoteAuth + jwt, + }, + HRef: c.urlProvider.GenerateGITCloneURL(ctx, repoRef) + "/info/lfs", + ExpiresIn: token.RemoteAuthTokenLifeTime, + }, nil +} diff --git a/app/api/controller/lfs/controller.go b/app/api/controller/lfs/controller.go new file mode 100644 index 000000000..a111b80ef --- /dev/null +++ b/app/api/controller/lfs/controller.go @@ -0,0 +1,97 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lfs + +import ( + "context" + "fmt" + + apiauth "github.com/harness/gitness/app/api/auth" + "github.com/harness/gitness/app/api/usererror" + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/app/auth/authz" + "github.com/harness/gitness/app/services/refcache" + "github.com/harness/gitness/app/services/remoteauth" + "github.com/harness/gitness/app/store" + "github.com/harness/gitness/app/url" + "github.com/harness/gitness/blob" + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" +) + +const ( + lfsObjectsPathFormat = "lfs/%s" +) + +type Controller struct { + authorizer authz.Authorizer + repoFinder refcache.RepoFinder + principalStore store.PrincipalStore + lfsStore store.LFSObjectStore + blobStore blob.Store + remoteAuth remoteauth.Service + urlProvider url.Provider +} + +func NewController( + authorizer authz.Authorizer, + repoFinder refcache.RepoFinder, + principalStore store.PrincipalStore, + lfsStore store.LFSObjectStore, + blobStore blob.Store, + remoteAuth remoteauth.Service, + urlProvider url.Provider, +) *Controller { + return &Controller{ + authorizer: authorizer, + repoFinder: repoFinder, + principalStore: principalStore, + lfsStore: lfsStore, + blobStore: blobStore, + remoteAuth: remoteAuth, + urlProvider: urlProvider, + } +} + +func (c *Controller) getRepoCheckAccess( + ctx context.Context, + session *auth.Session, + repoRef string, + reqPermission enum.Permission, + allowedRepoStates ...enum.RepoState, +) (*types.RepositoryCore, error) { + if repoRef == "" { + return nil, usererror.BadRequest("A valid repository reference must be provided.") + } + + repo, err := c.repoFinder.FindByRef(ctx, repoRef) + if err != nil { + return nil, fmt.Errorf("failed to find repository: %w", err) + } + + if err := apiauth.CheckRepoState(ctx, session, repo, reqPermission, allowedRepoStates...); err != nil { + return nil, err + } + + if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, reqPermission); err != nil { + return nil, fmt.Errorf("access check failed: %w", err) + } + + return repo, nil +} + +func getLFSObjectPath(oid string) string { + return fmt.Sprintf(lfsObjectsPathFormat, oid) +} diff --git a/app/api/controller/lfs/download.go b/app/api/controller/lfs/download.go new file mode 100644 index 000000000..ad55ea1e5 --- /dev/null +++ b/app/api/controller/lfs/download.go @@ -0,0 +1,48 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lfs + +import ( + "context" + "fmt" + "io" + + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/types/enum" +) + +func (c *Controller) Download(ctx context.Context, + session *auth.Session, + repoRef string, + oid string, +) (io.ReadCloser, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView) + if err != nil { + return nil, fmt.Errorf("failed to acquire access to repo: %w", err) + } + + _, err = c.lfsStore.Find(ctx, repo.ID, oid) + if err != nil { + return nil, fmt.Errorf("failed to find the oid %q for the repo: %w", oid, err) + } + + objPath := getLFSObjectPath(oid) + file, err := c.blobStore.Download(ctx, objPath) + if err != nil { + return nil, fmt.Errorf("failed to download file from blobstore: %w", err) + } + + return file, nil +} diff --git a/app/api/controller/lfs/errors.go b/app/api/controller/lfs/errors.go new file mode 100644 index 000000000..5a09c3c9c --- /dev/null +++ b/app/api/controller/lfs/errors.go @@ -0,0 +1,23 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lfs + +var ( + // These are per-object errors when returned status code is 200. + errNotFound = ObjectError{ + Code: 404, + Message: "The object does not exist on the server.", + } +) diff --git a/app/api/controller/lfs/transfer.go b/app/api/controller/lfs/transfer.go new file mode 100644 index 000000000..201c20685 --- /dev/null +++ b/app/api/controller/lfs/transfer.go @@ -0,0 +1,136 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lfs + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/harness/gitness/app/api/usererror" + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/app/url" + "github.com/harness/gitness/store" + "github.com/harness/gitness/types/enum" +) + +func (c *Controller) LFSTransfer(ctx context.Context, + session *auth.Session, + repoRef string, + in *TransferInput, +) (*TransferOutput, error) { + reqPermission := enum.PermissionRepoView + if in.Operation == enum.GitLFSOperationTypeUpload { + reqPermission = enum.PermissionRepoPush + } + + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, reqPermission) + if err != nil { + return nil, err + } + + // TODO check if server supports client's transfer adapters + var objResponses []ObjectResponse + switch { + case in.Operation == enum.GitLFSOperationTypeDownload: + for _, obj := range in.Objects { + var objResponse = ObjectResponse{ + Pointer: Pointer{ + OId: obj.OId, + Size: obj.Size, + }, + } + + object, err := c.lfsStore.Find(ctx, repo.ID, obj.OId) + if errors.Is(err, store.ErrResourceNotFound) { + objResponse.Error = &errNotFound + objResponses = append(objResponses, objResponse) + continue + } + if err != nil { + return nil, fmt.Errorf("failed to find object: %w", err) + } + + // size is not a required query param for download hence nil + downloadURL := getRedirectRef(ctx, c.urlProvider, repoRef, obj.OId, nil) + + objResponse = ObjectResponse{ + Pointer: Pointer{ + OId: object.OID, + Size: object.Size, + }, + Actions: map[string]Action{ + "download": { + Href: downloadURL, + Header: map[string]string{"Content-Type": "application/octet-stream"}, + }, + }, + } + + objResponses = append(objResponses, objResponse) + } + + case in.Operation == enum.GitLFSOperationTypeUpload: + for _, obj := range in.Objects { + objResponse := ObjectResponse{ + Pointer: Pointer{ + OId: obj.OId, + Size: obj.Size, + }, + } + // we dont create the object in lfs store here as the upload might fail in blob store. + _, err := c.lfsStore.Find(ctx, repo.ID, obj.OId) + if err == nil { + // no need to re-upload existing LFS objects + objResponses = append(objResponses, objResponse) + continue + } + + if !errors.Is(err, store.ErrResourceNotFound) { + return nil, fmt.Errorf("failed to find object: %w", err) + } + + uploadURL := getRedirectRef(ctx, c.urlProvider, repoRef, obj.OId, &obj.Size) + + objResponse.Actions = map[string]Action{ + "upload": { + Href: uploadURL, + Header: map[string]string{"Content-Type": "application/octet-stream"}, + }, + } + + objResponses = append(objResponses, objResponse) + } + + default: + return nil, usererror.BadRequestf("git-lfs operation %q is not supported", in.Operation) + } + + return &TransferOutput{ + Transfer: enum.GitLFSTransferTypeBasic, + Objects: objResponses, + }, nil +} + +func getRedirectRef(ctx context.Context, urlProvider url.Provider, repoPath, oID string, size *int64) string { + baseGitURL := urlProvider.GenerateGITCloneURL(ctx, repoPath) + queryParams := "oid=" + oID + if size != nil { + queryParams += "&size=" + strconv.FormatInt(*size, 10) + } + + return baseGitURL + "/info/lfs/objects/?" + queryParams +} diff --git a/app/api/controller/lfs/types.go b/app/api/controller/lfs/types.go new file mode 100644 index 000000000..57857902b --- /dev/null +++ b/app/api/controller/lfs/types.go @@ -0,0 +1,71 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lfs + +import ( + "time" + + "github.com/harness/gitness/types/enum" +) + +type Reference struct { + Name string `json:"name"` +} + +// Pointer contains LFS pointer data. +type Pointer struct { + OId string `json:"oid"` + Size int64 `json:"size"` +} + +type TransferInput struct { + Operation enum.GitLFSOperationType `json:"operation"` + Transfers []enum.GitLFSTransferType `json:"transfers,omitempty"` + Ref *Reference `json:"ref,omitempty"` + Objects []Pointer `json:"objects"` + HashAlgo string `json:"hash_algo,omitempty"` +} + +// ObjectError defines the JSON structure returned to the client in case of an error. +type ObjectError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// Action provides a structure with information about next actions fo the object. +type Action struct { + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` + ExpiresIn *time.Duration `json:"expires_in,omitempty"` +} + +// ObjectResponse is object metadata as seen by clients of the LFS server. +type ObjectResponse struct { + Pointer + Authenticated *bool `json:"authenticated,omitempty"` + Actions map[string]Action `json:"actions"` + Error *ObjectError `json:"error,omitempty"` +} + +type TransferOutput struct { + Transfer enum.GitLFSTransferType `json:"transfer"` + Objects []ObjectResponse `json:"objects"` +} + +type AuthenticateResponse struct { + Header map[string]string `json:"header"` + HRef string `json:"href"` + ExpiresIn time.Duration `json:"expires_in"` +} diff --git a/app/api/controller/lfs/upload.go b/app/api/controller/lfs/upload.go new file mode 100644 index 000000000..3f7a3e377 --- /dev/null +++ b/app/api/controller/lfs/upload.go @@ -0,0 +1,76 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lfs + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/harness/gitness/app/api/usererror" + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/store" + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" +) + +type UploadOut struct { + ObjectPath string `json:"object_path"` +} + +func (c *Controller) Upload(ctx context.Context, + session *auth.Session, + repoRef string, + pointer Pointer, + file io.Reader, +) (*UploadOut, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoPush) + if err != nil { + return nil, fmt.Errorf("failed to acquire access to repo: %w", err) + } + + if file == nil { + return nil, usererror.BadRequest("no file or content provided") + } + + bufReader := bufio.NewReader(file) + objPath := getLFSObjectPath(pointer.OId) + + err = c.blobStore.Upload(ctx, bufReader, objPath) + if err != nil { + return nil, fmt.Errorf("failed to upload file: %w", err) + } + + now := time.Now() + object := &types.LFSObject{ + OID: pointer.OId, + Size: pointer.Size, + Created: now.UnixMilli(), + CreatedBy: session.Principal.ID, + RepoID: repo.ID, + } + // create the object in lfs store after successful upload to the blob store. + err = c.lfsStore.Create(ctx, object) + if err != nil && !errors.Is(err, store.ErrDuplicate) { + return nil, fmt.Errorf("failed to find object: %w", err) + } + + return &UploadOut{ + ObjectPath: objPath, + }, nil +} diff --git a/app/api/controller/lfs/wire.go b/app/api/controller/lfs/wire.go new file mode 100644 index 000000000..e10f820cc --- /dev/null +++ b/app/api/controller/lfs/wire.go @@ -0,0 +1,42 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lfs + +import ( + "github.com/harness/gitness/app/auth/authz" + "github.com/harness/gitness/app/services/refcache" + "github.com/harness/gitness/app/services/remoteauth" + "github.com/harness/gitness/app/store" + "github.com/harness/gitness/app/url" + "github.com/harness/gitness/blob" + + "github.com/google/wire" +) + +var WireSet = wire.NewSet( + ProvideController, +) + +func ProvideController( + authorizer authz.Authorizer, + repoFinder refcache.RepoFinder, + principalStore store.PrincipalStore, + lfsStore store.LFSObjectStore, + blobStore blob.Store, + remoteAuth remoteauth.Service, + urlProvider url.Provider, +) *Controller { + return NewController(authorizer, repoFinder, principalStore, lfsStore, blobStore, remoteAuth, urlProvider) +} diff --git a/app/api/controller/upload/download.go b/app/api/controller/upload/download.go index 7730b4b30..6c098d981 100644 --- a/app/api/controller/upload/download.go +++ b/app/api/controller/upload/download.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "io" + "time" "github.com/harness/gitness/app/auth" "github.com/harness/gitness/blob" @@ -38,7 +39,7 @@ func (c *Controller) Download( fileBucketPath := getFileBucketPath(repo.ID, filePath) - signedURL, err := c.blobStore.GetSignedURL(ctx, fileBucketPath) + signedURL, err := c.blobStore.GetSignedURL(ctx, fileBucketPath, time.Now().Add(1*time.Hour)) if err != nil && !errors.Is(err, blob.ErrNotSupported) { return "", nil, fmt.Errorf("failed to get signed URL: %w", err) } diff --git a/app/api/controller/user/login.go b/app/api/controller/user/login.go index 846513809..d010bb830 100644 --- a/app/api/controller/user/login.go +++ b/app/api/controller/user/login.go @@ -16,11 +16,7 @@ package user import ( "context" - "crypto/rand" "errors" - "fmt" - "math/big" - "time" "github.com/harness/gitness/app/api/usererror" "github.com/harness/gitness/app/token" @@ -69,10 +65,8 @@ func (c *Controller) Login( return nil, usererror.ErrNotFound } - tokenIdentifier, err := GenerateSessionTokenIdentifier() - if err != nil { - return nil, err - } + tokenIdentifier := token.GenerateIdentifier("login") + token, jwtToken, err := token.CreateUserSession(ctx, c.tokenStore, user, tokenIdentifier) if err != nil { return nil, err @@ -80,11 +74,3 @@ func (c *Controller) Login( return &types.TokenResponse{Token: *token, AccessToken: jwtToken}, nil } - -func GenerateSessionTokenIdentifier() (string, error) { - r, err := rand.Int(rand.Reader, big.NewInt(10000)) - if err != nil { - return "", fmt.Errorf("failed to generate random number: %w", err) - } - return fmt.Sprintf("login-%d-%04d", time.Now().Unix(), r.Int64()), nil -} diff --git a/app/api/handler/lfs/download.go b/app/api/handler/lfs/download.go new file mode 100644 index 000000000..435edb8ad --- /dev/null +++ b/app/api/handler/lfs/download.go @@ -0,0 +1,58 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lfs + +import ( + "errors" + "net/http" + + apiauth "github.com/harness/gitness/app/api/auth" + "github.com/harness/gitness/app/api/controller/lfs" + "github.com/harness/gitness/app/api/render" + "github.com/harness/gitness/app/api/request" + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/app/url" +) + +func HandleLFSDownload(controller *lfs.Controller, urlProvider url.Provider) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + session, _ := request.AuthSessionFrom(ctx) + repoRef, err := request.GetRepoRefFromPath(r) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + oid, err := request.GetObjectIDFromQuery(r) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + file, err := controller.Download(ctx, session, repoRef, oid) + if errors.Is(err, apiauth.ErrNotAuthorized) && auth.IsAnonymousSession(session) { + render.GitBasicAuth(ctx, w, urlProvider) + return + } + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + defer file.Close() + // apply max byte size + render.Reader(ctx, w, http.StatusOK, file) + } +} diff --git a/app/api/handler/lfs/transfer.go b/app/api/handler/lfs/transfer.go new file mode 100644 index 000000000..84643c98c --- /dev/null +++ b/app/api/handler/lfs/transfer.go @@ -0,0 +1,60 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lfs + +import ( + "encoding/json" + "errors" + "net/http" + + apiauth "github.com/harness/gitness/app/api/auth" + "github.com/harness/gitness/app/api/controller/lfs" + "github.com/harness/gitness/app/api/render" + "github.com/harness/gitness/app/api/request" + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/app/url" +) + +func HandleLFSTransfer(lfsCtrl *lfs.Controller, urlProvider url.Provider) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + session, _ := request.AuthSessionFrom(ctx) + repoRef, err := request.GetRepoRefFromPath(r) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + in := new(lfs.TransferInput) + err = json.NewDecoder(r.Body).Decode(in) + if err != nil { + render.BadRequestf(ctx, w, "Invalid Request Body: %s.", err) + return + } + + w.Header().Set("Content-Type", "application/vnd.git-lfs+json") + out, err := lfsCtrl.LFSTransfer(ctx, session, repoRef, in) + if errors.Is(err, apiauth.ErrNotAuthorized) && auth.IsAnonymousSession(session) { + render.GitBasicAuth(ctx, w, urlProvider) + return + } + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + render.JSON(w, http.StatusOK, out) + } +} diff --git a/app/api/handler/lfs/upload.go b/app/api/handler/lfs/upload.go new file mode 100644 index 000000000..e97e04646 --- /dev/null +++ b/app/api/handler/lfs/upload.go @@ -0,0 +1,65 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lfs + +import ( + "errors" + "net/http" + + apiauth "github.com/harness/gitness/app/api/auth" + "github.com/harness/gitness/app/api/controller/lfs" + "github.com/harness/gitness/app/api/render" + "github.com/harness/gitness/app/api/request" + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/app/url" +) + +func HandleLFSUpload(controller *lfs.Controller, urlProvider url.Provider) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + session, _ := request.AuthSessionFrom(ctx) + repoRef, err := request.GetRepoRefFromPath(r) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + oid, err := request.GetObjectIDFromQuery(r) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + size, err := request.GetObjectSizeFromQuery(r) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + // apply max byte size from the request body + + res, err := controller.Upload(ctx, session, repoRef, lfs.Pointer{OId: oid, Size: size}, r.Body) + if errors.Is(err, apiauth.ErrNotAuthorized) && auth.IsAnonymousSession(session) { + render.GitBasicAuth(ctx, w, urlProvider) + return + } + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + render.JSON(w, http.StatusCreated, res) + } +} diff --git a/app/api/handler/repo/git_info_refs.go b/app/api/handler/repo/git_info_refs.go index 57959e38f..45353bc1a 100644 --- a/app/api/handler/repo/git_info_refs.go +++ b/app/api/handler/repo/git_info_refs.go @@ -57,7 +57,7 @@ func HandleGitInfoRefs(repoCtrl *repo.Controller, urlProvider url.Provider) http err = repoCtrl.GitInfoRefs(ctx, session, repoRef, service, gitProtocol, w) if errors.Is(err, apiauth.ErrNotAuthorized) && auth.IsAnonymousSession(session) { - renderBasicAuth(ctx, w, urlProvider) + render.GitBasicAuth(ctx, w, urlProvider) return } if err != nil { @@ -67,14 +67,6 @@ func HandleGitInfoRefs(repoCtrl *repo.Controller, urlProvider url.Provider) http } } -// renderBasicAuth renders a response that indicates that the client (GIT) requires basic authentication. -// This is required in order to tell git CLI to query user credentials. -func renderBasicAuth(ctx context.Context, w http.ResponseWriter, urlProvider url.Provider) { - // Git doesn't seem to handle "realm" - so it doesn't seem to matter for basic user CLI interactions. - w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Basic realm="%s"`, urlProvider.GetAPIHostname(ctx))) - w.WriteHeader(http.StatusUnauthorized) -} - func pktError(ctx context.Context, w http.ResponseWriter, err error) { terr := usererror.Translate(ctx, err) w.WriteHeader(terr.Status) diff --git a/app/api/handler/repo/git_service_pack.go b/app/api/handler/repo/git_service_pack.go index ed5cea941..0c728dfa3 100644 --- a/app/api/handler/repo/git_service_pack.go +++ b/app/api/handler/repo/git_service_pack.go @@ -80,7 +80,7 @@ func HandleGitServicePack( Protocol: gitProtocol, }) if errors.Is(err, apiauth.ErrNotAuthorized) && auth.IsAnonymousSession(session) { - renderBasicAuth(ctx, w, urlProvider) + render.GitBasicAuth(ctx, w, urlProvider) return } if err != nil { diff --git a/app/api/render/render.go b/app/api/render/render.go index a6031c3a6..22a1f09d5 100644 --- a/app/api/render/render.go +++ b/app/api/render/render.go @@ -17,6 +17,7 @@ package render import ( "context" "encoding/json" + "fmt" "io" "net/http" "os" @@ -24,6 +25,7 @@ import ( "github.com/harness/gitness/app/api/usererror" "github.com/harness/gitness/app/services/protection" + "github.com/harness/gitness/app/url" "github.com/harness/gitness/errors" "github.com/harness/gitness/git/api" "github.com/harness/gitness/types" @@ -136,6 +138,14 @@ func Violations(w http.ResponseWriter, violations []types.RuleViolations) { }) } +// GitBasicAuth renders a response that indicates that the client (GIT) requires basic authentication. +// This is required in order to tell git CLI to query user credentials. +func GitBasicAuth(ctx context.Context, w http.ResponseWriter, urlProvider url.Provider) { + // Git doesn't seem to handle "realm" - so it doesn't seem to matter for basic user CLI interactions. + w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Basic realm="%s"`, urlProvider.GetAPIHostname(ctx))) + w.WriteHeader(http.StatusUnauthorized) +} + func setCommonHeaders(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("X-Content-Type-Options", "nosniff") diff --git a/app/api/request/lfs.go b/app/api/request/lfs.go new file mode 100644 index 000000000..0b118ee4f --- /dev/null +++ b/app/api/request/lfs.go @@ -0,0 +1,30 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package request + +import "net/http" + +const ( + QueryParamObjectID = "oid" + QueryParamObjectSize = "size" +) + +func GetObjectIDFromQuery(r *http.Request) (string, error) { + return QueryParamOrError(r, QueryParamObjectID) +} + +func GetObjectSizeFromQuery(r *http.Request) (int64, error) { + return QueryParamAsPositiveInt64OrError(r, QueryParamObjectSize) +} diff --git a/app/auth/authn/jwt.go b/app/auth/authn/jwt.go index fdb04e01c..8a202b9f9 100644 --- a/app/auth/authn/jwt.go +++ b/app/auth/authn/jwt.go @@ -30,6 +30,12 @@ import ( gojwt "github.com/golang-jwt/jwt" ) +const ( + headerTokenPrefixBearer = "Bearer " + //nolint:gosec // wrong flagging + HeaderTokenPrefixRemoteAuth = "RemoteAuth " +) + var _ Authenticator = (*JWTAuthenticator)(nil) // JWTAuthenticator uses the provided JWT to authenticate the caller. @@ -162,8 +168,11 @@ func extractToken(r *http.Request, cookieName string) string { _, pwd, _ := r.BasicAuth() return pwd // strip bearer prefix if present - case strings.HasPrefix(headerToken, "Bearer "): - return headerToken[7:] + case strings.HasPrefix(headerToken, headerTokenPrefixBearer): + return headerToken[len(headerTokenPrefixBearer):] + // for ssh git-lfs-authenticate the returned token prefix would be RemoteAuth of type JWT + case strings.HasPrefix(headerToken, HeaderTokenPrefixRemoteAuth): + return headerToken[len(HeaderTokenPrefixRemoteAuth):] // otherwise use value as is case headerToken != "": return headerToken diff --git a/app/router/git.go b/app/router/git.go index 64a73360c..cf656b19b 100644 --- a/app/router/git.go +++ b/app/router/git.go @@ -18,7 +18,9 @@ import ( "fmt" "net/http" + "github.com/harness/gitness/app/api/controller/lfs" "github.com/harness/gitness/app/api/controller/repo" + handlerlfs "github.com/harness/gitness/app/api/handler/lfs" handlerrepo "github.com/harness/gitness/app/api/handler/repo" middlewareauthn "github.com/harness/gitness/app/api/middleware/authn" middlewareauthz "github.com/harness/gitness/app/api/middleware/authz" @@ -45,6 +47,7 @@ func NewGitHandler( authenticator authn.Authenticator, repoCtrl *repo.Controller, usageSender usage.Sender, + lfsCtrl *lfs.Controller, ) http.Handler { // maxRepoDepth depends on config maxRepoDepth := check.MaxRepoPathDepth @@ -98,6 +101,9 @@ func NewGitHandler( r.Get("/objects/{head:[0-9a-f]{2}}/{hash:[0-9a-f]{38}}", stubGitHandler()) r.Get("/objects/pack/pack-{file:[0-9a-f]{40}}.pack", stubGitHandler()) r.Get("/objects/pack/pack-{file:[0-9a-f]{40}}.idx", stubGitHandler()) + + // Git LFS API + GitLFSHandler(r, lfsCtrl, urlProvider) }) }) @@ -111,3 +117,14 @@ func stubGitHandler() http.HandlerFunc { w.WriteHeader(http.StatusBadGateway) } } + +func GitLFSHandler(r chi.Router, lfsCtrl *lfs.Controller, urlProvider url.Provider) { + r.Route("/info/lfs", func(r chi.Router) { + r.Route("/objects", func(r chi.Router) { + r.Post("/batch", handlerlfs.HandleLFSTransfer(lfsCtrl, urlProvider)) + // direct download and upload handlers for lfs objects + r.Put("/", handlerlfs.HandleLFSUpload(lfsCtrl, urlProvider)) + r.Get("/", handlerlfs.HandleLFSDownload(lfsCtrl, urlProvider)) + }) + }) +} diff --git a/app/router/wire.go b/app/router/wire.go index 1651df4ac..61d0f5016 100644 --- a/app/router/wire.go +++ b/app/router/wire.go @@ -25,6 +25,7 @@ import ( "github.com/harness/gitness/app/api/controller/gitspace" "github.com/harness/gitness/app/api/controller/infraprovider" "github.com/harness/gitness/app/api/controller/keywordsearch" + "github.com/harness/gitness/app/api/controller/lfs" "github.com/harness/gitness/app/api/controller/logs" "github.com/harness/gitness/app/api/controller/migrate" "github.com/harness/gitness/app/api/controller/pipeline" @@ -110,6 +111,7 @@ func ProvideRouter( openapi openapi.Service, registryRouter router.AppRouter, usageSender usage.Sender, + lfsCtrl *lfs.Controller, ) *Router { routers := make([]Interface, 4) @@ -120,6 +122,7 @@ func ProvideRouter( authenticator, repoCtrl, usageSender, + lfsCtrl, ) routers[0] = NewGitRouter(gitHandler, gitRoutingHost) routers[1] = router.NewRegistryRouter(registryRouter) diff --git a/app/services/remoteauth/user_jwt_provider.go b/app/services/remoteauth/user_jwt_provider.go new file mode 100644 index 000000000..e5a3d404d --- /dev/null +++ b/app/services/remoteauth/user_jwt_provider.go @@ -0,0 +1,67 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remoteauth + +import ( + "context" + "fmt" + + "github.com/harness/gitness/app/store" + "github.com/harness/gitness/app/token" + "github.com/harness/gitness/types/enum" +) + +type Service interface { + // GenerateToken generates a jwt for the given principle to access the resource (for git-lfs-authenticate response) + GenerateToken( + ctx context.Context, + principalID int64, + principalType enum.PrincipalType, + resource string, + ) (string, error) +} + +func NewService(tokenStore store.TokenStore, principalStore store.PrincipalStore) LocalService { + return LocalService{ + tokenStore: tokenStore, + principalStore: principalStore, + } +} + +type LocalService struct { + tokenStore store.TokenStore + principalStore store.PrincipalStore +} + +func (s LocalService) GenerateToken( + ctx context.Context, + principalID int64, + _ enum.PrincipalType, + _ string, +) (string, error) { + identifier := token.GenerateIdentifier("remoteAuth") + + principal, err := s.principalStore.Find(ctx, principalID) + if err != nil { + return "", fmt.Errorf("failed to find principal %d: %w", principalID, err) + } + + _, jwt, err := token.CreateRemoteAuthToken(ctx, s.tokenStore, principal, identifier) + if err != nil { + return "", fmt.Errorf("failed to create a remote auth token: %w", err) + } + + return jwt, nil +} diff --git a/app/services/remoteauth/wire.go b/app/services/remoteauth/wire.go new file mode 100644 index 000000000..8310df515 --- /dev/null +++ b/app/services/remoteauth/wire.go @@ -0,0 +1,32 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remoteauth + +import ( + "github.com/harness/gitness/app/store" + + "github.com/google/wire" +) + +var WireSet = wire.NewSet( + ProvideRemoteAuth, +) + +func ProvideRemoteAuth( + tokenStore store.TokenStore, + principalStore store.PrincipalStore, +) Service { + return NewService(tokenStore, principalStore) +} diff --git a/app/store/database.go b/app/store/database.go index 1091b0663..1c68b9e9e 100644 --- a/app/store/database.go +++ b/app/store/database.go @@ -1299,6 +1299,15 @@ type ( ) (map[int64][]*types.LabelPullReqAssignmentInfo, error) } + LFSObjectStore interface { + // Find finds an LFS object with a specified oid and repo-id. + Find(ctx context.Context, repoID int64, oid string) (*types.LFSObject, error) + // FindMany finds LFS objects for a specified repo. + FindMany(ctx context.Context, repoID int64, oids []string) ([]*types.LFSObject, error) + // Create creates an LFS object. + Create(ctx context.Context, lfsObject *types.LFSObject) error + } + InfraProviderTemplateStore interface { FindByIdentifier(ctx context.Context, spaceID int64, identifier string) (*types.InfraProviderTemplate, error) Find(ctx context.Context, id int64) (*types.InfraProviderTemplate, error) diff --git a/app/store/database/lfs_objects.go b/app/store/database/lfs_objects.go new file mode 100644 index 000000000..075bc211b --- /dev/null +++ b/app/store/database/lfs_objects.go @@ -0,0 +1,169 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "context" + "fmt" + + "github.com/harness/gitness/app/store" + "github.com/harness/gitness/store/database" + "github.com/harness/gitness/store/database/dbtx" + "github.com/harness/gitness/types" + + "github.com/Masterminds/squirrel" + "github.com/jmoiron/sqlx" +) + +var _ store.LFSObjectStore = (*LFSObjectStore)(nil) + +func NewLFSObjectStore(db *sqlx.DB) *LFSObjectStore { + return &LFSObjectStore{ + db: db, + } +} + +type LFSObjectStore struct { + db *sqlx.DB +} + +type lfsObject struct { + ID int64 `db:"lfs_object_id"` + OID string `db:"lfs_object_oid"` + Size int64 `db:"lfs_object_size"` + Created int64 `db:"lfs_object_created"` + CreatedBy int64 `db:"lfs_object_created_by"` + RepoID int64 `db:"lfs_object_repo_id"` +} + +const ( + lfsObjectColumns = ` + lfs_object_id + ,lfs_object_oid + ,lfs_object_size + ,lfs_object_created + ,lfs_object_created_by + ,lfs_object_repo_id` +) + +func (s *LFSObjectStore) Find( + ctx context.Context, + repoID int64, + oid string, +) (*types.LFSObject, error) { + stmt := database.Builder. + Select(lfsObjectColumns). + From("lfs_objects"). + Where("lfs_object_repo_id = ? AND lfs_object_oid = ?", repoID, oid) + + sql, args, err := stmt.ToSql() + if err != nil { + return nil, fmt.Errorf("failed to convert query to sql: %w", err) + } + + db := dbtx.GetAccessor(ctx, s.db) + + dst := &lfsObject{} + if err := db.GetContext(ctx, dst, sql, args...); err != nil { + return nil, database.ProcessSQLErrorf(ctx, err, "Select query failed") + } + + return mapLFSObject(dst), nil +} + +func (s *LFSObjectStore) FindMany( + ctx context.Context, + repoID int64, + oids []string, +) ([]*types.LFSObject, error) { + stmt := database.Builder. + Select(lfsObjectColumns). + From("lfs_objects"). + Where("lfs_object_repo_id = ?", repoID). + Where(squirrel.Eq{"lfs_object_oid": oids}) + + sql, args, err := stmt.ToSql() + if err != nil { + return nil, fmt.Errorf("failed to convert query to sql: %w", err) + } + db := dbtx.GetAccessor(ctx, s.db) + + var dst []*lfsObject + if err := db.SelectContext(ctx, &dst, sql, args...); err != nil { + return nil, database.ProcessSQLErrorf(ctx, err, "Select query failed") + } + + return mapLFSObjects(dst), nil +} + +func (s *LFSObjectStore) Create(ctx context.Context, obj *types.LFSObject) error { + const sqlQuery = ` + INSERT INTO lfs_objects ( + lfs_object_oid + ,lfs_object_size + ,lfs_object_created + ,lfs_object_created_by + ,lfs_object_repo_id + ) VALUES ( + :lfs_object_oid + ,:lfs_object_size + ,:lfs_object_created + ,:lfs_object_created_by + ,:lfs_object_repo_id + ) RETURNING lfs_object_id` + + db := dbtx.GetAccessor(ctx, s.db) + query, args, err := db.BindNamed(sqlQuery, mapInternalLFSObject(obj)) + if err != nil { + return database.ProcessSQLErrorf(ctx, err, "Failed to bind query") + } + + if err = db.QueryRowContext(ctx, query, args...).Scan(&obj.ID); err != nil { + return database.ProcessSQLErrorf(ctx, err, "Failed to create LFS object") + } + + return nil +} + +func mapInternalLFSObject(obj *types.LFSObject) *lfsObject { + return &lfsObject{ + ID: obj.ID, + OID: obj.OID, + Size: obj.Size, + Created: obj.Created, + CreatedBy: obj.CreatedBy, + RepoID: obj.RepoID, + } +} + +func mapLFSObject(obj *lfsObject) *types.LFSObject { + return &types.LFSObject{ + ID: obj.ID, + OID: obj.OID, + Size: obj.Size, + Created: obj.Created, + CreatedBy: obj.CreatedBy, + RepoID: obj.RepoID, + } +} + +func mapLFSObjects(objs []*lfsObject) []*types.LFSObject { + res := make([]*types.LFSObject, len(objs)) + for i := range objs { + res[i] = mapLFSObject(objs[i]) + } + + return res +} diff --git a/app/store/database/migrate/postgres/0103_create_table_lfs_objects.down.sql b/app/store/database/migrate/postgres/0103_create_table_lfs_objects.down.sql new file mode 100644 index 000000000..32b9544f6 --- /dev/null +++ b/app/store/database/migrate/postgres/0103_create_table_lfs_objects.down.sql @@ -0,0 +1,3 @@ +DROP INDEX lfs_objects_oid; + +DROP TABLE IF EXISTS lfs_objects; \ No newline at end of file diff --git a/app/store/database/migrate/postgres/0103_create_table_lfs_objects.up.sql b/app/store/database/migrate/postgres/0103_create_table_lfs_objects.up.sql new file mode 100644 index 000000000..23f7a45de --- /dev/null +++ b/app/store/database/migrate/postgres/0103_create_table_lfs_objects.up.sql @@ -0,0 +1,19 @@ +CREATE TABLE lfs_objects ( + lfs_object_id SERIAL PRIMARY KEY + ,lfs_object_oid TEXT NOT NULL + ,lfs_object_size BIGINT NOT NULL + ,lfs_object_created BIGINT NOT NULL + ,lfs_object_created_by INTEGER NOT NULL + ,lfs_object_repo_id INTEGER + ,CONSTRAINT fk_lfs_object_repo_id FOREIGN KEY (lfs_object_repo_id) + REFERENCES repositories (repo_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE SET NULL + ,CONSTRAINT fk_lfs_object_created_by FOREIGN KEY (lfs_object_created_by) + REFERENCES principals (principal_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE NO ACTION +); + +CREATE UNIQUE INDEX lfs_objects_oid + ON lfs_objects(lfs_object_repo_id, lfs_object_oid); \ No newline at end of file diff --git a/app/store/database/migrate/sqlite/0103_create_table_lfs_objects.down.sql b/app/store/database/migrate/sqlite/0103_create_table_lfs_objects.down.sql new file mode 100644 index 000000000..32b9544f6 --- /dev/null +++ b/app/store/database/migrate/sqlite/0103_create_table_lfs_objects.down.sql @@ -0,0 +1,3 @@ +DROP INDEX lfs_objects_oid; + +DROP TABLE IF EXISTS lfs_objects; \ No newline at end of file diff --git a/app/store/database/migrate/sqlite/0103_create_table_lfs_objects.up.sql b/app/store/database/migrate/sqlite/0103_create_table_lfs_objects.up.sql new file mode 100644 index 000000000..a369ffe14 --- /dev/null +++ b/app/store/database/migrate/sqlite/0103_create_table_lfs_objects.up.sql @@ -0,0 +1,19 @@ +CREATE TABLE lfs_objects ( + lfs_object_id INTEGER PRIMARY KEY AUTOINCREMENT + ,lfs_object_oid TEXT NOT NULL + ,lfs_object_size BIGINT NOT NULL + ,lfs_object_created BIGINT NOT NULL + ,lfs_object_created_by INTEGER NOT NULL + ,lfs_object_repo_id INTEGER + ,CONSTRAINT fk_lfs_object_repo_id FOREIGN KEY (lfs_object_repo_id) + REFERENCES repositories (repo_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE SET NULL + ,CONSTRAINT fk_lfs_object_created_by FOREIGN KEY (lfs_object_created_by) + REFERENCES principals (principal_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE NO ACTION +); + +CREATE UNIQUE INDEX lfs_objects_oid +ON lfs_objects(lfs_object_repo_id, lfs_object_oid); \ No newline at end of file diff --git a/app/store/database/wire.go b/app/store/database/wire.go index 153b751a2..3f7b30815 100644 --- a/app/store/database/wire.go +++ b/app/store/database/wire.go @@ -70,6 +70,7 @@ var WireSet = wire.NewSet( ProvideLabelStore, ProvideLabelValueStore, ProvidePullReqLabelStore, + ProvideLFSObjectStore, ProvideInfraProviderTemplateStore, ProvideInfraProvisionedStore, ProvideUsageMetricStore, @@ -336,6 +337,11 @@ func ProvidePullReqLabelStore(db *sqlx.DB) store.PullReqLabelAssignmentStore { return NewPullReqLabelStore(db) } +// ProvideLFSObjectStore provides an lfs object store. +func ProvideLFSObjectStore(db *sqlx.DB) store.LFSObjectStore { + return NewLFSObjectStore(db) +} + // ProvideInfraProviderTemplateStore provides a infraprovider template store. func ProvideInfraProviderTemplateStore(db *sqlx.DB) store.InfraProviderTemplateStore { return NewInfraProviderTemplateStore(db) diff --git a/app/token/token.go b/app/token/token.go index e280e84aa..fb42dbd13 100644 --- a/app/token/token.go +++ b/app/token/token.go @@ -17,6 +17,7 @@ package token import ( "context" "fmt" + "math/rand/v2" "time" "github.com/harness/gitness/app/jwt" @@ -32,6 +33,7 @@ const ( // NOTE: Users can list / delete session tokens via rest API if they want to cleanup earlier. userSessionTokenLifeTime time.Duration = 30 * 24 * time.Hour // 30 days. sessionTokenWithAccessPermissionsLifeTime time.Duration = 24 * time.Hour // 24 hours. + RemoteAuthTokenLifeTime time.Duration = 15 * time.Minute // 15 minutes. ) func CreateUserWithAccessPermissions( @@ -102,6 +104,29 @@ func CreateSAT( ) } +func CreateRemoteAuthToken( + ctx context.Context, + tokenStore store.TokenStore, + principal *types.Principal, + identifier string, +) (*types.Token, string, error) { + return create( + ctx, + tokenStore, + enum.TokenTypeRemoteAuth, + principal, + principal, + identifier, + ptr.Duration(RemoteAuthTokenLifeTime), + ) +} + +func GenerateIdentifier(prefix string) string { + //nolint:gosec // math/rand is sufficient for this use case + r := rand.IntN(0x10000) + return fmt.Sprintf("%s-%08x-%04x", prefix, time.Now().Unix(), r) +} + func create( ctx context.Context, tokenStore store.TokenStore, diff --git a/blob/filesystem.go b/blob/filesystem.go index 0fb4e0638..6505018d2 100644 --- a/blob/filesystem.go +++ b/blob/filesystem.go @@ -22,6 +22,7 @@ import ( "io/fs" "os" "path" + "time" "github.com/rs/zerolog/log" ) @@ -81,7 +82,7 @@ func (c FileSystemStore) Upload(ctx context.Context, return nil } -func (c FileSystemStore) GetSignedURL(_ context.Context, _ string) (string, error) { +func (c FileSystemStore) GetSignedURL(context.Context, string, time.Time) (string, error) { return "", ErrNotSupported } diff --git a/blob/gcs.go b/blob/gcs.go index fd4e5f03f..dfb502540 100644 --- a/blob/gcs.go +++ b/blob/gcs.go @@ -16,6 +16,7 @@ package blob import ( "context" + "errors" "fmt" "io" "net/http" @@ -65,7 +66,7 @@ func NewGCSStore(ctx context.Context, cfg Config) (Store, error) { } func (c *GCSStore) Upload(ctx context.Context, file io.Reader, filePath string) error { - gcsClient, err := c.getLatestClient(ctx) + gcsClient, err := c.getClient(ctx) if err != nil { return fmt.Errorf("failed to retrieve latest client: %w", err) } @@ -93,8 +94,8 @@ func (c *GCSStore) Upload(ctx context.Context, file io.Reader, filePath string) return nil } -func (c *GCSStore) GetSignedURL(ctx context.Context, filePath string) (string, error) { - gcsClient, err := c.getLatestClient(ctx) +func (c *GCSStore) GetSignedURL(ctx context.Context, filePath string, expire time.Time) (string, error) { + gcsClient, err := c.getClient(ctx) if err != nil { return "", fmt.Errorf("failed to retrieve latest client: %w", err) } @@ -102,7 +103,7 @@ func (c *GCSStore) GetSignedURL(ctx context.Context, filePath string) (string, e bkt := gcsClient.Bucket(c.config.Bucket) signedURL, err := bkt.SignedURL(filePath, &storage.SignedURLOptions{ Method: http.MethodGet, - Expires: time.Now().Add(1 * time.Hour), + Expires: expire, }) if err != nil { return "", fmt.Errorf("failed to create signed URL for file %q: %w", filePath, err) @@ -110,8 +111,22 @@ func (c *GCSStore) GetSignedURL(ctx context.Context, filePath string) (string, e return signedURL, nil } -func (c *GCSStore) Download(_ context.Context, _ string) (io.ReadCloser, error) { - return nil, fmt.Errorf("not implemented") +func (c *GCSStore) Download(ctx context.Context, filePath string) (io.ReadCloser, error) { + gcsClient, err := c.getClient(ctx) + if err != nil { + return nil, fmt.Errorf("failed to retrieve latest client: %w", err) + } + + bkt := gcsClient.Bucket(c.config.Bucket) + rc, err := bkt.Object(filePath).NewReader(ctx) + if err != nil { + if errors.Is(err, storage.ErrObjectNotExist) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("failed to create reader for file %q in bucket %q: %w", filePath, c.config.Bucket, err) + } + + return rc, nil } func createNewImpersonatedClient(ctx context.Context, cfg Config) (*storage.Client, error) { @@ -138,7 +153,7 @@ func createNewImpersonatedClient(ctx context.Context, cfg Config) (*storage.Clie return client, nil } -func (c *GCSStore) getLatestClient(ctx context.Context) (*storage.Client, error) { +func (c *GCSStore) getClient(ctx context.Context) (*storage.Client, error) { err := c.checkAndRefreshToken(ctx) if err != nil { return nil, fmt.Errorf("failed to refresh token: %w", err) diff --git a/blob/interface.go b/blob/interface.go index f3ab4f592..e688ba6db 100644 --- a/blob/interface.go +++ b/blob/interface.go @@ -18,6 +18,7 @@ import ( "context" "errors" "io" + "time" ) var ( @@ -30,7 +31,7 @@ type Store interface { Upload(ctx context.Context, file io.Reader, filePath string) error // GetSignedURL returns the URL for a file in the blob store. - GetSignedURL(ctx context.Context, filePath string) (string, error) + GetSignedURL(ctx context.Context, filePath string, expire time.Time) (string, error) // Download returns a reader for a file in the blob store. Download(ctx context.Context, filePath string) (io.ReadCloser, error) diff --git a/cmd/gitness/wire.go b/cmd/gitness/wire.go index e739e0568..d687ba5ca 100644 --- a/cmd/gitness/wire.go +++ b/cmd/gitness/wire.go @@ -27,6 +27,7 @@ import ( gitspaceCtrl "github.com/harness/gitness/app/api/controller/gitspace" infraproviderCtrl "github.com/harness/gitness/app/api/controller/infraprovider" controllerkeywordsearch "github.com/harness/gitness/app/api/controller/keywordsearch" + "github.com/harness/gitness/app/api/controller/lfs" "github.com/harness/gitness/app/api/controller/limiter" controllerlogs "github.com/harness/gitness/app/api/controller/logs" "github.com/harness/gitness/app/api/controller/migrate" @@ -99,6 +100,7 @@ import ( "github.com/harness/gitness/app/services/publickey" pullreqservice "github.com/harness/gitness/app/services/pullreq" "github.com/harness/gitness/app/services/refcache" + "github.com/harness/gitness/app/services/remoteauth" reposervice "github.com/harness/gitness/app/services/repo" "github.com/harness/gitness/app/services/rules" secretservice "github.com/harness/gitness/app/services/secret" @@ -253,6 +255,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e audit.WireSet, ssh.WireSet, publickey.WireSet, + remoteauth.WireSet, migrate.WireSet, scm.WireSet, platformconnector.WireSet, @@ -274,6 +277,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e docker.ProvideReporter, secretservice.WireSet, runarg.WireSet, + lfs.WireSet, usage.WireSet, registryevents.WireSet, registrywebhooks.WireSet, diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index 25b0c520f..b0ba757f2 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -16,6 +16,7 @@ import ( gitspace2 "github.com/harness/gitness/app/api/controller/gitspace" infraprovider3 "github.com/harness/gitness/app/api/controller/infraprovider" keywordsearch2 "github.com/harness/gitness/app/api/controller/keywordsearch" + "github.com/harness/gitness/app/api/controller/lfs" "github.com/harness/gitness/app/api/controller/limiter" logs2 "github.com/harness/gitness/app/api/controller/logs" migrate2 "github.com/harness/gitness/app/api/controller/migrate" @@ -90,6 +91,7 @@ import ( "github.com/harness/gitness/app/services/publickey" "github.com/harness/gitness/app/services/pullreq" "github.com/harness/gitness/app/services/refcache" + "github.com/harness/gitness/app/services/remoteauth" repo2 "github.com/harness/gitness/app/services/repo" "github.com/harness/gitness/app/services/rules" secret3 "github.com/harness/gitness/app/services/secret" @@ -409,7 +411,8 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - githookController := githook.ProvideController(authorizer, principalStore, repoStore, repoFinder, reporter5, reporter, gitInterface, pullReqStore, provider, protectionManager, clientFactory, resourceLimiter, settingsService, preReceiveExtender, updateExtender, postReceiveExtender, streamer) + lfsObjectStore := database.ProvideLFSObjectStore(db) + githookController := githook.ProvideController(authorizer, principalStore, repoStore, repoFinder, reporter5, reporter, gitInterface, pullReqStore, provider, protectionManager, clientFactory, resourceLimiter, settingsService, preReceiveExtender, updateExtender, postReceiveExtender, streamer, lfsObjectStore) serviceaccountController := serviceaccount.NewController(principalUID, authorizer, principalStore, spaceStore, repoStore, tokenStore) principalController := principal.ProvideController(principalStore, authorizer) usergroupController := usergroup2.ProvideController(userGroupStore, spaceStore, authorizer, searchService) @@ -504,10 +507,12 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro handler4 := router.PackageHandlerProvider(packagesHandler, mavenHandler, genericHandler, pypiHandler) appRouter := router.AppRouterProvider(registryOCIHandler, apiHandler, handler2, handler3, handler4) sender := usage.ProvideMediator(ctx, config, spaceFinder, usageMetricStore) - routerRouter := router2.ProvideRouter(ctx, config, authenticator, repoController, reposettingsController, executionController, logsController, spaceController, pipelineController, secretController, triggerController, connectorController, templateController, pluginController, pullreqController, webhookController, githookController, gitInterface, serviceaccountController, controller, principalController, usergroupController, checkController, systemController, uploadController, keywordsearchController, infraproviderController, gitspaceController, migrateController, provider, openapiService, appRouter, sender) + remoteauthService := remoteauth.ProvideRemoteAuth(tokenStore, principalStore) + lfsController := lfs.ProvideController(authorizer, repoFinder, principalStore, lfsObjectStore, blobStore, remoteauthService, provider) + routerRouter := router2.ProvideRouter(ctx, config, authenticator, repoController, reposettingsController, executionController, logsController, spaceController, pipelineController, secretController, triggerController, connectorController, templateController, pluginController, pullreqController, webhookController, githookController, gitInterface, serviceaccountController, controller, principalController, usergroupController, checkController, systemController, uploadController, keywordsearchController, infraproviderController, gitspaceController, migrateController, provider, openapiService, appRouter, sender, lfsController) serverServer := server2.ProvideServer(config, routerRouter) publickeyService := publickey.ProvidePublicKey(publicKeyStore, principalInfoCache) - sshServer := ssh.ProvideServer(config, publickeyService, repoController) + sshServer := ssh.ProvideServer(config, publickeyService, repoController, lfsController) executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, streamer, fileService, converterService, logStore, logStream, checkStore, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore, publicaccessService, reporter3) client := manager.ProvideExecutionClient(executionManager, provider, config) resolverManager := resolver.ProvideResolver(config, pluginStore, templateStore, executionStore, repoStore) diff --git a/git/blob.go b/git/blob.go index 60d999e0f..0b307b780 100644 --- a/git/blob.go +++ b/git/blob.go @@ -16,12 +16,20 @@ package git import ( "context" + "fmt" "io" + "github.com/harness/gitness/errors" "github.com/harness/gitness/git/api" + "github.com/harness/gitness/git/parser" "github.com/harness/gitness/git/sha" ) +// lfsPointerMaxSize is the maximum size for an LFS pointer file. +// This is used to identify blobs that are too large to be valid LFS pointers. +// lfs-pointer specification ref: https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md#the-pointer +const lfsPointerMaxSize = 200 + type GetBlobParams struct { ReadParams SHA string @@ -64,3 +72,87 @@ func (s *Service) GetBlob(ctx context.Context, params *GetBlobParams) (*GetBlobO Content: reader.Content, }, nil } + +type ListLFSPointersParams struct { + ReadParams +} + +type ListLFSPointersOutput struct { + LFSInfos []LFSInfo +} + +type LFSInfo struct { + OID string `json:"oid"` + SHA sha.SHA `json:"sha"` +} + +func (s *Service) ListLFSPointers( + ctx context.Context, + params *ListLFSPointersParams, +) (*ListLFSPointersOutput, error) { + if params.RepoUID == "" { + return nil, api.ErrRepositoryPathEmpty + } + + repoPath := getFullPathForRepo(s.reposRoot, params.RepoUID) + + var lfsInfos []LFSInfo + var candidateObjects []parser.BatchCheckObject + // first get the sha of the objects that could be lfs pointers + for _, gitObjDir := range params.AlternateObjectDirs { + objects, err := catFileBatchCheckAllObjects(ctx, repoPath, gitObjDir) + if err != nil { + return nil, err + } + + for _, obj := range objects { + if obj.Type == string(TreeNodeTypeBlob) && obj.Size <= lfsPointerMaxSize { + candidateObjects = append(candidateObjects, obj) + } + } + } + + if len(candidateObjects) == 0 { + return &ListLFSPointersOutput{LFSInfos: lfsInfos}, nil + } + + // check the short-listed objects for lfs-pointers content + stdIn, stdOut, cancel := api.CatFileBatch(ctx, repoPath, params.AlternateObjectDirs) + defer cancel() + + for _, obj := range candidateObjects { + line := obj.SHA.String() + "\n" + + _, err := stdIn.Write([]byte(line)) + if err != nil { + return nil, fmt.Errorf("failed to write blob sha to git stdin: %w", err) + } + + // first line is always the object type, sha, and size + _, err = stdOut.ReadString('\n') + if err != nil { + return nil, fmt.Errorf("failed to read the git cat-file output: %w", err) + } + + content, err := io.ReadAll(io.LimitReader(stdOut, obj.Size)) + if err != nil { + return nil, fmt.Errorf("failed to read the git cat-file output: %w", err) + } + + oid, err := parser.GetLFSOID(content) + if err != nil && !errors.Is(err, parser.ErrInvalidLFSPointer) { + return nil, fmt.Errorf("failed to scan git cat-file output for %s: %w", obj.SHA, err) + } + if err == nil { + lfsInfos = append(lfsInfos, LFSInfo{OID: oid, SHA: obj.SHA}) + } + + // skip the trailing new line + _, err = stdOut.ReadString('\n') + if err != nil { + return nil, fmt.Errorf("failed to read trailing newline after object: %w", err) + } + } + + return &ListLFSPointersOutput{LFSInfos: lfsInfos}, nil +} diff --git a/git/interface.go b/git/interface.go index 7626507ee..b9a1498ba 100644 --- a/git/interface.go +++ b/git/interface.go @@ -39,6 +39,7 @@ type Interface interface { GetRef(ctx context.Context, params GetRefParams) (GetRefResponse, error) PathsDetails(ctx context.Context, params PathsDetailsParams) (PathsDetailsOutput, error) Summary(ctx context.Context, params SummaryParams) (SummaryOutput, error) + ListLFSPointers(ctx context.Context, params *ListLFSPointersParams) (*ListLFSPointersOutput, error) // GetRepositorySize calculates the size of a repo in KiB. GetRepositorySize(ctx context.Context, params *GetRepositorySizeParams) (*GetRepositorySizeOutput, error) diff --git a/git/parser/lfs_pointers.go b/git/parser/lfs_pointers.go new file mode 100644 index 000000000..c1c437be8 --- /dev/null +++ b/git/parser/lfs_pointers.go @@ -0,0 +1,47 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "bytes" + "errors" + "regexp" +) + +const lfsPointerVersionPrefix = "version https://git-lfs.github.com/spec" + +var ( + regexLFSOID = regexp.MustCompile(`(?m)^oid sha256:([a-f0-9]{64})$`) + regexLFSSize = regexp.MustCompile(`(?m)^size [0-9]+$`) + + ErrInvalidLFSPointer = errors.New("invalid lfs pointer") +) + +func GetLFSOID(content []byte) (string, error) { + if !bytes.HasPrefix(content, []byte(lfsPointerVersionPrefix)) { + return "", ErrInvalidLFSPointer + } + + oidMatch := regexLFSOID.FindSubmatch(content) + if oidMatch == nil { + return "", ErrInvalidLFSPointer + } + + if !regexLFSSize.Match(content) { + return "", ErrInvalidLFSPointer + } + + return string(oidMatch[1]), nil +} diff --git a/go.mod b/go.mod index a43fdba20..553cf759a 100644 --- a/go.mod +++ b/go.mod @@ -152,6 +152,7 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect diff --git a/go.sum b/go.sum index f7231b9c7..a28871122 100644 --- a/go.sum +++ b/go.sum @@ -691,8 +691,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sercand/kuberesolver/v5 v5.1.1 h1:CYH+d67G0sGBj7q5wLK61yzqJJ8gLLC8aeprPTHb6yY= github.com/sercand/kuberesolver/v5 v5.1.1/go.mod h1:Fs1KbKhVRnB2aDWN12NjKCB+RgYMWZJ294T3BtmVCpQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -1053,6 +1053,7 @@ gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gG gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/ssh/server.go b/ssh/server.go index 735a68a8c..7e8391ec4 100644 --- a/ssh/server.go +++ b/ssh/server.go @@ -19,6 +19,7 @@ import ( "crypto/rand" "crypto/rsa" "crypto/x509" + "encoding/json" "encoding/pem" "fmt" "io" @@ -30,6 +31,7 @@ import ( "strings" "time" + "github.com/harness/gitness/app/api/controller/lfs" "github.com/harness/gitness/app/api/controller/repo" "github.com/harness/gitness/app/api/request" "github.com/harness/gitness/app/auth" @@ -53,6 +55,8 @@ var ( allowedCommands = []string{ "git-upload-pack", "git-receive-pack", + "git-lfs-authenticate", + "git-lfs-transfer", } defaultCiphers = []string{ "chacha20-poly1305@openssh.com", @@ -97,6 +101,7 @@ type Server struct { Verifier publickey.Service RepoCtrl *repo.Controller + LFSCtrl *lfs.Controller ServerKeyPath string } @@ -225,12 +230,70 @@ func (s *Server) sessionHandler(session ssh.Session) { } // first part is git service pack command: git-upload-pack, git-receive-pack + // of git-lfs client command: git-lfs-authenticate, git-lfs-transfer gitCommand := parts[0] if !slices.Contains(allowedCommands, gitCommand) { _, _ = fmt.Fprintf(session.Stderr(), "command not supported: %q\n", command) return } + // handle git-lfs commands + //nolint:nestif + if strings.HasPrefix(gitCommand, "git-lfs-") { + gitLFSservice, err := enum.ParseGitLFSServiceType(gitCommand) + if err != nil { + _, _ = fmt.Fprintf(session.Stderr(), "failed to parse git-lfs service command: %q\n", gitCommand) + return + } + repoRef := getRepoRefFromCommand(parts[1]) + + // when git-lfs-transfer not supported, git-lfs client uses git-lfs-authenticate + // to gain a token from server and continue with http transfer APIs + if gitLFSservice == enum.GitLFSServiceTypeTransfer { + _, _ = fmt.Fprint(session.Stderr(), "git-lfs-transfer is not supported.") + return + } + + // handling git-lfs-authenticate + principal := types.Principal{ + ID: principal.ID, + UID: principal.UID, + Email: principal.Email, + Type: principal.Type, + DisplayName: principal.DisplayName, + Created: principal.Created, + Updated: principal.Updated, + } + ctx, cancel := context.WithCancel(session.Context()) + defer cancel() + + response, err := s.LFSCtrl.Authenticate( + ctx, + &auth.Session{ + Principal: principal, + }, + repoRef) + if err != nil { + log.Error().Err(err).Msg("git lfs authenticate failed") + writeErrorToSession(session, err.Error()) + return + } + + responseJSON, err := json.Marshal(response) + if err != nil { + log.Error().Err(err).Msg("failed to marshal lfs authenticate response") + writeErrorToSession(session, err.Error()) + return + } + + if _, err := session.Write(responseJSON); err != nil { + log.Error().Err(err).Msg("failed to write response of git lfs authenticate") + writeErrorToSession(session, err.Error()) + } + return + } + + // handle git service pack commands gitServicePack := strings.TrimPrefix(gitCommand, "git-") service, err := enum.ParseGitServiceType(gitServicePack) if err != nil { @@ -241,12 +304,7 @@ func (s *Server) sessionHandler(session ssh.Session) { // git command args gitArgs := parts[1:] - // first git service pack cmd arg is path: 'space/repository.git' so we need to remove - // single quotes. - repoRef := strings.Trim(gitArgs[0], "'") - // remove .git suffix - repoRef = strings.TrimSuffix(repoRef, ".git") - + repoRef := getRepoRefFromCommand(gitArgs[0]) gitProtocol := "" for _, key := range session.Environ() { if strings.HasPrefix(key, "GIT_PROTOCOL=") { @@ -432,3 +490,19 @@ func GenerateKeyPair(keyPath string) error { } return nil } + +func getRepoRefFromCommand(gitArg string) string { + // first git service pack cmd arg is path: 'space/repository.git' so we need to remove + // single quotes. + repoRef := strings.Trim(gitArg, "'") + // remove .git suffix + repoRef = strings.TrimSuffix(repoRef, ".git") + + return repoRef +} + +func writeErrorToSession(session ssh.Session, message string) { + if _, err := io.Copy(session.Stderr(), strings.NewReader(message+"\n")); err != nil { + log.Printf("error writing to session stderr: %v", err) + } +} diff --git a/ssh/wire.go b/ssh/wire.go index 667e253e6..02ba27879 100644 --- a/ssh/wire.go +++ b/ssh/wire.go @@ -15,6 +15,7 @@ package ssh import ( + "github.com/harness/gitness/app/api/controller/lfs" "github.com/harness/gitness/app/api/controller/repo" "github.com/harness/gitness/app/services/publickey" "github.com/harness/gitness/types" @@ -30,6 +31,7 @@ func ProvideServer( config *types.Config, verifier publickey.Service, repoctrl *repo.Controller, + lfsCtrl *lfs.Controller, ) *Server { return &Server{ Host: config.SSH.Host, @@ -44,6 +46,7 @@ func ProvideServer( KeepAliveInterval: config.SSH.KeepAliveInterval, Verifier: verifier, RepoCtrl: repoctrl, + LFSCtrl: lfsCtrl, ServerKeyPath: config.SSH.ServerKeyPath, } } diff --git a/types/enum/git_lfs.go b/types/enum/git_lfs.go new file mode 100644 index 000000000..094cb8206 --- /dev/null +++ b/types/enum/git_lfs.go @@ -0,0 +1,78 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enum + +import ( + "fmt" + "strings" +) + +type GitLFSTransferType string + +const ( + GitLFSTransferTypeBasic GitLFSTransferType = "basic" + GitLFSTransferTypeSSH GitLFSTransferType = "ssh" + // TODO GitLFSTransferTypeMultipart +) + +func ParseGitLFSTransferType(s string) (GitLFSTransferType, error) { + switch strings.ToLower(s) { + case string(GitLFSTransferTypeBasic): + return GitLFSTransferTypeBasic, nil + case string(GitLFSTransferTypeSSH): + return GitLFSTransferTypeSSH, nil + default: + return "", fmt.Errorf("unknown git-lfs transfer type provided: %q", s) + } +} + +type GitLFSOperationType string + +const ( + GitLFSOperationTypeDownload GitLFSOperationType = "download" + GitLFSOperationTypeUpload GitLFSOperationType = "upload" +) + +func ParseGitLFSOperationType(s string) (GitLFSOperationType, error) { + switch strings.ToLower(s) { + case string(GitLFSOperationTypeDownload): + return GitLFSOperationTypeDownload, nil + case string(GitLFSOperationTypeUpload): + return GitLFSOperationTypeUpload, nil + default: + return "", fmt.Errorf("unknown git-lfs operation type provided: %q", s) + } +} + +// GitLFSServiceType represents the different types of services git-lfs client sends over ssh. +type GitLFSServiceType string + +const ( + // GitLFSServiceTypeTransfer is sent by git lfs client for transfer LFS objects. + GitLFSServiceTypeTransfer GitLFSServiceType = "git-lfs-transfer" + // GitLFSServiceTypeAuthenticate is sent by git lfs client for authentication. + GitLFSServiceTypeAuthenticate GitLFSServiceType = "git-lfs-authenticate" +) + +func ParseGitLFSServiceType(s string) (GitLFSServiceType, error) { + switch strings.ToLower(s) { + case string(GitLFSServiceTypeTransfer): + return GitLFSServiceTypeTransfer, nil + case string(GitLFSServiceTypeAuthenticate): + return GitLFSServiceTypeAuthenticate, nil + default: + return "", fmt.Errorf("unknown git-lfs service type provided: %q", s) + } +} diff --git a/types/enum/token.go b/types/enum/token.go index 338f192bb..9e146c271 100644 --- a/types/enum/token.go +++ b/types/enum/token.go @@ -26,4 +26,7 @@ const ( // TokenTypeSAT is a service account access token. TokenTypeSAT TokenType = "sat" + + // TokenTypeRemoteAuth is the token returned during ssh git-lfs-authenticate. + TokenTypeRemoteAuth TokenType = "remoteAuth" ) diff --git a/types/lfs.go b/types/lfs.go new file mode 100644 index 000000000..ce64ca84b --- /dev/null +++ b/types/lfs.go @@ -0,0 +1,32 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +type LFSObject struct { + ID int64 `json:"id"` + OID string `json:"oid"` + Size int64 `json:"size"` + Created int64 `json:"created"` + CreatedBy int64 `json:"created_by"` + RepoID int64 `json:"repo_id"` +} + +type LFSLock struct { + ID int64 `json:"id"` + Path string `json:"path"` + Ref string `json:"ref"` + Created int64 `json:"created"` + RepoID int64 `json:"repo_id"` +}