diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index 84163467e..d408ab14a 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -124,14 +124,16 @@ import ( "github.com/harness/gitness/lock" "github.com/harness/gitness/pubsub" api2 "github.com/harness/gitness/registry/app/api" + python2 "github.com/harness/gitness/registry/app/api/controller/pkg/python" "github.com/harness/gitness/registry/app/api/router" events10 "github.com/harness/gitness/registry/app/events" "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/base" "github.com/harness/gitness/registry/app/pkg/docker" "github.com/harness/gitness/registry/app/pkg/filemanager" "github.com/harness/gitness/registry/app/pkg/generic" "github.com/harness/gitness/registry/app/pkg/maven" - "github.com/harness/gitness/registry/app/pkg/pypi" + "github.com/harness/gitness/registry/app/pkg/python" database2 "github.com/harness/gitness/registry/app/store/database" "github.com/harness/gitness/registry/gc" webhook3 "github.com/harness/gitness/registry/services/webhook" @@ -515,9 +517,12 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro genericHandler := api2.NewGenericHandlerProvider(spaceStore, genericController, tokenStore, controller, authenticator, provider, authorizer) handler3 := router.GenericHandlerProvider(genericHandler) packagesHandler := api2.NewPackageHandlerProvider(registryRepository, spaceStore, tokenStore, controller, authenticator, provider, authorizer) - pypiController := pypi.ControllerProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider) - pypiHandler := api2.NewPypiHandlerProvider(pypiController, packagesHandler) - handler4 := router.PackageHandlerProvider(packagesHandler, mavenHandler, genericHandler, pypiHandler) + localBase := base.LocalBaseProvider(registryRepository, fileManager, transactor, imageRepository, artifactRepository) + pythonLocalRegistry := python.LocalRegistryProvider(localBase, fileManager, upstreamProxyConfigRepository, transactor, registryRepository, imageRepository, artifactRepository, provider) + proxy := python.ProxyProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider) + pythonController := python2.ControllerProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider, pythonLocalRegistry, proxy) + pythonHandler := api2.NewPythonHandlerProvider(pythonController, packagesHandler) + handler4 := router.PackageHandlerProvider(packagesHandler, mavenHandler, genericHandler, pythonHandler) appRouter := router.AppRouterProvider(registryOCIHandler, apiHandler, handler2, handler3, handler4) sender := usage.ProvideMediator(ctx, config, spaceFinder, usageMetricStore) remoteauthService := remoteauth.ProvideRemoteAuth(tokenStore, principalStore) diff --git a/registry/app/api/controller/metadata/artifact_mapper.go b/registry/app/api/controller/metadata/artifact_mapper.go index 30bb1a1a3..dd8330719 100644 --- a/registry/app/api/controller/metadata/artifact_mapper.go +++ b/registry/app/api/controller/metadata/artifact_mapper.go @@ -23,7 +23,7 @@ import ( "github.com/harness/gitness/app/url" artifactapi "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" - "github.com/harness/gitness/registry/app/store/database" + "github.com/harness/gitness/registry/app/metadata" "github.com/harness/gitness/registry/types" "github.com/rs/zerolog/log" @@ -58,12 +58,12 @@ func GetRegistryArtifactMetadata(artifacts []types.ArtifactMetadata) []artifacta func GetMavenArtifactDetail( image *types.Image, artifact *types.Artifact, - metadata database.MavenMetadata, + mavenMetadata metadata.MavenMetadata, ) artifactapi.ArtifactDetail { createdAt := GetTimeInMs(artifact.CreatedAt) modifiedAt := GetTimeInMs(artifact.UpdatedAt) var size int64 - for _, file := range metadata.Files { + for _, file := range mavenMetadata.Files { size += file.Size } sizeVal := GetSize(size) @@ -229,7 +229,7 @@ func GetArtifactFilesMetadata( filePathPrefix := "/" + artifactName + "/" + version + "/" filename := strings.Replace(file.Path, filePathPrefix, "", 1) var downloadCommand string - if artifactapi.PackageTypeGENERIC == packageType { + if artifactapi.PackageTypeGENERIC == packageType || artifactapi.PackageTypePYTHON == packageType { downloadCommand = GetGenericArtifactFileDownloadCommand(registryURL, artifactName, version, filename) } else if artifactapi.PackageTypeMAVEN == packageType { artifactName = strings.ReplaceAll(artifactName, ".", "/") @@ -457,7 +457,7 @@ func GetHelmArtifactDetails( func GetGenericArtifactDetail( image *types.Image, artifact *types.Artifact, - metadata database.GenericMetadata, + metadata metadata.GenericMetadata, ) artifactapi.ArtifactDetail { createdAt := GetTimeInMs(artifact.CreatedAt) modifiedAt := GetTimeInMs(artifact.UpdatedAt) @@ -476,6 +476,27 @@ func GetGenericArtifactDetail( return *artifactDetail } +func GetPythonArtifactDetail( + image *types.Image, artifact *types.Artifact, + metadata map[string]interface{}, +) artifactapi.ArtifactDetail { + createdAt := GetTimeInMs(artifact.CreatedAt) + modifiedAt := GetTimeInMs(artifact.UpdatedAt) + artifactDetail := &artifactapi.ArtifactDetail{ + CreatedAt: &createdAt, + ModifiedAt: &modifiedAt, + Name: &image.Name, + Version: artifact.Version, + } + err := artifactDetail.FromPythonArtifactDetailConfig(artifactapi.PythonArtifactDetailConfig{ + Metadata: &metadata, + }) + if err != nil { + return artifactapi.ArtifactDetail{} + } + return *artifactDetail +} + func GetArtifactSummary(artifact types.ArtifactMetadata) *artifactapi.ArtifactSummaryResponseJSONResponse { createdAt := GetTimeInMs(artifact.CreatedAt) modifiedAt := GetTimeInMs(artifact.ModifiedAt) diff --git a/registry/app/api/controller/metadata/get_artifact_detail.go b/registry/app/api/controller/metadata/get_artifact_detail.go index aa4849903..7669be8cb 100644 --- a/registry/app/api/controller/metadata/get_artifact_detail.go +++ b/registry/app/api/controller/metadata/get_artifact_detail.go @@ -22,7 +22,7 @@ import ( apiauth "github.com/harness/gitness/app/api/auth" "github.com/harness/gitness/app/api/request" "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" - "github.com/harness/gitness/registry/app/store/database" + "github.com/harness/gitness/registry/app/metadata" "github.com/harness/gitness/types/enum" ) @@ -97,8 +97,10 @@ func (c *APIController) GetArtifactDetails( var artifactDetails artifact.ArtifactDetail - if artifact.PackageTypeMAVEN == registry.PackageType { - var metadata database.MavenMetadata + // FIXME: Arvind: Unify the metadata structure to avoid this type checking + switch registry.PackageType { + case artifact.PackageTypeMAVEN: + var metadata metadata.MavenMetadata err := json.Unmarshal(art.Metadata, &metadata) if err != nil { return artifact.GetArtifactDetails500JSONResponse{ @@ -108,8 +110,8 @@ func (c *APIController) GetArtifactDetails( }, nil } artifactDetails = GetMavenArtifactDetail(img, art, metadata) - } else if artifact.PackageTypeGENERIC == registry.PackageType { - var metadata database.GenericMetadata + case artifact.PackageTypeGENERIC: + var metadata metadata.GenericMetadata err := json.Unmarshal(art.Metadata, &metadata) if err != nil { return artifact.GetArtifactDetails500JSONResponse{ @@ -119,7 +121,27 @@ func (c *APIController) GetArtifactDetails( }, nil } artifactDetails = GetGenericArtifactDetail(img, art, metadata) + case artifact.PackageTypePYTHON: + var result map[string]interface{} + err := json.Unmarshal(art.Metadata, &result) + if err != nil { + return artifact.GetArtifactDetails500JSONResponse{ + InternalServerErrorJSONResponse: artifact.InternalServerErrorJSONResponse( + *GetErrorResponse(http.StatusInternalServerError, err.Error()), + ), + }, nil + } + artifactDetails = GetPythonArtifactDetail(img, art, result) + case artifact.PackageTypeDOCKER: + case artifact.PackageTypeHELM: + default: + return artifact.GetArtifactDetails400JSONResponse{ + BadRequestJSONResponse: artifact.BadRequestJSONResponse( + *GetErrorResponse(http.StatusBadRequest, "unsupported package type"), + ), + }, nil } + return artifact.GetArtifactDetails200JSONResponse{ ArtifactDetailResponseJSONResponse: artifact.ArtifactDetailResponseJSONResponse{ Data: artifactDetails, diff --git a/registry/app/api/controller/metadata/get_artifact_files.go b/registry/app/api/controller/metadata/get_artifact_files.go index 266fb102d..f87a11bb9 100644 --- a/registry/app/api/controller/metadata/get_artifact_files.go +++ b/registry/app/api/controller/metadata/get_artifact_files.go @@ -133,7 +133,7 @@ func (c *APIController) GetArtifactFiles( //nolint:exhaustive switch registry.PackageType { - case artifact.PackageTypeGENERIC, artifact.PackageTypeMAVEN: + case artifact.PackageTypeGENERIC, artifact.PackageTypeMAVEN, artifact.PackageTypePYTHON: return artifact.GetArtifactFiles200JSONResponse{ FileDetailResponseJSONResponse: *GetAllArtifactFilesResponse( fileMetadataList, count, reqInfo.pageNumber, reqInfo.limit, registryURL, img.Name, art.Version, diff --git a/registry/app/pkg/pypi/controller.go b/registry/app/api/controller/pkg/python/controller.go similarity index 72% rename from registry/app/pkg/pypi/controller.go rename to registry/app/api/controller/pkg/python/controller.go index b1f58852b..8c4a0db0f 100644 --- a/registry/app/pkg/pypi/controller.go +++ b/registry/app/api/controller/pkg/python/controller.go @@ -12,23 +12,34 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pypi +package python import ( "context" "mime/multipart" urlprovider "github.com/harness/gitness/app/url" - "github.com/harness/gitness/registry/app/dist_temp/errcode" - "github.com/harness/gitness/registry/app/pkg" - "github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/filemanager" - "github.com/harness/gitness/registry/app/storage" + "github.com/harness/gitness/registry/app/pkg/python" + pythontype "github.com/harness/gitness/registry/app/pkg/types/python" "github.com/harness/gitness/registry/app/store" "github.com/harness/gitness/store/database/dbtx" ) -// Controller handles PyPI package operations. +type Controller interface { + GetPackageMetadata(ctx context.Context, info pythontype.ArtifactInfo) (pythontype.PackageMetadata, error) + + UploadPackageFile( + ctx context.Context, + info pythontype.ArtifactInfo, + file multipart.File, + fileHeader *multipart.FileHeader, + ) *PutArtifactResponse + + DownloadPackageFile(ctx context.Context, info pythontype.ArtifactInfo) *GetArtifactResponse +} + +// Controller handles Python package operations. type controller struct { fileManager filemanager.FileManager proxyStore store.UpstreamProxyConfigRepository @@ -37,26 +48,11 @@ type controller struct { imageDao store.ImageRepository artifactDao store.ArtifactRepository urlProvider urlprovider.Provider + local python.LocalRegistry + proxy python.Proxy } -type Controller interface { - GetPackageMetadata(ctx context.Context, info ArtifactInfo, packageName string) (PackageMetadata, error) - UploadPackageFile( - ctx context.Context, - info ArtifactInfo, - file multipart.File, - fileHeader *multipart.FileHeader, - ) (*commons.ResponseHeaders, string, errcode.Error) - - DownloadPackageFile(ctx context.Context, info pkg.ArtifactInfo, image, version, filename string) ( - *commons.ResponseHeaders, - *storage.FileReader, - string, - errcode.Error, - ) -} - -// NewController creates a new PyPI controller. +// NewController creates a new Python controller. func NewController( proxyStore store.UpstreamProxyConfigRepository, registryDao store.RegistryRepository, @@ -65,6 +61,8 @@ func NewController( fileManager filemanager.FileManager, tx dbtx.Transactor, urlProvider urlprovider.Provider, + local python.LocalRegistry, + proxy python.Proxy, ) Controller { return &controller{ proxyStore: proxyStore, @@ -74,5 +72,7 @@ func NewController( fileManager: fileManager, tx: tx, urlProvider: urlProvider, + local: local, + proxy: proxy, } } diff --git a/registry/app/api/controller/pkg/python/download.go b/registry/app/api/controller/pkg/python/download.go new file mode 100644 index 000000000..f08494c08 --- /dev/null +++ b/registry/app/api/controller/pkg/python/download.go @@ -0,0 +1,59 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "context" + "fmt" + + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/base" + "github.com/harness/gitness/registry/app/pkg/python" + "github.com/harness/gitness/registry/app/pkg/response" + pythontype "github.com/harness/gitness/registry/app/pkg/types/python" + registrytypes "github.com/harness/gitness/registry/types" +) + +func (c *controller) DownloadPackageFile( + ctx context.Context, + info pythontype.ArtifactInfo, +) *GetArtifactResponse { + f := func(registry registrytypes.Registry, a pkg.Artifact) response.Response { + info.RegIdentifier = registry.Name + info.RegistryID = registry.ID + pythonRegistry, ok := a.(python.Registry) + if !ok { + return &GetArtifactResponse{ + []error{fmt.Errorf("invalid registry type: expected python.Registry")}, + nil, "", nil, nil, + } + } + headers, fileReader, redirectURL, errs := pythonRegistry.DownloadPackageFile(ctx, info) + return &GetArtifactResponse{ + errs, headers, redirectURL, + fileReader, nil, + } + } + + result := base.ProxyWrapper(ctx, c.registryDao, f, info.BaseArtifactInfo()) + getResponse, ok := result.(*GetArtifactResponse) + if !ok { + return &GetArtifactResponse{ + []error{fmt.Errorf("invalid response type: expected GetArtifactResponse")}, + nil, "", nil, nil, + } + } + return getResponse +} diff --git a/registry/app/api/controller/pkg/python/metadata.go b/registry/app/api/controller/pkg/python/metadata.go new file mode 100644 index 000000000..4f7c231d5 --- /dev/null +++ b/registry/app/api/controller/pkg/python/metadata.go @@ -0,0 +1,59 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "context" + "fmt" + + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/base" + "github.com/harness/gitness/registry/app/pkg/python" + "github.com/harness/gitness/registry/app/pkg/response" + pythontype "github.com/harness/gitness/registry/app/pkg/types/python" + registrytypes "github.com/harness/gitness/registry/types" +) + +// Metadata represents the metadata of a Python package. +func (c *controller) GetPackageMetadata(ctx context.Context, info pythontype.ArtifactInfo) ( + pythontype.PackageMetadata, + error, +) { + f := func(registry registrytypes.Registry, a pkg.Artifact) response.Response { + info.RegIdentifier = registry.Name + info.RegistryID = registry.ID + + pythonRegistry, ok := a.(python.Registry) + if !ok { + return &GetMetadataResponse{ + []error{fmt.Errorf("invalid registry type: expected python.Registry")}, + nil, pythontype.PackageMetadata{}, + } + } + + metadata, err := pythonRegistry.GetPackageMetadata(ctx, info) + return &GetMetadataResponse{ + []error{err}, nil, metadata, + } + } + + result := base.ProxyWrapper(ctx, c.registryDao, f, info.BaseArtifactInfo()) + metadataResponse, ok := result.(*GetMetadataResponse) + if !ok { + return pythontype.PackageMetadata{}, + fmt.Errorf("invalid response type: expected GetMetadataResponse, got %T", result) + } + return metadataResponse.PackageMetadata, nil +} diff --git a/registry/app/api/controller/pkg/python/response.go b/registry/app/api/controller/pkg/python/response.go new file mode 100644 index 000000000..f4cef781a --- /dev/null +++ b/registry/app/api/controller/pkg/python/response.go @@ -0,0 +1,72 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "io" + + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/response" + pythontype "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/app/storage" +) + +var _ response.Response = (*GetMetadataResponse)(nil) +var _ response.Response = (*GetArtifactResponse)(nil) +var _ response.Response = (*PutArtifactResponse)(nil) + +type GetMetadataResponse struct { + Errors []error + ResponseHeaders *commons.ResponseHeaders + PackageMetadata pythontype.PackageMetadata +} + +func (r *GetMetadataResponse) GetErrors() []error { + return r.Errors +} +func (r *GetMetadataResponse) SetError(err error) { + r.Errors = make([]error, 1) + r.Errors[0] = err +} + +type GetArtifactResponse struct { + Errors []error + ResponseHeaders *commons.ResponseHeaders + RedirectURL string + Body *storage.FileReader + ReadCloser io.ReadCloser +} + +func (r *GetArtifactResponse) GetErrors() []error { + return r.Errors +} +func (r *GetArtifactResponse) SetError(err error) { + r.Errors = make([]error, 1) + r.Errors[0] = err +} + +type PutArtifactResponse struct { + Sha256 string + Errors []error + ResponseHeaders *commons.ResponseHeaders +} + +func (r *PutArtifactResponse) GetErrors() []error { + return r.Errors +} +func (r *PutArtifactResponse) SetError(err error) { + r.Errors = make([]error, 1) + r.Errors[0] = err +} diff --git a/registry/app/api/controller/pkg/python/upload_package.go b/registry/app/api/controller/pkg/python/upload_package.go new file mode 100644 index 000000000..220b8becf --- /dev/null +++ b/registry/app/api/controller/pkg/python/upload_package.go @@ -0,0 +1,71 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "context" + "fmt" + "mime/multipart" + + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/base" + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/python" + "github.com/harness/gitness/registry/app/pkg/response" + pythontype "github.com/harness/gitness/registry/app/pkg/types/python" + registrytypes "github.com/harness/gitness/registry/types" +) + +// UploadPackageFile FIXME: Extract this upload function for all types of packageTypes +// uploads the package file to the storage. +func (c *controller) UploadPackageFile( + ctx context.Context, + info pythontype.ArtifactInfo, + file multipart.File, + fileHeader *multipart.FileHeader, +) *PutArtifactResponse { + f := func(registry registrytypes.Registry, a pkg.Artifact) response.Response { + info.RegIdentifier = registry.Name + info.RegistryID = registry.ID + pythonRegistry, ok := a.(python.Registry) + if !ok { + return &PutArtifactResponse{ + "", + []error{fmt.Errorf("invalid registry type: expected python.Registry")}, + nil, + } + } + headers, sha256, err := pythonRegistry.UploadPackageFile(ctx, info, file, fileHeader) + if commons.IsEmptyError(err) { + return &PutArtifactResponse{ + sha256, []error{}, headers, + } + } + return &PutArtifactResponse{ + sha256, []error{err}, headers, + } + } + + result := base.NoProxyWrapper(ctx, c.registryDao, f, info.BaseArtifactInfo()) + response, ok := result.(*PutArtifactResponse) + if !ok { + return &PutArtifactResponse{ + "", + []error{fmt.Errorf("invalid response type: expected PutArtifactResponse")}, + nil, + } + } + return response +} diff --git a/registry/app/pkg/pypi/wire.go b/registry/app/api/controller/pkg/python/wire.go similarity index 88% rename from registry/app/pkg/pypi/wire.go rename to registry/app/api/controller/pkg/python/wire.go index 931ddac79..201a4bad5 100644 --- a/registry/app/pkg/pypi/wire.go +++ b/registry/app/api/controller/pkg/python/wire.go @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pypi +package python import ( urlprovider "github.com/harness/gitness/app/url" "github.com/harness/gitness/registry/app/pkg/filemanager" + "github.com/harness/gitness/registry/app/pkg/python" "github.com/harness/gitness/registry/app/store" "github.com/harness/gitness/store/database/dbtx" @@ -31,9 +32,10 @@ func ControllerProvider( fileManager filemanager.FileManager, tx dbtx.Transactor, urlProvider urlprovider.Provider, + local python.LocalRegistry, + proxy python.Proxy, ) Controller { - return NewController(proxyStore, registryDao, imageDao, artifactDao, fileManager, tx, urlProvider) + return NewController(proxyStore, registryDao, imageDao, artifactDao, fileManager, tx, urlProvider, local, proxy) } var ControllerSet = wire.NewSet(ControllerProvider) -var WireSet = wire.NewSet(ControllerSet) diff --git a/registry/app/api/handler/packages/handler.go b/registry/app/api/handler/packages/handler.go index 831510c99..8ae09526e 100644 --- a/registry/app/api/handler/packages/handler.go +++ b/registry/app/api/handler/packages/handler.go @@ -16,6 +16,8 @@ package packages import ( "context" + "encoding/json" + "errors" "fmt" "net/http" "strings" @@ -72,7 +74,8 @@ type Handler interface { ) error GetArtifactInfo(r *http.Request) (pkg.ArtifactInfo, errcode.Error) GetAuthenticator() authn.Authenticator - HandleErrors(ctx context.Context, errors errcode.Error, w http.ResponseWriter) + HandleErrors2(ctx context.Context, errors errcode.Error, w http.ResponseWriter) + HandleErrors(ctx context.Context, errors errcode.Errors, w http.ResponseWriter) ServeContent( w http.ResponseWriter, r *http.Request, fileReader *storage.FileReader, filename string, ) @@ -144,10 +147,12 @@ func (h *handler) GetArtifactInfo(r *http.Request) (pkg.ArtifactInfo, errcode.Er PathPackageType: pathPackageType, }, RegIdentifier: registryIdentifier, + RegistryID: registry.ID, + Image: "", }, errcode.Error{} } -func (h *handler) HandleErrors(ctx context.Context, err errcode.Error, w http.ResponseWriter) { +func (h *handler) HandleErrors2(ctx context.Context, err errcode.Error, w http.ResponseWriter) { if !commons.IsEmptyError(err) { w.WriteHeader(err.Code.Descriptor().HTTPStatusCode) _ = errcode.ServeJSON(w, err) @@ -155,6 +160,34 @@ func (h *handler) HandleErrors(ctx context.Context, err errcode.Error, w http.Re } } +// HandleErrors TODO: Improve Error Handling +// HandleErrors handles errors and writes the appropriate response to the client. +func (h *handler) HandleErrors(ctx context.Context, errs errcode.Errors, w http.ResponseWriter) { + if !commons.IsEmpty(errs) { + LogError(errs) + log.Ctx(ctx).Error().Errs("errs occurred during artifact operation: ", errs).Msgf("Error occurred") + err := errs[0] + var e *commons.Error + if errors.As(err, &e) { + code := e.Status + w.WriteHeader(code) + } else { + w.WriteHeader(http.StatusInternalServerError) + } + w.Header().Set("Content-Type", "application/json") + err = json.NewEncoder(w).Encode(errs) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msgf("Error occurred during artifact error encoding") + } + } +} + +func LogError(errList errcode.Errors) { + for _, e1 := range errList { + log.Error().Err(e1).Msgf("error: %v", e1) + } +} + // extractPathVars extracts rootSpace, registryId, pathPackageType from the path // Path format: /pkg/:rootSpace/:registry/:pathPackageType/... func extractPathVars(r *http.Request) ( diff --git a/registry/app/api/handler/pypi/download.go b/registry/app/api/handler/pypi/download.go deleted file mode 100644 index 4f269e421..000000000 --- a/registry/app/api/handler/pypi/download.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2023 Harness, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pypi - -import ( - "net/http" - - "github.com/harness/gitness/registry/app/pkg/commons" - - "github.com/go-chi/chi/v5" -) - -func (h *handler) DownloadPackageFile(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - info, err := h.GetArtifactInfo(r) - if !commons.IsEmptyError(err) { - h.HandleErrors(r.Context(), err, w) - return - } - - image := chi.URLParam(r, "image") - filename := chi.URLParam(r, "filename") - version := chi.URLParam(r, "version") - headers, fileReader, redirectURL, err := h.controller.DownloadPackageFile(ctx, info, image, - version, filename) - if commons.IsEmptyError(err) { - w.Header().Set("Content-Disposition", "attachment; filename="+filename) - if redirectURL != "" { - http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect) - return - } - h.ServeContent(w, r, fileReader, filename) - headers.WriteToResponse(w) - return - } - h.HandleErrors(r.Context(), err, w) -} diff --git a/registry/app/api/handler/pypi/upload.go b/registry/app/api/handler/pypi/upload.go deleted file mode 100644 index 8652c3214..000000000 --- a/registry/app/api/handler/pypi/upload.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2023 Harness, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pypi - -import ( - "fmt" - "net/http" - - "github.com/harness/gitness/registry/app/dist_temp/errcode" - "github.com/harness/gitness/registry/app/pkg/commons" -) - -func (h *handler) UploadPackageFile(w http.ResponseWriter, r *http.Request) { - file, fileHeader, err := r.FormFile("content") - if err != nil { - h.HandleErrors(r.Context(), errcode.ErrCodeInvalidRequest.WithMessage(fmt.Sprintf("failed to parse file: %s, "+ - "please provide correct file path ", err.Error())), w) - return - } - - defer file.Close() - - info, err := h.getPackageArtifactInfo(r) - if err != nil { - h.HandleErrors(r.Context(), - errcode.ErrCodeInvalidRequest.WithMessage("failed to get artifact info "+err.Error()), w) - return - } - - headers, sha256, err2 := h.controller.UploadPackageFile(r.Context(), info, file, fileHeader) - - if commons.IsEmptyError(err2) { - headers.WriteToResponse(w) - _, err := w.Write([]byte(fmt.Sprintf("Pushed.\nSha256: %s", sha256))) - if err != nil { - h.HandleErrors(r.Context(), errcode.ErrCodeUnknown.WithDetail(err2), w) - return - } - } - h.HandleErrors(r.Context(), err2, w) -} diff --git a/registry/app/api/handler/python/download.go b/registry/app/api/handler/python/download.go new file mode 100644 index 000000000..d8a68838e --- /dev/null +++ b/registry/app/api/handler/python/download.go @@ -0,0 +1,59 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "fmt" + "net/http" + + "github.com/harness/gitness/registry/app/pkg/commons" + pythontype "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/request" + + "github.com/rs/zerolog/log" +) + +func (h *handler) DownloadPackageFile(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + info, ok := request.ArtifactInfoFrom(ctx).(*pythontype.ArtifactInfo) + if !ok { + log.Ctx(ctx).Error().Msg("Failed to get python artifact info from context") + h.HandleErrors(r.Context(), []error{fmt.Errorf("failed to fetch python artifact info from context")}, w) + return + } + + response := h.controller.DownloadPackageFile(ctx, *info) + + defer func() { + if response.Body != nil { + err := response.Body.Close() + if err != nil { + log.Ctx(r.Context()).Error().Msgf("Failed to close body: %v", err) + } + } + }() + + if !commons.IsEmpty(response.GetErrors()) { + h.HandleErrors(r.Context(), response.GetErrors(), w) + } + + w.Header().Set("Content-Disposition", "attachment; filename="+info.Filename) + if response.RedirectURL != "" { + http.Redirect(w, r, response.RedirectURL, http.StatusTemporaryRedirect) + return + } + h.ServeContent(w, r, response.Body, info.Filename) + response.ResponseHeaders.WriteToResponse(w) +} diff --git a/registry/app/api/handler/pypi/handler.go b/registry/app/api/handler/python/handler.go similarity index 55% rename from registry/app/api/handler/pypi/handler.go rename to registry/app/api/handler/python/handler.go index bd8d55b2c..8fe1e90cb 100644 --- a/registry/app/api/handler/pypi/handler.go +++ b/registry/app/api/handler/python/handler.go @@ -12,32 +12,36 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pypi +package python import ( "net/http" + "github.com/harness/gitness/registry/app/api/controller/pkg/python" "github.com/harness/gitness/registry/app/api/handler/packages" "github.com/harness/gitness/registry/app/api/handler/utils" - pypi2 "github.com/harness/gitness/registry/app/metadata/pypi" - "github.com/harness/gitness/registry/app/pkg/pypi" + python2 "github.com/harness/gitness/registry/app/metadata/python" + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/commons" + pythontype "github.com/harness/gitness/registry/app/pkg/types/python" "github.com/go-chi/chi/v5" ) type Handler interface { - DownloadPackageFile(http.ResponseWriter, *http.Request) + pkg.ArtifactInfoProvider UploadPackageFile(writer http.ResponseWriter, request *http.Request) + DownloadPackageFile(http.ResponseWriter, *http.Request) PackageMetadata(writer http.ResponseWriter, request *http.Request) } type handler struct { packages.Handler - controller pypi.Controller + controller python.Controller } func NewHandler( - controller pypi.Controller, + controller python.Controller, packageHandler packages.Handler, ) Handler { return &handler{ @@ -48,26 +52,33 @@ func NewHandler( var _ Handler = (*handler)(nil) -func (h *handler) getPackageArtifactInfo(r *http.Request) (pypi.ArtifactInfo, error) { - info, e := h.GetArtifactInfo(r) - - if e.Error() != "" { - return pypi.ArtifactInfo{}, e +func (h *handler) GetPackageArtifactInfo(r *http.Request) (pkg.PackageArtifactInfo, error) { + info, err := h.Handler.GetArtifactInfo(r) + if !commons.IsEmptyError(err) { + return nil, err } - var md pypi2.Metadata - err := utils.FillFromForm(r, &md) - if err != nil { - return pypi.ArtifactInfo{}, err + image := chi.URLParam(r, "image") + filename := chi.URLParam(r, "filename") + version := chi.URLParam(r, "version") + + var md python2.Metadata + err2 := utils.FillFromForm(r, &md) + + if err2 == nil { + if image == "" { + image = md.Name + } + if version == "" { + version = md.Version + } } - info.Image = chi.URLParam(r, "image") - if info.Image == "" { - info.Image = md.Name - } - - return pypi.ArtifactInfo{ - ArtifactInfo: &info, + info.Image = image + return &pythontype.ArtifactInfo{ + ArtifactInfo: info, Metadata: md, + Filename: filename, + Version: version, }, nil } diff --git a/registry/app/api/handler/pypi/list.go b/registry/app/api/handler/python/list.go similarity index 84% rename from registry/app/api/handler/pypi/list.go rename to registry/app/api/handler/python/list.go index 6becebebb..e03a2a518 100644 --- a/registry/app/api/handler/pypi/list.go +++ b/registry/app/api/handler/python/list.go @@ -12,11 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pypi +package python import ( "html/template" "net/http" + + pythontype "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/request" ) const HTMLTemplate = ` @@ -37,18 +40,18 @@ const HTMLTemplate = ` ` func (h *handler) PackageMetadata(w http.ResponseWriter, r *http.Request) { - info, err := h.getPackageArtifactInfo(r) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) + contextInfo := request.ArtifactInfoFrom(r.Context()) + info, ok := contextInfo.(*pythontype.ArtifactInfo) + if !ok { + http.Error(w, "Invalid request context", http.StatusBadRequest) return } - if info.Image == "" { http.Error(w, "Package name required", http.StatusBadRequest) return } - packageData, err := h.controller.GetPackageMetadata(r.Context(), info, info.Image) + packageData, err := h.controller.GetPackageMetadata(r.Context(), *info) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/registry/app/api/handler/python/upload.go b/registry/app/api/handler/python/upload.go new file mode 100644 index 000000000..1c55d5b99 --- /dev/null +++ b/registry/app/api/handler/python/upload.go @@ -0,0 +1,60 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "fmt" + "net/http" + + "github.com/harness/gitness/registry/app/dist_temp/errcode" + pythontype "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/request" +) + +func (h *handler) UploadPackageFile(w http.ResponseWriter, r *http.Request) { + file, fileHeader, err := r.FormFile("content") + + if err != nil { + h.HandleErrors2(r.Context(), errcode.ErrCodeInvalidRequest.WithMessage(fmt.Sprintf("failed to parse file: %s, "+ + "please provide correct file path ", err.Error())), w) + return + } + + defer file.Close() + + contextInfo := request.ArtifactInfoFrom(r.Context()) + info, ok := contextInfo.(*pythontype.ArtifactInfo) + if !ok { + h.HandleErrors2(r.Context(), errcode.ErrCodeInvalidRequest.WithMessage("failed to fetch info from context"), w) + return + } + // TODO: Can we extract this out to ArtifactInfoProvider + if info.Filename == "" { + info.Filename = fileHeader.Filename + request.WithArtifactInfo(r.Context(), info) + } + + response := h.controller.UploadPackageFile(r.Context(), *info, file, fileHeader) + + if len(response.Errors) == 0 { + response.ResponseHeaders.WriteToResponse(w) + _, err := w.Write([]byte(fmt.Sprintf("Pushed.\nSha256: %s", response.Sha256))) + if err != nil { + h.HandleErrors2(r.Context(), errcode.ErrCodeUnknown.WithDetail(err), w) + return + } + } + h.HandleErrors(r.Context(), response.GetErrors(), w) +} diff --git a/registry/app/api/handler/utils/jsonform.go b/registry/app/api/handler/utils/jsonform.go index 160024cc5..21f9edfb8 100644 --- a/registry/app/api/handler/utils/jsonform.go +++ b/registry/app/api/handler/utils/jsonform.go @@ -31,6 +31,9 @@ func FillFromForm(r *http.Request, data interface{}) error { if err := r.ParseForm(); err != nil { return err } + if err := r.ParseMultipartForm(32 << 22); err != nil { + return err + } v := reflect.ValueOf(data).Elem() t := v.Type() diff --git a/registry/app/api/middleware/artifact_info.go b/registry/app/api/middleware/artifact_info.go new file mode 100644 index 000000000..1fbb1c9bd --- /dev/null +++ b/registry/app/api/middleware/artifact_info.go @@ -0,0 +1,41 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package middleware + +import ( + "net/http" + + "github.com/harness/gitness/app/api/render" + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/request" +) + +// StoreArtifactInfo stores the Base Artifact Info for any package type. +func StoreArtifactInfo( + provider pkg.ArtifactInfoProvider, +) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + packageInfo, err := provider.GetPackageArtifactInfo(r) + if err != nil { + render.BadRequestf(r.Context(), w, err.Error()) + return + } + // Store the base artifact info in the context + ctx := request.WithArtifactInfo(r.Context(), packageInfo) + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} diff --git a/registry/app/api/middleware/bandwidth_stats.go b/registry/app/api/middleware/bandwidth_stats.go index 93544b9c9..71817f45a 100644 --- a/registry/app/api/middleware/bandwidth_stats.go +++ b/registry/app/api/middleware/bandwidth_stats.go @@ -25,12 +25,12 @@ import ( "github.com/harness/gitness/registry/app/api/handler/oci" "github.com/harness/gitness/registry/app/api/router/utils" "github.com/harness/gitness/registry/app/dist_temp/errcode" + "github.com/harness/gitness/registry/app/metadata" "github.com/harness/gitness/registry/app/pkg" "github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/docker" generic2 "github.com/harness/gitness/registry/app/pkg/generic" maven2 "github.com/harness/gitness/registry/app/pkg/maven" - "github.com/harness/gitness/registry/app/store/database" "github.com/harness/gitness/registry/types" "github.com/harness/gitness/store" @@ -225,7 +225,7 @@ func dbBandwidthStatForGenericArtifact( return errcode.ErrCodeInvalidRequest.WithDetail(err) } - var metadata database.GenericMetadata + var metadata metadata.GenericMetadata err = json.Unmarshal(art.Metadata, &metadata) if err != nil { @@ -273,7 +273,7 @@ func dbBandwidthStatForMavenArtifact( return errcode.ErrCodeInvalidRequest.WithDetail(err) } - var metadata database.MavenMetadata + var metadata metadata.MavenMetadata err = json.Unmarshal(art.Metadata, &metadata) if err != nil { @@ -351,7 +351,8 @@ func getImageFromUpstreamProxy(ctx context.Context, c *docker.Controller, info p return nil, errors.New("image not found in upstream proxy") } -func getMavenArtifactFromUpstreamProxy(ctx context.Context, +func getMavenArtifactFromUpstreamProxy( + ctx context.Context, c *maven2.Controller, info pkg.MavenArtifactInfo, ) (*types.Image, error) { diff --git a/registry/app/api/openapi/api.yaml b/registry/app/api/openapi/api.yaml index 01f9c3c3e..5b8d9ff5a 100644 --- a/registry/app/api/openapi/api.yaml +++ b/registry/app/api/openapi/api.yaml @@ -1786,10 +1786,9 @@ components: type: object description: Config for python artifact details properties: - groupId: - type: string - artifactId: - type: string + metadata: + type: object + additionalProperties: true HelmArtifactDetailConfig: type: object description: Config for helm artifact details diff --git a/registry/app/api/openapi/contracts/artifact/services.gen.go b/registry/app/api/openapi/contracts/artifact/services.gen.go index 32380cbad..5d8788707 100644 --- a/registry/app/api/openapi/contracts/artifact/services.gen.go +++ b/registry/app/api/openapi/contracts/artifact/services.gen.go @@ -1,6 +1,6 @@ // Package artifact provides primitives to interact with the openapi HTTP API. // -// Code generated by github.com/deepmap/oapi-codegen/v2 version v2.1.0 DO NOT EDIT. +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.4.1 DO NOT EDIT. package artifact import ( @@ -346,7 +346,6 @@ type MiddlewareFunc func(http.Handler) http.Handler // CreateRegistry operation middleware func (siw *ServerInterfaceWrapper) CreateRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -369,12 +368,11 @@ func (siw *ServerInterfaceWrapper) CreateRegistry(w http.ResponseWriter, r *http handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // DeleteRegistry operation middleware func (siw *ServerInterfaceWrapper) DeleteRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -395,12 +393,11 @@ func (siw *ServerInterfaceWrapper) DeleteRegistry(w http.ResponseWriter, r *http handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetRegistry operation middleware func (siw *ServerInterfaceWrapper) GetRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -421,12 +418,11 @@ func (siw *ServerInterfaceWrapper) GetRegistry(w http.ResponseWriter, r *http.Re handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // ModifyRegistry operation middleware func (siw *ServerInterfaceWrapper) ModifyRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -447,12 +443,11 @@ func (siw *ServerInterfaceWrapper) ModifyRegistry(w http.ResponseWriter, r *http handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // ListArtifactLabels operation middleware func (siw *ServerInterfaceWrapper) ListArtifactLabels(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -500,12 +495,11 @@ func (siw *ServerInterfaceWrapper) ListArtifactLabels(w http.ResponseWriter, r * handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactStatsForRegistry operation middleware func (siw *ServerInterfaceWrapper) GetArtifactStatsForRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -545,12 +539,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactStatsForRegistry(w http.ResponseWr handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // DeleteArtifact operation middleware func (siw *ServerInterfaceWrapper) DeleteArtifact(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -580,12 +573,11 @@ func (siw *ServerInterfaceWrapper) DeleteArtifact(w http.ResponseWriter, r *http handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // UpdateArtifactLabels operation middleware func (siw *ServerInterfaceWrapper) UpdateArtifactLabels(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -615,12 +607,11 @@ func (siw *ServerInterfaceWrapper) UpdateArtifactLabels(w http.ResponseWriter, r handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactStats operation middleware func (siw *ServerInterfaceWrapper) GetArtifactStats(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -669,12 +660,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactStats(w http.ResponseWriter, r *ht handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactSummary operation middleware func (siw *ServerInterfaceWrapper) GetArtifactSummary(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -704,12 +694,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactSummary(w http.ResponseWriter, r * handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // DeleteArtifactVersion operation middleware func (siw *ServerInterfaceWrapper) DeleteArtifactVersion(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -748,12 +737,11 @@ func (siw *ServerInterfaceWrapper) DeleteArtifactVersion(w http.ResponseWriter, handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactDetails operation middleware func (siw *ServerInterfaceWrapper) GetArtifactDetails(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -803,12 +791,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactDetails(w http.ResponseWriter, r * handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetDockerArtifactDetails operation middleware func (siw *ServerInterfaceWrapper) GetDockerArtifactDetails(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -865,12 +852,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactDetails(w http.ResponseWrite handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetDockerArtifactLayers operation middleware func (siw *ServerInterfaceWrapper) GetDockerArtifactLayers(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -927,12 +913,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactLayers(w http.ResponseWriter handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetDockerArtifactManifest operation middleware func (siw *ServerInterfaceWrapper) GetDockerArtifactManifest(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -989,12 +974,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactManifest(w http.ResponseWrit handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetDockerArtifactManifests operation middleware func (siw *ServerInterfaceWrapper) GetDockerArtifactManifests(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1033,12 +1017,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactManifests(w http.ResponseWri handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactFiles operation middleware func (siw *ServerInterfaceWrapper) GetArtifactFiles(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1120,12 +1103,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactFiles(w http.ResponseWriter, r *ht handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetHelmArtifactDetails operation middleware func (siw *ServerInterfaceWrapper) GetHelmArtifactDetails(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1164,12 +1146,11 @@ func (siw *ServerInterfaceWrapper) GetHelmArtifactDetails(w http.ResponseWriter, handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetHelmArtifactManifest operation middleware func (siw *ServerInterfaceWrapper) GetHelmArtifactManifest(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1208,12 +1189,11 @@ func (siw *ServerInterfaceWrapper) GetHelmArtifactManifest(w http.ResponseWriter handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactVersionSummary operation middleware func (siw *ServerInterfaceWrapper) GetArtifactVersionSummary(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1252,12 +1232,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactVersionSummary(w http.ResponseWrit handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetAllArtifactVersions operation middleware func (siw *ServerInterfaceWrapper) GetAllArtifactVersions(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1330,12 +1309,11 @@ func (siw *ServerInterfaceWrapper) GetAllArtifactVersions(w http.ResponseWriter, handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetAllArtifactsByRegistry operation middleware func (siw *ServerInterfaceWrapper) GetAllArtifactsByRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1407,12 +1385,11 @@ func (siw *ServerInterfaceWrapper) GetAllArtifactsByRegistry(w http.ResponseWrit handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetClientSetupDetails operation middleware func (siw *ServerInterfaceWrapper) GetClientSetupDetails(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1452,12 +1429,11 @@ func (siw *ServerInterfaceWrapper) GetClientSetupDetails(w http.ResponseWriter, handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // ListWebhooks operation middleware func (siw *ServerInterfaceWrapper) ListWebhooks(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1521,12 +1497,11 @@ func (siw *ServerInterfaceWrapper) ListWebhooks(w http.ResponseWriter, r *http.R handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // CreateWebhook operation middleware func (siw *ServerInterfaceWrapper) CreateWebhook(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1547,12 +1522,11 @@ func (siw *ServerInterfaceWrapper) CreateWebhook(w http.ResponseWriter, r *http. handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // DeleteWebhook operation middleware func (siw *ServerInterfaceWrapper) DeleteWebhook(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1582,12 +1556,11 @@ func (siw *ServerInterfaceWrapper) DeleteWebhook(w http.ResponseWriter, r *http. handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetWebhook operation middleware func (siw *ServerInterfaceWrapper) GetWebhook(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1617,12 +1590,11 @@ func (siw *ServerInterfaceWrapper) GetWebhook(w http.ResponseWriter, r *http.Req handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // UpdateWebhook operation middleware func (siw *ServerInterfaceWrapper) UpdateWebhook(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1652,12 +1624,11 @@ func (siw *ServerInterfaceWrapper) UpdateWebhook(w http.ResponseWriter, r *http. handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // ListWebhookExecutions operation middleware func (siw *ServerInterfaceWrapper) ListWebhookExecutions(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1706,12 +1677,11 @@ func (siw *ServerInterfaceWrapper) ListWebhookExecutions(w http.ResponseWriter, handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetWebhookExecution operation middleware func (siw *ServerInterfaceWrapper) GetWebhookExecution(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1750,12 +1720,11 @@ func (siw *ServerInterfaceWrapper) GetWebhookExecution(w http.ResponseWriter, r handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // ReTriggerWebhookExecution operation middleware func (siw *ServerInterfaceWrapper) ReTriggerWebhookExecution(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1794,12 +1763,11 @@ func (siw *ServerInterfaceWrapper) ReTriggerWebhookExecution(w http.ResponseWrit handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactStatsForSpace operation middleware func (siw *ServerInterfaceWrapper) GetArtifactStatsForSpace(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1839,12 +1807,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactStatsForSpace(w http.ResponseWrite handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetAllArtifacts operation middleware func (siw *ServerInterfaceWrapper) GetAllArtifacts(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1932,12 +1899,11 @@ func (siw *ServerInterfaceWrapper) GetAllArtifacts(w http.ResponseWriter, r *htt handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetAllRegistries operation middleware func (siw *ServerInterfaceWrapper) GetAllRegistries(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -2025,7 +1991,7 @@ func (siw *ServerInterfaceWrapper) GetAllRegistries(w http.ResponseWriter, r *ht handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } type UnescapedCookieParamError struct { @@ -5817,48 +5783,48 @@ var swaggerSpec = []string{ "e9RYoqbBHUMXoWKhYvIIEqOm+iYLdGytk+aq3s72Cuz1K7DcMb6L7mq4uusBcOxI6SJD5vZzaqUWJHTM", "+qCCRoWyNjieoP1WJa1Xg7+RGswjtyxEppCUIsaqV4OnpgafLWZUP5NW2kCJL2nUeXm7bchTgiq3w6AS", "C6lxqLFpvLXRLpwpBSL1+vGk9aMyyTqYmoMTmo6cVrRW+5mTLDDRn9ktUJKlE9vjqGn57LAamDWHCDsk", - "ccQRm+IVIOJyZMRLHsBShN2IGBqdn0BDCEYTg1JW7cAcMtv7JouwfvsJoih5hsEUEAJR3O0U4TFK/B9b", - "1vWrToeW3iZqLV2z+VTZmH+FF1jL0Xjjgb7nhs3etyfk1206/tZftpQSRAvfG7uDbaOB/apjVmrPXxzC", - "BXw/Lt4v6cldEafaFN9mj0JVimwCPtOZ30JEMhA5CXK+ppggCFaqnmpyDs2fvjGwULaX+4XKV3MM5QUp", - "e/IKrbbWXLpCa90T1MZ9UX11yN6js7brt19IzOKab9A6hRi1aNjtfOr2r5ZbVcQON56mm0wpf7emG83u", - "ALFcBoTKV8dUWRRoM03IMgZK9nbHsSyLXRBK9z4zWr71jlxjOdjaDG0WP61Id0n5Uz3hGXSeisUkEwpV", - "t4YY9Hqnx9U8V3WfbyC0lOBZxFBIF+aulIlwCBHhoCUqT41RzQYYsHQq2AnnJWepZ4AdzNO+zDPGuTgh", - "qnf119FofHvreu7VcHL9dUZ7H89mX2ba7tWgBs3+FjwKn3Os8zlfHj7wpQY/TVRGyzAcX5oXlQUbPNqT", - "W+KbHaEoXCx0bmmKiIgixWQOZ3eTq+Ho7mE0Gw/vJmw7nP928+VycjUZ1X6/HF+P2W+6Ca/YLYbdcYZ4", - "1Jo2bEw2MUXJT90FFMj4kmhndpUi4dqsriIkrrVkPaJuc7/xGHE2SjQP7GMvZmXIh+pzjtwddJk9up47", - "yjBhT/ANn/HYp7PHTmtGMCYI0F3XdD0NtXNhtZrnBNeUr+f+fFdSSO+Ek2KhBumEq/ytJ6qySVKDt8hN", - "gy1S0mQYIoOPboUJeUk6hWVDvQOC5dbF5hY1q4B8x3BEeVhtNNVncCEOfmXRHTKH7MF0hzF4jErWcv44", - "n+fCws3WXmWqvrm6A+lmDIYxhn6GoJ6gUOTO1X/lW2M1HRh7e9D6EF1U2CGbylHFTCwrHRY3sQ5pZsnC", - "DdPuzUmD0cnPqcQ5h8SgMvv3ZtniM5VvXNrE7NPd3VTKmiPrVWXuMQn07uDLAvz2ar2Z8iIpW0fSRcW9", - "0F5kajN8Gom4A5ukHnURajB+aqnrtDbtbHw3mww/XI8fuE1Lrdy74fWD2cKtXYbZq2BnrNCiVca2ylas", - "RpbFoQz50GwsLZtAhSBYK7n8tR+kYNFeReZpBtH2+hVBoay+zK0HKmpQVaFX/6KAjdWnaL78Kcetc9rU", - "nwPsEnDw2pbgt7r2VVczyaTS8mVY4vTJLMN4nsjcnMIBSzwBaj7XfecE8AlGFF5Y9HHhLglJ8cVg8Pz8", - "fLbkVc/ChA0tJFFzg8PpRAmUuHD/ODs/O2eH6SmMQRq6F+4/2U/8CJTxdYCUq8000a3DI/E6Zd7Rmcua", - "5PqRTr0ool59AgRWkLBZNGwmiyIDzbu/m3v17dO1CQKl51HrL4NWHs/8x/kf5oZEuUEtx/LGc9+fn7dX", - "VN56Y1Us+tKk4X1//k/bekX23P+xoU/3QgZLpSrDeuVMq/NMwIJOoatsu+5ppRw3g1/qI90bDp8IEo1V", - "dMl+V4DkhDyoEPh+ksX8LVT6/0X4BGOHx+WVgcab2Bpo2gfKOdRKMLHgpkw4/QrQ8f78fXulPNn5/uBU", - "m28Tnjx3ATWKZwZJhmJcwEXE8HaHzUdITgEzr1G1HAs8psk3YyjNNBj6ylJH452UDruGXL8EgPa+vvUg", - "3CsI6+jZYkkcyHv6QXGJqNV31yEm1VC3uq1VC6DDe0Kk11ovBQv4mflO2pZmN+kWZTEEyF/eQbStajU/", - "uNXD2whvHeAUgBehIpb4xjJhrhbeHyGp5Mw90y3Upey7Vwnas95tx+IcJatLQKB1BZIoxbdCr/5p9h65", - "RuTWsbQLbn/Jv2y2L7L1M8PmRImQOgxeJfH9juZQOxpliveAOcUsaDBh2w0DXu5IpoEJhB0tXG3u/80u", - "KrU3BjrZuvs0BxSI798yOCayexuityGawF7kSbSAOy/cDPgioeKrsigq9Peg7ArKfN73AUtxMTT4Jf7o", - "YuzKtwDajN5vShb+k1XOMml9by8f6gYgrgHppTA9UPJitivf4kzZqHuVNwZeE6Lb6/jLMAq+yYq7K3nO", - "qF7H20gFBeQj1OHwhYSC+T5byYY+MbtWRHTJvn9DQeF5kHcRER2jekHpICjG1wKkuFQK7FVqitzk1kKT", - "JwBvkZkiUXgvMjqR4fzpRWUHUckhdghRUXPRWguLktm2RVzUHLi9wDStMZJTvejsIDoK3A4pPHgr6cH2", - "4oPfxPa88iJFLwl7kIQXX0fmYQQt9+68aMPO/UoU+M2Wihd0wkkQ+YICu4Zp4asQRsFB3HuKNzh6Od7m", - "gEEKy8scLyxhtLI6XNC9rKGV4fpzDW9j0aqPu8d7B7wbXm6RqC993iP0rbY9xuc8GsH/Wrc8O6O/38Hs", - "jH/N/uUFJKDTbXflSfHGW+/Kc+VvQQD0Q+9FoOO9ef3h+j3aPc3++9gBUcTiSarUGHyaomhYfY3kpJH+", - "JrcfmhdoeqHsGl+g4Htbcewqe5hFdykRBE3yhz+sDx5rwJ0se+FrE75qluNe+jpKX00SOkex8RyL71iO", - "xXdtm30ZvTm6njilh/BlCO8jwDBwklhmlZfZGmsCqiQZPN5BQFcrcHsLsD7cHur2wcImuG2Dd/UplEaM", - "X4uHQGQGItOxVunFnN8gZPO0VwzJ6TeYxqMCNIn8/CcWLK9N9SIh3QZlnihEyXp4pID4SrqmrfK95G28", - "0XQvxSxqgGKjIAe/xF8PRc4kuzwwRdc6n/L9wqtd7eTZw+QgegfxAzmIN0KwJTlMm6r6CMmrB9LbVVGl", - "2dMvZNkO4OAxjyeHj34VPCDEqhjY5yo4KD8haKXI8gSm+V6Z2nNNu4lx6QnDo0P45TYlO28G1HzSb3hX", - "UALMC+G9+J7/9hAGm+3FoGFlL+X8fQX4f66QPQn2ZCG8ZXzr4XBYdA/y1MZNOOcltBmrywifQZHatsd5", - "j/PirNMMCgPaWX5dPPjF/j1Ezi6W4HnrNMB9po23lGmDYcUCqZ3vftv8LfBhADqrvfz5Zg7v20uXX0C1", - "GmT+tN0uIqw6dPQS3PUuuYP0ouK6zU58i/s5k/yWX4B6eQGuQ85e6DtV+v3FHUE/Qzh82ll2+yTGHWW3", - "JDR14WVPc9AGuBhVtyy51wh/pGIA0nDw9AebP9FW7bHx6YS/zMnumDwnY6dsnhNRYpBKjHgnQyGQAknf", - "2gIS0QRQdJFooVBPjQ04wnfFSeYOj7zUNVaLbrNucwmjla7Fiuu1uT0ty56L+1zRXm7jb+43/x8AAP//", - "BTFjuznkAAA=", + "ccQRm+IVIOJyZMRLHsBShN2IGBqdn0BDCEYTg1JWrZ1DK8UbFuRXIlOlDM++W41M0/DHbO2b7MH63SeI", + "ouQZBlNACERxtzOExyjxf2xZ16+6HFr6mqi1dM3mE2Vj/BU+YC0H443H+Z4bNvvenpBXt+nwW3/VUkoP", + "LTxv7I61jeb1q45YqT1+cQgH8P04eL+kH3dFnGpTfJs9CkUpcgn4TGN+CxHJQOQkyPmaYoIgWKl6qsk1", + "NH/4xsBC2V7uFSrfzDGUF6TsySe02lpz6QqtdT9QG+dF9c0he3/O2p7ffiExi2u+PesUYNSiYbfzqNu/", + "Wm5VETvcd5ruMaX83ZruM7sDxHIZECpfHVNlUaDNNCHLGCbZ2x3Hsix2QSjd+cxo+dYbco3lYGsztNn7", + "tCLdI+UP9YRn0HkqFpNMKFTdGmLQ652eVvNc1Xm+gdBSemcRQSEdmLtSJoIhRHyDlqg8MUY1F2DAkqlg", + "J5yXXKWeAXYwT/oyzxjn4oSovtVfR6Px7a3ruVfDyfXXGe19PJt9mWm7V0MaNLtb8Cg8zrHO43x5+LCX", + "Gvw0MRktw3B8aV5UFmzwaE9uiW92hKJwsdA5pSkiIooUkzmc3U2uhqO7h9FsPLybsM1w/tvNl8vJ1WRU", + "+/1yfD1mv+kmvGK3GPbGGeIxa9qgMdnEFCU/dddPIONLop3ZVYqDa7O6ioC41pL1eLrN/cZjxNko0Tys", + "j72XlSEfqo85cmfQZfboeu4ow4Q9wDd8xmOfzh47qxnBmCBAd13T9TTUzoXVap4TXFO+nvvzXUkhvRMu", + "ioUapBOu8reepsomRQ3eIjMNtkhIk2GIDB66FSbkJekUlg31DgiWWxebO9SsAvIdgxHlUbXRVJ/BhTj2", + "lUV3yBuyB9MdxuAxKlnL+dN8ngsLJ1t7lal65uqOo5sxGMYY+hmCeoJCkTlX/5VvjdVkYOzlQesjdFFh", + "h1wqRxUzsax0WNzEOqSZJQsnTLsXJw1GJz+nEuccEoPK7N+bZYvPVL5xaROzT3d3UylrjqxXlbnHJNA7", + "gy8L8Nur9WbKi5RsHUkXFfdCe5GnzfBpJKIObFJ61EWowfipJa7T2rSz8d1sMvxwPX7gNi21cu+G1w9m", + "C7d2FWavgp2xQotWGdsqW7EaWRaHMuBDs7G0bAIVgmCt5PK3fpCCRXsVmScZRNvrVwSFsvoytx6oqEFV", + "hV79iwI2Vp+i+fKHHLfOaFN/DLBLuMFrW4Lf6tpXXc0kk0rLl2GJ06eyDON5IjNzCvcr8QCo+Vz3nRPA", + "JxhReGHRx4W7JCTFF4PB8/Pz2ZJXPQsTNrSQRM0NDqcTJUziwv3j7PzsnB2mpzAGaeheuP9kP/EjUMbX", + "AVKuNtNEtw6PxNuUeUdnLmuS60c69aKIevUJEFhBwmbRsJksigw0r/5u7tWXT9cmCJQeR62/C1p5OvMf", + "53+YGxLlBrUMyxvPfX9+3l5ReemNVbHoS5OE9/35P23rFblz/8eGPt37GCyRqgzqlTOtzjMBCzqFrrLt", + "uqeVctwMfqlPdG84fCJINFbRJftdAZIT8pBC4PtJFvOXUOn/F+ETjB0elVcGGm9ia6BpnyfnUCvBxIKb", + "Mt30K0DH+/P37ZXyVOf7g1Ntvk148twF1CieGSQZinEBFxHB2x02HyE5Bcy8RtVyLPCYJt+MoTTTYOgr", + "SxyNd1I67Bpy/RIA2vv61oNwryCso2eLJXEg7+kHxSWiVt9dh5hUA93qtlYtfA7vCZFea70ULOBn5jlp", + "W5rdpFuUxRAgf3kH0baq1fzcVg9vI7x1gFMAXgSKWOIby3S5Wnh/hKSSMfdMt1CXcu9eJWjPercdi3OU", + "rC4BgdYVSKIU3wq9+ofZe+QakVvH0i64/SX/stm+yNbPDJsTJT7qMHiVxPc7mkPtaJQp3gPmFLOgwYRt", + "Nwx4uSOZBiYQdrRwtZn/N7uo1N4Y6GTr7tMcUCC+f8vgmMjubYjehmgCe5El0QLuvHAz4It0iq/KoqjQ", + "34OyKyjzed8HLMXF0OCX+KOLsStfAmgzer8pOfhPVjnLlPW9vXyoG4C4BqSXwvRAyYrZrnyLM2Wj7lVe", + "GHhNiG6v4y/DKPgmK+6u5Dmjeh1vIxUUkI9Qh8MXEgrm+2wlG/q07FoR0aX6/g0FhWdB3kVEdIzqBaWD", + "oBjfCpDiUimwV6kpMpNbC02e/rtFZoo04b3I6ESG86cXlR1EJYfYIURFzURrLSxKXtsWcVEz4PYC07TG", + "SE71orOD6ChwO6Tw4K2kB9uLD34T2/PKexS9JOxBEl58HZmHEbTcu/OiDTv3K1HgN1sqXtAJJ0HkCwrs", + "GqaFr0IYBQdx7yle4OjleJsDBiksL3O8sITRyupwQfeuhlaG6481vI1Fqz7uHu8d8G54t0WivvR5j9C3", + "2vYYH/NoBP9r3fLsjP5+B7Mz/jX7lxeQgE633ZUHxRtvvSuPlb8FAdAPvReBjvfm9Wfr92j3NPvvYwdE", + "EYsnqVJj8GmKomH1LZKTRvqb3H5o3p/phbJrfIGC723FsavsYRbdpUQQNMkf/rA+eKwBd7Lsha9N+KpZ", + "jnvp6yh9NUnoHMXGcyy+YzkW37Vt9mX05uh64pSewZchvI8Aw8BJYplTXmZrrAmokmTweAcBXa3A7S3A", + "+nB7qNsHC5vgtg3e1YdQGjF+LZ4BkRmITMdapfdyfoOQzdNeMSSn32AajwrQJPLzn1iwvDbVi4R0G5R5", + "ohAl6+GRAuIr6Zq2yveSt/FG070Us6gBio2CHPwSfz0UOZPs8sAUXet8yvcLr3a1k2cPk4PoHcQP5CDe", + "CMGW5DBtquojJK8eSG9XRZVmT7+QZTuAg8c8nhw++lXwgBCrYmCfq+Cg/ICglSLLE5jme2VqzzXtJsal", + "BwyPDuGX25TsvBlQ80m/4V1BCTAvhPfie/7bQxhstheDhpW9lPP3FeD/uUL2JNiThfCW8a2Hw2HRPchT", + "GzfhnJfQZqwuI3wGRWrbHuc9zouzTjMoDGhn+XXx4Bf79xA5u1iC563TAPeZNt5Spg2GFQukdr77bfO3", + "wIcB6Kz28uebObxvL11+AdVqkPnTdruIsOrQ0Utw17vkDtKLius2O/Et7udM8lt+AerlBbgOOXuh71Tp", + "9xd3BP0M4fBpZ9ntkxh3lN2S0NSFlz3NQRvgYlTdsuReI/yRigFIw8HTH2z+RFu1x8anE/4yJ7tj8pyM", + "nbJ5TkSJQSox4p0MhUAKJH1rC0hEE0DRRaKFQj01NuAI3xUnmTs88lLXWC26zbrNJYxWuhYrrtfm9rQs", + "ey7uc0V7uY2/ud/8fwAAAP//9679LzfkAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/registry/app/api/openapi/contracts/artifact/types.gen.go b/registry/app/api/openapi/contracts/artifact/types.gen.go index 7cdaa8204..43510ffa5 100644 --- a/registry/app/api/openapi/contracts/artifact/types.gen.go +++ b/registry/app/api/openapi/contracts/artifact/types.gen.go @@ -1,6 +1,6 @@ // Package artifact provides primitives to interact with the openapi HTTP API. // -// Code generated by github.com/deepmap/oapi-codegen/v2 version v2.1.0 DO NOT EDIT. +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.4.1 DO NOT EDIT. package artifact import ( @@ -480,8 +480,7 @@ type PackageType string // PythonArtifactDetailConfig Config for python artifact details type PythonArtifactDetailConfig struct { - ArtifactId *string `json:"artifactId,omitempty"` - GroupId *string `json:"groupId,omitempty"` + Metadata *map[string]interface{} `json:"metadata,omitempty"` } // Registry Harness Artifact Registry diff --git a/registry/app/api/router/packages/route.go b/registry/app/api/router/packages/route.go index 8074cb568..0c532e520 100644 --- a/registry/app/api/router/packages/route.go +++ b/registry/app/api/router/packages/route.go @@ -21,7 +21,7 @@ import ( "github.com/harness/gitness/registry/app/api/handler/generic" "github.com/harness/gitness/registry/app/api/handler/maven" "github.com/harness/gitness/registry/app/api/handler/packages" - "github.com/harness/gitness/registry/app/api/handler/pypi" + "github.com/harness/gitness/registry/app/api/handler/python" "github.com/harness/gitness/registry/app/api/middleware" "github.com/harness/gitness/types/enum" @@ -42,7 +42,7 @@ func NewRouter( packageHandler packages.Handler, mavenHandler *maven.Handler, genericHandler *generic.Handler, - pypiHandler pypi.Handler, + pythonHandler python.Handler, ) Handler { r := chi.NewRouter() @@ -71,14 +71,20 @@ func NewRouter( r.Route("/python", func(r chi.Router) { r.Use(middlewareauthn.Attempt(packageHandler.GetAuthenticator())) - r.With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsUpload)). - Post("/*", pypiHandler.UploadPackageFile) - r.With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)). - Get("/files/{image}/{version}/{filename}", pypiHandler.DownloadPackageFile) - r.With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)). - Get("/simple/{image}", pypiHandler.PackageMetadata) - r.With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)). - Get("/simple/{image}/", pypiHandler.PackageMetadata) + + // TODO (Arvind): Move this to top layer with total abstraction + r.With(middleware.StoreArtifactInfo(pythonHandler)). + With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsUpload)). + Post("/*", pythonHandler.UploadPackageFile) + r.With(middleware.StoreArtifactInfo(pythonHandler)). + With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)). + Get("/files/{image}/{version}/{filename}", pythonHandler.DownloadPackageFile) + r.With(middleware.StoreArtifactInfo(pythonHandler)). + With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)). + Get("/simple/{image}", pythonHandler.PackageMetadata) + r.With(middleware.StoreArtifactInfo(pythonHandler)). + With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)). + Get("/simple/{image}/", pythonHandler.PackageMetadata) }) }) diff --git a/registry/app/api/router/wire.go b/registry/app/api/router/wire.go index 216800ef3..1053a8e92 100644 --- a/registry/app/api/router/wire.go +++ b/registry/app/api/router/wire.go @@ -26,7 +26,7 @@ import ( "github.com/harness/gitness/registry/app/api/handler/maven" hoci "github.com/harness/gitness/registry/app/api/handler/oci" "github.com/harness/gitness/registry/app/api/handler/packages" - "github.com/harness/gitness/registry/app/api/handler/pypi" + "github.com/harness/gitness/registry/app/api/handler/python" generic2 "github.com/harness/gitness/registry/app/api/router/generic" "github.com/harness/gitness/registry/app/api/router/harness" mavenRouter "github.com/harness/gitness/registry/app/api/router/maven" @@ -112,7 +112,7 @@ func PackageHandlerProvider( handler packages.Handler, mavenHandler *maven.Handler, genericHandler *generic.Handler, - pypiHandler pypi.Handler, + pypiHandler python.Handler, ) packagerrouter.Handler { return packagerrouter.NewRouter(handler, mavenHandler, genericHandler, pypiHandler) } diff --git a/registry/app/api/wire.go b/registry/app/api/wire.go index 3d17b8f9b..21abacd1f 100644 --- a/registry/app/api/wire.go +++ b/registry/app/api/wire.go @@ -21,22 +21,24 @@ import ( "github.com/harness/gitness/app/services/refcache" corestore "github.com/harness/gitness/app/store" urlprovider "github.com/harness/gitness/app/url" + python2 "github.com/harness/gitness/registry/app/api/controller/pkg/python" "github.com/harness/gitness/registry/app/api/handler/generic" mavenhandler "github.com/harness/gitness/registry/app/api/handler/maven" ocihandler "github.com/harness/gitness/registry/app/api/handler/oci" "github.com/harness/gitness/registry/app/api/handler/packages" - pypi2 "github.com/harness/gitness/registry/app/api/handler/pypi" + pypi2 "github.com/harness/gitness/registry/app/api/handler/python" "github.com/harness/gitness/registry/app/api/router" storagedriver "github.com/harness/gitness/registry/app/driver" "github.com/harness/gitness/registry/app/driver/factory" "github.com/harness/gitness/registry/app/driver/filesystem" "github.com/harness/gitness/registry/app/driver/s3-aws" "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/base" "github.com/harness/gitness/registry/app/pkg/docker" "github.com/harness/gitness/registry/app/pkg/filemanager" generic2 "github.com/harness/gitness/registry/app/pkg/generic" "github.com/harness/gitness/registry/app/pkg/maven" - "github.com/harness/gitness/registry/app/pkg/pypi" + "github.com/harness/gitness/registry/app/pkg/python" "github.com/harness/gitness/registry/app/store" "github.com/harness/gitness/registry/app/store/database" "github.com/harness/gitness/registry/config" @@ -124,8 +126,8 @@ func NewPackageHandlerProvider( ) } -func NewPypiHandlerProvider( - controller pypi.Controller, +func NewPythonHandlerProvider( + controller python2.Controller, packageHandler packages.Handler, ) pypi2.Handler { return pypi2.NewHandler(controller, packageHandler) @@ -153,16 +155,18 @@ var WireSet = wire.NewSet( NewMavenHandlerProvider, NewGenericHandlerProvider, NewPackageHandlerProvider, - NewPypiHandlerProvider, + NewPythonHandlerProvider, database.WireSet, pkg.WireSet, docker.WireSet, filemanager.WireSet, maven.WireSet, - pypi.WireSet, + python.WireSet, router.WireSet, gc.WireSet, generic2.WireSet, + python2.ControllerSet, + base.WireSet, ) func Wire(_ *types.Config) (RegistryApp, error) { diff --git a/registry/app/metadata/file.go b/registry/app/metadata/file.go new file mode 100644 index 000000000..4654e13da --- /dev/null +++ b/registry/app/metadata/file.go @@ -0,0 +1,21 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +type File struct { + Size int64 `json:"size"` + Filename string `json:"file_name"` + CreatedAt int64 `json:"created_at"` +} diff --git a/registry/app/metadata/pkg.go b/registry/app/metadata/pkg.go new file mode 100644 index 000000000..2bff1981e --- /dev/null +++ b/registry/app/metadata/pkg.go @@ -0,0 +1,26 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +type GenericMetadata struct { + Files []File `json:"files"` + Description string `json:"desc"` + FileCount int64 `json:"file_count"` +} + +type MavenMetadata struct { + Files []File `json:"files"` + FileCount int64 `json:"file_count"` +} diff --git a/registry/app/metadata/provider.go b/registry/app/metadata/provider.go new file mode 100644 index 000000000..e05250dce --- /dev/null +++ b/registry/app/metadata/provider.go @@ -0,0 +1,20 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +type Metadata interface { + GetFiles() []File + SetFiles([]File) +} diff --git a/registry/app/metadata/pypi/metadata.go b/registry/app/metadata/python/metadata.go similarity index 86% rename from registry/app/metadata/pypi/metadata.go rename to registry/app/metadata/python/metadata.go index 7819e4ce9..39783f9e3 100644 --- a/registry/app/metadata/pypi/metadata.go +++ b/registry/app/metadata/python/metadata.go @@ -12,7 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pypi +package python + +import "github.com/harness/gitness/registry/app/metadata" + +var _ metadata.Metadata = (*PythonMetadata)(nil) // Metadata Source: https://github.com/pypa/twine/blob/main/twine/package.py type Metadata struct { @@ -73,3 +77,21 @@ type Metadata struct { ProjectURL string `json:"project_url,omitempty"` Dependencies []string `json:"dependencies,omitempty"` } + +// PythonMetadata represents the metadata for a Python package. +// +//nolint:revive +type PythonMetadata struct { + Metadata + Files []metadata.File `json:"files"` + FileCount int64 `json:"file_count"` +} + +func (p *PythonMetadata) GetFiles() []metadata.File { + return p.Files +} + +func (p *PythonMetadata) SetFiles(files []metadata.File) { + p.Files = files + p.FileCount = int64(len(files)) +} diff --git a/registry/app/pkg/artifact.go b/registry/app/pkg/artifact.go index f94d4f5ad..a9cc90d4d 100644 --- a/registry/app/pkg/artifact.go +++ b/registry/app/pkg/artifact.go @@ -14,6 +14,10 @@ package pkg +import "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" + +// Artifact Fixme: Name change to Registry as it provides Registry Type type Artifact interface { - GetArtifactType() string + GetArtifactType() artifact.RegistryType + GetPackageTypes() []artifact.PackageType } diff --git a/registry/app/pkg/artifact_info.go b/registry/app/pkg/artifact_info.go new file mode 100644 index 000000000..e4af160b4 --- /dev/null +++ b/registry/app/pkg/artifact_info.go @@ -0,0 +1,34 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pkg + +import ( + "net/http" +) + +// PackageArtifactInfo is an interface that must be implemented by all package-specific +// artifact info types. It ensures that all package artifact infos can be converted to +// the base ArtifactInfo type. +type PackageArtifactInfo interface { + BaseArtifactInfo() ArtifactInfo +} + +// ArtifactInfoProvider is an interface that must be implemented by package handlers +// to provide artifact information from HTTP requests. +type ArtifactInfoProvider interface { + // GetPackageArtifactInfo returns package-specific artifact info that implements + // the PackageArtifactInfo interface + GetPackageArtifactInfo(r *http.Request) (PackageArtifactInfo, error) +} diff --git a/registry/app/pkg/base/base.go b/registry/app/pkg/base/base.go new file mode 100644 index 000000000..39bc5406f --- /dev/null +++ b/registry/app/pkg/base/base.go @@ -0,0 +1,263 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package base + +import ( + "context" + "encoding/json" + "fmt" + "mime/multipart" + "net/http" + "strings" + "time" + + "github.com/harness/gitness/registry/app/dist_temp/errcode" + "github.com/harness/gitness/registry/app/metadata" + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/filemanager" + "github.com/harness/gitness/registry/app/storage" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/registry/types" + "github.com/harness/gitness/store/database/dbtx" +) + +var _ LocalBase = (*localBase)(nil) + +type LocalBase interface { + Upload( + ctx context.Context, + info pkg.ArtifactInfo, + fileName string, + version string, + path string, + file multipart.File, + metadata metadata.Metadata, + ) ( + // TODO: Check the scope if we should remove the response message / headers setup here or + // each package implementation should have their own. + headers *commons.ResponseHeaders, sha256 string, err errcode.Error, + ) + Download(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) ( + *commons.ResponseHeaders, + *storage.FileReader, + string, + []error, + ) +} + +type localBase struct { + registryDao store.RegistryRepository + fileManager filemanager.FileManager + tx dbtx.Transactor + imageDao store.ImageRepository + artifactDao store.ArtifactRepository +} + +func NewLocalBase( + registryDao store.RegistryRepository, + fileManager filemanager.FileManager, + tx dbtx.Transactor, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, +) LocalBase { + return &localBase{ + registryDao: registryDao, + fileManager: fileManager, + tx: tx, + imageDao: imageDao, + artifactDao: artifactDao, + } +} + +func (l *localBase) Upload( + ctx context.Context, + info pkg.ArtifactInfo, + fileName string, + version string, + path string, + file multipart.File, + // TODO: Metadata shouldn't be provided as a parameter, it should be fetched or created. + metadata metadata.Metadata, +) (*commons.ResponseHeaders, string, errcode.Error) { + responseHeaders := &commons.ResponseHeaders{ + Headers: make(map[string]string), + Code: 0, + } + + err := l.CheckIfFileAlreadyExist(ctx, info, version, metadata, fileName) + + if err != nil { + return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(err) + } + + registry, err := l.registryDao.GetByRootParentIDAndName(ctx, info.RootParentID, info.RegIdentifier) + if err != nil { + return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err) + } + fileInfo, err := l.fileManager.UploadFile(ctx, path, info.RegIdentifier, registry.ID, + info.RootParentID, info.RootIdentifier, file, nil, fileName) + if err != nil { + return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err) + } + err = l.tx.WithTx( + ctx, func(ctx context.Context) error { + image := &types.Image{ + Name: info.Image, + RegistryID: registry.ID, + Enabled: true, + } + err := l.imageDao.CreateOrUpdate(ctx, image) + if err != nil { + return fmt.Errorf("failed to create image for artifact: [%s], error: %w", info.Image, err) + } + + dbArtifact, err := l.artifactDao.GetByName(ctx, image.ID, version) + + if err != nil && !strings.Contains(err.Error(), "resource not found") { + return fmt.Errorf("failed to fetch artifact : [%s] with error: %w", info.Image, err) + } + + err2 := l.updateMetadata(dbArtifact, metadata, info, fileInfo) + if err2 != nil { + return fmt.Errorf("failed to update metadata for artifact: [%s] with error: %w", info.Image, err2) + } + + metadataJSON, err := json.Marshal(metadata) + + if err != nil { + return fmt.Errorf("failed to parse metadata for artifact: [%s] with error: %w", info.Image, err) + } + + err = l.artifactDao.CreateOrUpdate(ctx, &types.Artifact{ + ImageID: image.ID, + Version: version, + Metadata: metadataJSON, + }) + if err != nil { + return fmt.Errorf("failed to create artifact : [%s] with error: %w", info.Image, err) + } + return nil + }) + + if err != nil { + return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err) + } + responseHeaders.Code = http.StatusCreated + return responseHeaders, fileInfo.Sha256, errcode.Error{} +} + +func (l *localBase) Download(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) ( + *commons.ResponseHeaders, + *storage.FileReader, + string, + []error, +) { + responseHeaders := &commons.ResponseHeaders{ + Headers: make(map[string]string), + Code: 0, + } + + path := "/" + info.Image + "/" + version + "/" + fileName + + fileReader, _, redirectURL, err := l.fileManager.DownloadFile(ctx, path, types.Registry{ + ID: info.RegistryID, + Name: info.RegIdentifier, + }, info.RootIdentifier) + if err != nil { + return responseHeaders, nil, "", []error{err} + } + responseHeaders.Code = http.StatusOK + return responseHeaders, fileReader, redirectURL, nil +} + +func (l *localBase) updateMetadata( + dbArtifact *types.Artifact, + inputMetadata metadata.Metadata, + info pkg.ArtifactInfo, + fileInfo pkg.FileInfo, +) error { + var files []metadata.File + if dbArtifact != nil { + err := json.Unmarshal(dbArtifact.Metadata, inputMetadata) + if err != nil { + return fmt.Errorf("failed to get metadata for artifact: [%s] with registry: [%s] and error: %w", info.Image, + info.RegIdentifier, err) + } + fileExist := false + files = inputMetadata.GetFiles() + for _, file := range files { + if file.Filename == fileInfo.Filename { + fileExist = true + } + } + if !fileExist { + files = append(files, metadata.File{ + Size: fileInfo.Size, Filename: fileInfo.Filename, + CreatedAt: time.Now().UnixMilli(), + }) + inputMetadata.SetFiles(files) + } + } else { + files = append(files, metadata.File{ + Size: fileInfo.Size, Filename: fileInfo.Filename, + CreatedAt: time.Now().UnixMilli(), + }) + inputMetadata.SetFiles(files) + } + return nil +} + +func (l *localBase) CheckIfFileAlreadyExist( + ctx context.Context, + info pkg.ArtifactInfo, + version string, + metadata metadata.Metadata, + fileName string, +) error { + image, err := l.imageDao.GetByName(ctx, info.RegistryID, info.Image) + if err != nil && !strings.Contains(err.Error(), "resource not found") { + return fmt.Errorf("failed to fetch the image for artifact : [%s] with registry : [%s]", info.Image, + info.RegIdentifier) + } + if image == nil { + return nil + } + + dbArtifact, err := l.artifactDao.GetByName(ctx, image.ID, version) + + if err != nil && !strings.Contains(err.Error(), "resource not found") { + return fmt.Errorf("failed to fetch artifact : [%s] with registry : [%s]", info.Image, info.RegIdentifier) + } + + if dbArtifact == nil { + return nil + } + + err = json.Unmarshal(dbArtifact.Metadata, metadata) + if err != nil { + return fmt.Errorf("failed to get metadata for artifact: [%s] with registry: [%s] and error: %w", info.Image, + info.RegIdentifier, err) + } + + for _, file := range metadata.GetFiles() { + if file.Filename == fileName { + return fmt.Errorf("file: [%s] with Artifact: [%s], Version: [%s] and registry: [%s] already exist", + fileName, info.Image, version, info.RegIdentifier) + } + } + + return nil +} diff --git a/registry/app/pkg/base/wire.go b/registry/app/pkg/base/wire.go new file mode 100644 index 000000000..9cb452e18 --- /dev/null +++ b/registry/app/pkg/base/wire.go @@ -0,0 +1,35 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package base + +import ( + "github.com/harness/gitness/registry/app/pkg/filemanager" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/store/database/dbtx" + + "github.com/google/wire" +) + +func LocalBaseProvider( + registryDao store.RegistryRepository, + fileManager filemanager.FileManager, + tx dbtx.Transactor, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, +) LocalBase { + return NewLocalBase(registryDao, fileManager, tx, imageDao, artifactDao) +} + +var WireSet = wire.NewSet(LocalBaseProvider) diff --git a/registry/app/pkg/base/wrapper.go b/registry/app/pkg/base/wrapper.go new file mode 100644 index 000000000..77ccd8411 --- /dev/null +++ b/registry/app/pkg/base/wrapper.go @@ -0,0 +1,123 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package base + +import ( + "context" + + "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/response" + "github.com/harness/gitness/registry/app/store" + registrytypes "github.com/harness/gitness/registry/types" + + "github.com/rs/zerolog/log" +) + +var TypeRegistry = map[string]pkg.Artifact{} + +func Register(registries ...pkg.Artifact) { + for _, r := range registries { + for _, packageType := range r.GetPackageTypes() { + log.Info().Msgf("Registering package type %s with artifact type %s", packageType, r.GetArtifactType()) + key := getFactoryKey(packageType, r.GetArtifactType()) + TypeRegistry[key] = r + } + } +} + +func NoProxyWrapper( + ctx context.Context, + registryDao store.RegistryRepository, + f func(registry registrytypes.Registry, a pkg.Artifact) response.Response, + info pkg.ArtifactInfo, +) response.Response { + var result response.Response + registry, err := registryDao.Get(ctx, info.RegistryID) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msgf("Failed to get registry by ID %d", info.RegistryID) + return result + } + + log.Ctx(ctx).Info().Msgf("Using Repository: %s, Type: %s", registry.Name, registry.Type) + art := getArtifactRegistry(*registry) + result = f(*registry, art) + if pkg.IsEmpty(result.GetErrors()) { + return result + } + log.Ctx(ctx).Warn().Msgf("Repository: %s, Type: %s, errors: %v", registry.Name, registry.Type, + result.GetErrors()) + + return result +} + +func ProxyWrapper( + ctx context.Context, + registryDao store.RegistryRepository, + f func(registry registrytypes.Registry, a pkg.Artifact) response.Response, + info pkg.ArtifactInfo, +) response.Response { + var response response.Response + requestRepoKey := info.RegIdentifier + if repos, err := getOrderedRepos(ctx, registryDao, requestRepoKey, *info.BaseInfo); err == nil { + for _, registry := range repos { + log.Ctx(ctx).Info().Msgf("Using Repository: %s, Type: %s", registry.Name, registry.Type) + reg := getArtifactRegistry(registry) + if reg != nil { + response = f(registry, reg) + if pkg.IsEmpty(response.GetErrors()) { + return response + } + log.Ctx(ctx).Warn().Msgf("Repository: %s, Type: %s, errors: %v", registry.Name, registry.Type, + response.GetErrors()) + } + } + } + return response +} + +func factory(key string) pkg.Artifact { + return TypeRegistry[key] +} + +func getFactoryKey(packageType artifact.PackageType, registryType artifact.RegistryType) string { + return string(packageType) + ":" + string(registryType) +} + +func getArtifactRegistry(registry registrytypes.Registry) pkg.Artifact { + key := getFactoryKey(registry.PackageType, registry.Type) + return factory(key) +} + +func getOrderedRepos( + ctx context.Context, + registryDao store.RegistryRepository, + repoKey string, + artInfo pkg.BaseInfo, +) ([]registrytypes.Registry, error) { + var result []registrytypes.Registry + if registry, err := registryDao.GetByParentIDAndName(ctx, artInfo.ParentID, repoKey); err == nil { + result = append(result, *registry) + proxies := registry.UpstreamProxies + if len(proxies) > 0 { + upstreamRepos, _ := registryDao.GetByIDIn(ctx, proxies) + result = append(result, *upstreamRepos...) + } + } else { + return result, err + } + + return result, nil +} diff --git a/registry/app/pkg/context.go b/registry/app/pkg/context.go index 88c37d618..464355590 100644 --- a/registry/app/pkg/context.go +++ b/registry/app/pkg/context.go @@ -33,6 +33,7 @@ type BaseInfo struct { type ArtifactInfo struct { *BaseInfo RegIdentifier string + RegistryID int64 Image string } diff --git a/registry/app/pkg/docker/local.go b/registry/app/pkg/docker/local.go index 6ea32155b..0de40596b 100644 --- a/registry/app/pkg/docker/local.go +++ b/registry/app/pkg/docker/local.go @@ -35,6 +35,7 @@ import ( "time" "github.com/harness/gitness/app/paths" + "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" "github.com/harness/gitness/registry/app/dist_temp/dcontext" "github.com/harness/gitness/registry/app/dist_temp/errcode" "github.com/harness/gitness/registry/app/manifest" @@ -157,8 +158,12 @@ func (r *LocalRegistry) CanBeMount() (mount bool, repository string, err error) panic("implement me") } -func (r *LocalRegistry) GetArtifactType() string { - return ArtifactTypeLocalRegistry +func (r *LocalRegistry) GetArtifactType() artifact.RegistryType { + return artifact.RegistryTypeVIRTUAL +} + +func (r *LocalRegistry) GetPackageTypes() []artifact.PackageType { + return []artifact.PackageType{artifact.PackageTypeDOCKER, artifact.PackageTypeHELM} } func (r *LocalRegistry) getManifest( diff --git a/registry/app/pkg/docker/remote.go b/registry/app/pkg/docker/remote.go index f82b8247b..0101f6962 100644 --- a/registry/app/pkg/docker/remote.go +++ b/registry/app/pkg/docker/remote.go @@ -25,6 +25,7 @@ import ( "github.com/harness/gitness/app/api/request" "github.com/harness/gitness/app/services/refcache" + "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" "github.com/harness/gitness/registry/app/common/lib/errors" "github.com/harness/gitness/registry/app/manifest" "github.com/harness/gitness/registry/app/manifest/manifestlist" @@ -75,8 +76,12 @@ func NewRemoteRegistry( } } -func (r *RemoteRegistry) GetArtifactType() string { - return "Remote Registry" +func (r *RemoteRegistry) GetArtifactType() artifact.RegistryType { + return artifact.RegistryTypeUPSTREAM +} + +func (r *RemoteRegistry) GetPackageTypes() []artifact.PackageType { + return []artifact.PackageType{artifact.PackageTypeDOCKER, artifact.PackageTypeHELM} } type RemoteRegistry struct { diff --git a/registry/app/pkg/generic/controller.go b/registry/app/pkg/generic/controller.go index 40e072791..abdf6abc2 100644 --- a/registry/app/pkg/generic/controller.go +++ b/registry/app/pkg/generic/controller.go @@ -26,12 +26,12 @@ import ( "github.com/harness/gitness/app/auth/authz" corestore "github.com/harness/gitness/app/store" "github.com/harness/gitness/registry/app/dist_temp/errcode" + "github.com/harness/gitness/registry/app/metadata" "github.com/harness/gitness/registry/app/pkg" "github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/filemanager" "github.com/harness/gitness/registry/app/storage" "github.com/harness/gitness/registry/app/store" - "github.com/harness/gitness/registry/app/store/database" "github.com/harness/gitness/registry/types" "github.com/harness/gitness/store/database/dbtx" "github.com/harness/gitness/types/enum" @@ -136,7 +136,7 @@ func (c Controller) UploadArtifact( regNameFormat, info.Image, info.RegIdentifier) } - metadata := &database.GenericMetadata{ + metadata := &metadata.GenericMetadata{ Description: info.Description, } err2 := c.updateMetadata(dbArtifact, metadata, info, fileInfo) @@ -172,38 +172,38 @@ func (c Controller) UploadArtifact( } func (c Controller) updateMetadata( - dbArtifact *types.Artifact, metadata *database.GenericMetadata, + dbArtifact *types.Artifact, metadataInput *metadata.GenericMetadata, info pkg.GenericArtifactInfo, fileInfo pkg.FileInfo, ) error { - var files []database.File + var files []metadata.File if dbArtifact != nil { - err := json.Unmarshal(dbArtifact.Metadata, metadata) + err := json.Unmarshal(dbArtifact.Metadata, metadataInput) if err != nil { return fmt.Errorf("failed to get metadata for artifact : [%s] with registry : [%s]", info.Image, info.RegIdentifier) } fileExist := false - files = metadata.Files + files = metadataInput.Files for _, file := range files { if file.Filename == info.FileName { fileExist = true } } if !fileExist { - files = append(files, database.File{ + files = append(files, metadata.File{ Size: fileInfo.Size, Filename: fileInfo.Filename, CreatedAt: time.Now().UnixMilli(), }) - metadata.Files = files - metadata.FileCount++ + metadataInput.Files = files + metadataInput.FileCount++ } } else { - files = append(files, database.File{ + files = append(files, metadata.File{ Size: fileInfo.Size, Filename: fileInfo.Filename, CreatedAt: time.Now().UnixMilli(), }) - metadata.Files = files - metadata.FileCount++ + metadataInput.Files = files + metadataInput.FileCount++ } return nil } @@ -257,7 +257,7 @@ func (c Controller) CheckIfFileAlreadyExist(ctx context.Context, info pkg.Generi return nil } - metadata := &database.GenericMetadata{} + metadata := &metadata.GenericMetadata{} err = json.Unmarshal(dbArtifact.Metadata, metadata) diff --git a/registry/app/pkg/maven/local.go b/registry/app/pkg/maven/local.go index 6861e787f..b3d905c5f 100644 --- a/registry/app/pkg/maven/local.go +++ b/registry/app/pkg/maven/local.go @@ -24,12 +24,12 @@ import ( "time" "github.com/harness/gitness/registry/app/dist_temp/errcode" + "github.com/harness/gitness/registry/app/metadata" "github.com/harness/gitness/registry/app/pkg" "github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/filemanager" "github.com/harness/gitness/registry/app/pkg/maven/utils" "github.com/harness/gitness/registry/app/storage" - "github.com/harness/gitness/registry/app/store/database" "github.com/harness/gitness/registry/types" "github.com/harness/gitness/store/database/dbtx" ) @@ -38,7 +38,8 @@ const ( ArtifactTypeLocalRegistry = "Local Registry" ) -func NewLocalRegistry(dBStore *DBStore, tx dbtx.Transactor, +func NewLocalRegistry( + dBStore *DBStore, tx dbtx.Transactor, fileManager filemanager.FileManager, ) Registry { @@ -60,14 +61,16 @@ func (r *LocalRegistry) GetMavenArtifactType() string { } func (r *LocalRegistry) HeadArtifact(ctx context.Context, info pkg.MavenArtifactInfo) ( - responseHeaders *commons.ResponseHeaders, errs []error) { + responseHeaders *commons.ResponseHeaders, errs []error, +) { responseHeaders, _, _, _, errs = r.FetchArtifact(ctx, info, false) return responseHeaders, errs } func (r *LocalRegistry) GetArtifact(ctx context.Context, info pkg.MavenArtifactInfo) ( responseHeaders *commons.ResponseHeaders, body *storage.FileReader, readCloser io.ReadCloser, - redirectURL string, errs []error) { + redirectURL string, errs []error, +) { return r.FetchArtifact(ctx, info, true) } @@ -76,7 +79,8 @@ func (r *LocalRegistry) FetchArtifact(ctx context.Context, info pkg.MavenArtifac body *storage.FileReader, readCloser io.ReadCloser, redirectURL string, - errs []error) { + errs []error, +) { filePath := utils.GetFilePath(info) name := info.GroupID + ":" + info.ArtifactID dbImage, err2 := r.DBStore.ImageDao.GetByName(ctx, info.RegistryID, name) @@ -115,7 +119,8 @@ func (r *LocalRegistry) FetchArtifact(ctx context.Context, info pkg.MavenArtifac } func (r *LocalRegistry) PutArtifact(ctx context.Context, info pkg.MavenArtifactInfo, fileReader io.Reader) ( - responseHeaders *commons.ResponseHeaders, errs []error) { + responseHeaders *commons.ResponseHeaders, errs []error, +) { filePath := utils.GetFilePath(info) fileInfo, err := r.fileManager.UploadFile(ctx, filePath, info.RegIdentifier, info.RegistryID, info.RootParentID, info.RootIdentifier, nil, fileReader, info.FileName) @@ -140,7 +145,7 @@ func (r *LocalRegistry) PutArtifact(ctx context.Context, info pkg.MavenArtifactI return nil } - metadata := &database.MavenMetadata{} + metadata := &metadata.MavenMetadata{} dbArtifact, err3 := r.DBStore.ArtifactDao.GetByName(ctx, dbImage.ID, info.Version) @@ -183,39 +188,46 @@ func (r *LocalRegistry) PutArtifact(ctx context.Context, info pkg.MavenArtifactI return responseHeaders, nil } -func (r *LocalRegistry) updateArtifactMetadata(dbArtifact *types.Artifact, metadata *database.MavenMetadata, - info pkg.MavenArtifactInfo, fileInfo pkg.FileInfo) error { - var files []database.File +func (r *LocalRegistry) updateArtifactMetadata( + dbArtifact *types.Artifact, mavenMetadata *metadata.MavenMetadata, + info pkg.MavenArtifactInfo, fileInfo pkg.FileInfo, +) error { + var files []metadata.File if dbArtifact != nil { - err := json.Unmarshal(dbArtifact.Metadata, metadata) + err := json.Unmarshal(dbArtifact.Metadata, mavenMetadata) if err != nil { return err } fileExist := false - files = metadata.Files + files = mavenMetadata.Files for _, file := range files { if file.Filename == info.FileName { fileExist = true } } if !fileExist { - files = append(files, database.File{Size: fileInfo.Size, Filename: fileInfo.Filename, - CreatedAt: time.Now().UnixMilli()}) - metadata.Files = files - metadata.FileCount++ + files = append(files, metadata.File{ + Size: fileInfo.Size, Filename: fileInfo.Filename, + CreatedAt: time.Now().UnixMilli(), + }) + mavenMetadata.Files = files + mavenMetadata.FileCount++ } } else { - files = append(files, database.File{Size: fileInfo.Size, Filename: fileInfo.Filename, - CreatedAt: time.Now().UnixMilli()}) - metadata.Files = files - metadata.FileCount++ + files = append(files, metadata.File{ + Size: fileInfo.Size, Filename: fileInfo.Filename, + CreatedAt: time.Now().UnixMilli(), + }) + mavenMetadata.Files = files + mavenMetadata.FileCount++ } return nil } func processError(err error) ( responseHeaders *commons.ResponseHeaders, body *storage.FileReader, readCloser io.ReadCloser, - redirectURL string, errs []error) { + redirectURL string, errs []error, +) { if strings.Contains(err.Error(), sql.ErrNoRows.Error()) || strings.Contains(err.Error(), "resource not found") || strings.Contains(err.Error(), "http status code: 404") { diff --git a/registry/app/pkg/maven/registry.go b/registry/app/pkg/maven/registry.go index 7c719e830..646b5570b 100644 --- a/registry/app/pkg/maven/registry.go +++ b/registry/app/pkg/maven/registry.go @@ -27,12 +27,15 @@ type Registry interface { Artifact HeadArtifact(ctx context.Context, artInfo pkg.MavenArtifactInfo) ( - responseHeaders *commons.ResponseHeaders, errs []error) + responseHeaders *commons.ResponseHeaders, errs []error, + ) GetArtifact(ctx context.Context, artInfo pkg.MavenArtifactInfo) ( responseHeaders *commons.ResponseHeaders, body *storage.FileReader, readCloser io.ReadCloser, - redirectURL string, errs []error) + redirectURL string, errs []error, + ) PutArtifact(ctx context.Context, artInfo pkg.MavenArtifactInfo, fileReader io.Reader) ( - responseHeaders *commons.ResponseHeaders, errs []error) + responseHeaders *commons.ResponseHeaders, errs []error, + ) } diff --git a/registry/app/pkg/maven/remote.go b/registry/app/pkg/maven/remote.go index 7ee23f8b8..a212c2123 100644 --- a/registry/app/pkg/maven/remote.go +++ b/registry/app/pkg/maven/remote.go @@ -31,7 +31,8 @@ const ( ArtifactTypeRemoteRegistry = "Remote Registry" ) -func NewRemoteRegistry(dBStore *DBStore, tx dbtx.Transactor, local *LocalRegistry, +func NewRemoteRegistry( + dBStore *DBStore, tx dbtx.Transactor, local *LocalRegistry, proxyController maven.Controller, ) Registry { return &RemoteRegistry{ @@ -54,25 +55,30 @@ func (r *RemoteRegistry) GetMavenArtifactType() string { } func (r *RemoteRegistry) HeadArtifact(ctx context.Context, info pkg.MavenArtifactInfo) ( - responseHeaders *commons.ResponseHeaders, errs []error) { - responseHeaders, _, _, _, errs = r.FetchArtifact(ctx, info, false) + responseHeaders *commons.ResponseHeaders, errs []error, +) { + responseHeaders, _, _, _, errs = r.fetchArtifact(ctx, info, false) return responseHeaders, errs } func (r *RemoteRegistry) GetArtifact(ctx context.Context, info pkg.MavenArtifactInfo) ( responseHeaders *commons.ResponseHeaders, body *storage.FileReader, readCloser io.ReadCloser, - redirectURL string, errs []error) { - return r.FetchArtifact(ctx, info, true) + redirectURL string, errs []error, +) { + return r.fetchArtifact(ctx, info, true) } -func (r *RemoteRegistry) PutArtifact(_ context.Context, _ pkg.MavenArtifactInfo, _ io.Reader) ( - responseHeaders *commons.ResponseHeaders, errs []error) { +func (r *RemoteRegistry) PutArtifact(ctx context.Context, _ pkg.MavenArtifactInfo, _ io.Reader) ( + responseHeaders *commons.ResponseHeaders, errs []error, +) { + log.Error().Ctx(ctx).Msg("Not implemented") return nil, nil } -func (r *RemoteRegistry) FetchArtifact(ctx context.Context, info pkg.MavenArtifactInfo, serveFile bool) ( +func (r *RemoteRegistry) fetchArtifact(ctx context.Context, info pkg.MavenArtifactInfo, serveFile bool) ( responseHeaders *commons.ResponseHeaders, body *storage.FileReader, readCloser io.ReadCloser, - redirectURL string, errs []error) { + redirectURL string, errs []error, +) { log.Ctx(ctx).Info().Msgf("Maven Proxy: %s", info.RegIdentifier) responseHeaders, body, redirectURL, useLocal := r.proxyController.UseLocalFile(ctx, info) diff --git a/registry/app/pkg/pypi/download.go b/registry/app/pkg/pypi/download.go deleted file mode 100644 index 660425c9b..000000000 --- a/registry/app/pkg/pypi/download.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2023 Harness, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pypi - -import ( - "context" - "net/http" - - "github.com/harness/gitness/registry/app/dist_temp/errcode" - "github.com/harness/gitness/registry/app/pkg" - "github.com/harness/gitness/registry/app/pkg/commons" - "github.com/harness/gitness/registry/app/storage" - "github.com/harness/gitness/registry/types" -) - -func (c *controller) DownloadPackageFile(ctx context.Context, info pkg.ArtifactInfo, image, version, filename string) ( - *commons.ResponseHeaders, - *storage.FileReader, - string, - errcode.Error, -) { - responseHeaders := &commons.ResponseHeaders{ - Headers: make(map[string]string), - Code: 0, - } - - path := "/" + image + "/" + version + "/" + filename - reg, _ := c.registryDao.GetByRootParentIDAndName(ctx, info.RootParentID, info.RegIdentifier) - - fileReader, _, redirectURL, err := c.fileManager.DownloadFile(ctx, path, types.Registry{ - ID: reg.ID, - Name: info.RegIdentifier, - }, info.RootIdentifier) - if err != nil { - return responseHeaders, nil, "", errcode.ErrCodeRootNotFound.WithDetail(err) - } - responseHeaders.Code = http.StatusOK - return responseHeaders, fileReader, redirectURL, errcode.Error{} -} diff --git a/registry/app/pkg/pypi/metadata.go b/registry/app/pkg/pypi/metadata.go deleted file mode 100644 index 7e987cb0f..000000000 --- a/registry/app/pkg/pypi/metadata.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2023 Harness, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pypi - -import ( - "context" - "encoding/json" - "fmt" - "sort" - - "github.com/harness/gitness/registry/app/store/database" -) - -// Metadata represents the metadata of a PyPI package. -func (c *controller) GetPackageMetadata(ctx context.Context, info ArtifactInfo, packageName string) ( - PackageMetadata, - error, -) { - registry, err := c.registryDao.GetByRootParentIDAndName(ctx, info.RootParentID, info.RegIdentifier) - packageMetadata := PackageMetadata{} - packageMetadata.Name = packageName - packageMetadata.Files = []File{} - - if err != nil { - return packageMetadata, err - } - - artifacts, err := c.artifactDao.GetByRegistryIDAndImage(ctx, registry.ID, packageName) - if err != nil { - return packageMetadata, err - } - - for _, artifact := range *artifacts { - metadata := &database.PyPiMetadata{} - err = json.Unmarshal(artifact.Metadata, metadata) - if err != nil { - return packageMetadata, err - } - - for _, file := range metadata.Files { - fileInfo := File{ - Name: file.Filename, - FileURL: c.urlProvider.RegistryURL(ctx) + fmt.Sprintf( - "/pkg/%s/%s/python/files/%s/%s/%s", - info.RootIdentifier, - info.RegIdentifier, - packageName, - artifact.Version, - file.Filename, - ), - RequiresPython: metadata.RequiresPython, - } - packageMetadata.Files = append(packageMetadata.Files, fileInfo) - } - } - - // Sort files by Name - sort.Slice(packageMetadata.Files, func(i, j int) bool { - return packageMetadata.Files[i].Name < packageMetadata.Files[j].Name - }) - - return packageMetadata, nil -} diff --git a/registry/app/pkg/pypi/upload_package.go b/registry/app/pkg/pypi/upload_package.go deleted file mode 100644 index 64031548e..000000000 --- a/registry/app/pkg/pypi/upload_package.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2023 Harness, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pypi - -import ( - "context" - "encoding/json" - "fmt" - "mime/multipart" - "net/http" - "strings" - "time" - - "github.com/harness/gitness/registry/app/dist_temp/errcode" - "github.com/harness/gitness/registry/app/pkg" - "github.com/harness/gitness/registry/app/pkg/commons" - "github.com/harness/gitness/registry/app/store/database" - "github.com/harness/gitness/registry/types" -) - -// UploadPackageFile FIXME: Extract this upload function for all types of packageTypes -// uploads the package file to the storage. -func (c *controller) UploadPackageFile( - ctx context.Context, - info ArtifactInfo, - file multipart.File, - fileHeader *multipart.FileHeader, -) (*commons.ResponseHeaders, string, errcode.Error) { - responseHeaders := &commons.ResponseHeaders{ - Headers: make(map[string]string), - Code: 0, - } - - // Fixme: Generalize this check for all package types - // err = c.CheckIfFileAlreadyExist(ctx, info) - // - // if err != nil { - // return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(err) - // } - - registry, err := c.registryDao.GetByRootParentIDAndName(ctx, info.RootParentID, info.RegIdentifier) - if err != nil { - return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err) - } - fileName := fileHeader.Filename - path := info.Image + "/" + info.Metadata.Version + "/" + fileName - fileInfo, err := c.fileManager.UploadFile(ctx, path, info.RegIdentifier, registry.ID, - info.RootParentID, info.RootIdentifier, file, nil, fileName) - if err != nil { - return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err) - } - err = c.tx.WithTx( - ctx, func(ctx context.Context) error { - image := &types.Image{ - Name: info.Image, - RegistryID: registry.ID, - Enabled: true, - } - err := c.imageDao.CreateOrUpdate(ctx, image) - if err != nil { - return fmt.Errorf("failed to create image for artifact: [%s], error: %w", info.Image, err) - } - - dbArtifact, err := c.artifactDao.GetByName(ctx, image.ID, info.Metadata.Version) - - if err != nil && !strings.Contains(err.Error(), "resource not found") { - return fmt.Errorf("failed to fetch artifact : [%s] with error: %w", info.Image, err) - } - - metadata := &database.PyPiMetadata{ - Metadata: info.Metadata, - } - err2 := c.updateMetadata(dbArtifact, metadata, info, fileInfo) - if err2 != nil { - return fmt.Errorf("failed to update metadata for artifact: [%s] with error: %w", info.Image, err2) - } - - metadataJSON, err := json.Marshal(metadata) - - if err != nil { - return fmt.Errorf("failed to parse metadata for artifact: [%s] with error: %w", info.Image, err) - } - - err = c.artifactDao.CreateOrUpdate(ctx, &types.Artifact{ - ImageID: image.ID, - Version: info.Metadata.Version, - Metadata: metadataJSON, - }) - if err != nil { - return fmt.Errorf("failed to create artifact : [%s] with error: %w", info.Image, err) - } - return nil - }) - - if err != nil { - return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err) - } - responseHeaders.Code = http.StatusCreated - return responseHeaders, fileInfo.Sha256, errcode.Error{} -} - -func (c *controller) updateMetadata( - dbArtifact *types.Artifact, metadata *database.PyPiMetadata, - info ArtifactInfo, fileInfo pkg.FileInfo, -) error { - var files []database.File - if dbArtifact != nil { - err := json.Unmarshal(dbArtifact.Metadata, metadata) - if err != nil { - return fmt.Errorf("failed to get metadata for artifact: [%s] with registry: [%s] and error: %w", info.Image, - info.RegIdentifier, err) - } - fileExist := false - files = metadata.Files - for _, file := range files { - if file.Filename == fileInfo.Filename { - fileExist = true - } - } - if !fileExist { - files = append(files, database.File{ - Size: fileInfo.Size, Filename: fileInfo.Filename, - CreatedAt: time.Now().UnixMilli(), - }) - metadata.Files = files - } - } else { - files = append(files, database.File{ - Size: fileInfo.Size, Filename: fileInfo.Filename, - CreatedAt: time.Now().UnixMilli(), - }) - metadata.Files = files - } - return nil -} diff --git a/registry/app/pkg/python/local.go b/registry/app/pkg/python/local.go new file mode 100644 index 000000000..75eb4ab20 --- /dev/null +++ b/registry/app/pkg/python/local.go @@ -0,0 +1,174 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "context" + "encoding/json" + "fmt" + "mime/multipart" + "net/http" + "sort" + + urlprovider "github.com/harness/gitness/app/url" + "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" + "github.com/harness/gitness/registry/app/dist_temp/errcode" + pythonmetadata "github.com/harness/gitness/registry/app/metadata/python" + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/base" + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/filemanager" + pythontype "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/app/storage" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/registry/types" + "github.com/harness/gitness/store/database/dbtx" +) + +var _ pkg.Artifact = (*localRegistry)(nil) +var _ Registry = (*localRegistry)(nil) + +type localRegistry struct { + localBase base.LocalBase + fileManager filemanager.FileManager + proxyStore store.UpstreamProxyConfigRepository + tx dbtx.Transactor + registryDao store.RegistryRepository + imageDao store.ImageRepository + artifactDao store.ArtifactRepository + urlProvider urlprovider.Provider +} + +type LocalRegistry interface { + Registry +} + +func NewLocalRegistry( + localBase base.LocalBase, + fileManager filemanager.FileManager, + proxyStore store.UpstreamProxyConfigRepository, + tx dbtx.Transactor, + registryDao store.RegistryRepository, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + urlProvider urlprovider.Provider, +) LocalRegistry { + return &localRegistry{ + localBase: localBase, + fileManager: fileManager, + proxyStore: proxyStore, + tx: tx, + registryDao: registryDao, + imageDao: imageDao, + artifactDao: artifactDao, + urlProvider: urlProvider, + } +} + +func (c *localRegistry) GetArtifactType() artifact.RegistryType { + return artifact.RegistryTypeVIRTUAL +} + +func (c *localRegistry) GetPackageTypes() []artifact.PackageType { + return []artifact.PackageType{artifact.PackageTypePYTHON} +} + +func (c *localRegistry) DownloadPackageFile(ctx context.Context, info pythontype.ArtifactInfo) ( + *commons.ResponseHeaders, + *storage.FileReader, + string, + []error, +) { + responseHeaders := &commons.ResponseHeaders{ + Headers: make(map[string]string), + Code: 0, + } + + path := "/" + info.Image + "/" + info.Version + "/" + info.Filename + + fileReader, _, redirectURL, err := c.fileManager.DownloadFile(ctx, path, types.Registry{ + ID: info.RegistryID, + Name: info.RegIdentifier, + }, info.RootIdentifier) + if err != nil { + return responseHeaders, nil, "", []error{err} + } + responseHeaders.Code = http.StatusOK + return responseHeaders, fileReader, redirectURL, nil +} + +// Metadata represents the metadata of a Python package. +func (c *localRegistry) GetPackageMetadata( + ctx context.Context, + info pythontype.ArtifactInfo, +) (pythontype.PackageMetadata, error) { + registry, err := c.registryDao.GetByRootParentIDAndName(ctx, info.RootParentID, info.RegIdentifier) + packageMetadata := pythontype.PackageMetadata{} + packageMetadata.Name = info.Image + packageMetadata.Files = []pythontype.File{} + + if err != nil { + return packageMetadata, err + } + + artifacts, err := c.artifactDao.GetByRegistryIDAndImage(ctx, registry.ID, info.Image) + if err != nil { + return packageMetadata, err + } + + for _, artifact := range *artifacts { + metadata := &pythonmetadata.PythonMetadata{} + err = json.Unmarshal(artifact.Metadata, metadata) + if err != nil { + return packageMetadata, err + } + + for _, file := range metadata.Files { + fileInfo := pythontype.File{ + Name: file.Filename, + FileURL: c.urlProvider.RegistryURL(ctx) + fmt.Sprintf( + "/pkg/%s/%s/python/files/%s/%s/%s", + info.RootIdentifier, + info.RegIdentifier, + info.Image, + artifact.Version, + file.Filename, + ), + RequiresPython: metadata.RequiresPython, + } + packageMetadata.Files = append(packageMetadata.Files, fileInfo) + } + } + + // Sort files by Name + sort.Slice(packageMetadata.Files, func(i, j int) bool { + return packageMetadata.Files[i].Name < packageMetadata.Files[j].Name + }) + + return packageMetadata, nil +} + +func (c *localRegistry) UploadPackageFile( + ctx context.Context, + info pythontype.ArtifactInfo, + file multipart.File, + fileHeader *multipart.FileHeader, +) (headers *commons.ResponseHeaders, sha256 string, err errcode.Error) { + path := info.Image + "/" + info.Metadata.Version + "/" + fileHeader.Filename + return c.localBase.Upload(ctx, info.ArtifactInfo, fileHeader.Filename, info.Metadata.Version, path, file, + &pythonmetadata.PythonMetadata{ + Metadata: info.Metadata, + }) +} diff --git a/registry/app/pkg/python/local_helper.go b/registry/app/pkg/python/local_helper.go new file mode 100644 index 000000000..1766d3da6 --- /dev/null +++ b/registry/app/pkg/python/local_helper.go @@ -0,0 +1,25 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +type LocalRegistryHelper interface { +} + +type localRegistryHelper struct { +} + +func NewLocalRegistryHelper() LocalRegistryHelper { + return &localRegistryHelper{} +} diff --git a/registry/app/pkg/python/proxy.go b/registry/app/pkg/python/proxy.go new file mode 100644 index 000000000..8c9cc3e79 --- /dev/null +++ b/registry/app/pkg/python/proxy.go @@ -0,0 +1,136 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "context" + "fmt" + "io" + "mime/multipart" + + urlprovider "github.com/harness/gitness/app/url" + "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" + "github.com/harness/gitness/registry/app/dist_temp/errcode" + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/filemanager" + pythontype "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/app/remote/controller/proxy/python" + "github.com/harness/gitness/registry/app/storage" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/store/database/dbtx" + + "github.com/rs/zerolog/log" +) + +var _ pkg.Artifact = (*proxy)(nil) +var _ Registry = (*proxy)(nil) + +type proxy struct { + fileManager filemanager.FileManager + proxyStore store.UpstreamProxyConfigRepository + tx dbtx.Transactor + registryDao store.RegistryRepository + imageDao store.ImageRepository + artifactDao store.ArtifactRepository + urlProvider urlprovider.Provider + proxyController python.Controller +} + +type Proxy interface { + Registry +} + +func NewProxy( + fileManager filemanager.FileManager, + proxyStore store.UpstreamProxyConfigRepository, + tx dbtx.Transactor, + registryDao store.RegistryRepository, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + urlProvider urlprovider.Provider, +) Proxy { + return &proxy{ + proxyStore: proxyStore, + registryDao: registryDao, + imageDao: imageDao, + artifactDao: artifactDao, + fileManager: fileManager, + tx: tx, + urlProvider: urlProvider, + } +} + +func (r *proxy) GetArtifactType() artifact.RegistryType { + return artifact.RegistryTypeUPSTREAM +} + +func (r *proxy) GetPackageTypes() []artifact.PackageType { + return []artifact.PackageType{artifact.PackageTypePYTHON} +} + +func (r *proxy) DownloadPackageFile(ctx context.Context, info pythontype.ArtifactInfo) ( + *commons.ResponseHeaders, + *storage.FileReader, + string, + []error, +) { + headers, body, _, url, errs := r.fetchFile(ctx, info, true) + return headers, body, url, errs +} + +// Metadata represents the metadata of a Python package. +func (r *proxy) GetPackageMetadata( + _ context.Context, + _ pythontype.ArtifactInfo, +) (pythontype.PackageMetadata, error) { + return pythontype.PackageMetadata{}, nil +} + +// UploadPackageFile FIXME: Extract this upload function for all types of packageTypes +// uploads the package file to the storage. +func (r *proxy) UploadPackageFile( + ctx context.Context, + _ pythontype.ArtifactInfo, + _ multipart.File, + _ *multipart.FileHeader, +) (*commons.ResponseHeaders, string, errcode.Error) { + log.Error().Ctx(ctx).Msg("Not implemented") + return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(fmt.Errorf("not implemented")) +} + +func (r *proxy) fetchFile(ctx context.Context, info pythontype.ArtifactInfo, serveFile bool) ( + responseHeaders *commons.ResponseHeaders, body *storage.FileReader, readCloser io.ReadCloser, + redirectURL string, errs []error, +) { + log.Ctx(ctx).Info().Msgf("Maven Proxy: %s", info.RegIdentifier) + + responseHeaders, body, redirectURL, useLocal := r.proxyController.UseLocalFile(ctx, info) + if useLocal { + return responseHeaders, body, readCloser, redirectURL, errs + } + + upstreamProxy, err := r.proxyStore.GetByRegistryIdentifier(ctx, info.ParentID, info.RegIdentifier) + if err != nil { + return responseHeaders, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)} + } + + // This is start of proxy Code. + responseHeaders, readCloser, err = r.proxyController.ProxyFile(ctx, info, *upstreamProxy, serveFile) + if err != nil { + return responseHeaders, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)} + } + return responseHeaders, nil, readCloser, "", errs +} diff --git a/registry/app/pkg/python/registry.go b/registry/app/pkg/python/registry.go new file mode 100644 index 000000000..66720d08f --- /dev/null +++ b/registry/app/pkg/python/registry.go @@ -0,0 +1,46 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "context" + "mime/multipart" + + "github.com/harness/gitness/registry/app/dist_temp/errcode" + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/app/storage" +) + +type Registry interface { + pkg.Artifact + + GetPackageMetadata(ctx context.Context, info python.ArtifactInfo) (python.PackageMetadata, error) + + UploadPackageFile( + ctx context.Context, + info python.ArtifactInfo, + file multipart.File, + fileHeader *multipart.FileHeader, + ) (*commons.ResponseHeaders, string, errcode.Error) + + DownloadPackageFile(ctx context.Context, info python.ArtifactInfo) ( + *commons.ResponseHeaders, + *storage.FileReader, + string, + []error, + ) +} diff --git a/registry/app/pkg/python/remote_helper.go b/registry/app/pkg/python/remote_helper.go new file mode 100644 index 000000000..55902ce5a --- /dev/null +++ b/registry/app/pkg/python/remote_helper.go @@ -0,0 +1,25 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +type RemoteRegistryHelper interface { +} + +type remoteRegistryHelper struct { +} + +func NewRemoteRegistryHelper() RemoteRegistryHelper { + return &remoteRegistryHelper{} +} diff --git a/registry/app/pkg/python/wire.go b/registry/app/pkg/python/wire.go new file mode 100644 index 000000000..576c5b7af --- /dev/null +++ b/registry/app/pkg/python/wire.go @@ -0,0 +1,57 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + urlprovider "github.com/harness/gitness/app/url" + "github.com/harness/gitness/registry/app/pkg/base" + "github.com/harness/gitness/registry/app/pkg/filemanager" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/store/database/dbtx" + + "github.com/google/wire" +) + +func LocalRegistryProvider( + localBase base.LocalBase, + fileManager filemanager.FileManager, + proxyStore store.UpstreamProxyConfigRepository, + tx dbtx.Transactor, + registryDao store.RegistryRepository, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + urlProvider urlprovider.Provider, +) LocalRegistry { + registry := NewLocalRegistry(localBase, fileManager, proxyStore, tx, registryDao, imageDao, artifactDao, + urlProvider) + base.Register(registry) + return registry +} + +func ProxyProvider( + proxyStore store.UpstreamProxyConfigRepository, + registryDao store.RegistryRepository, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + fileManager filemanager.FileManager, + tx dbtx.Transactor, + urlProvider urlprovider.Provider, +) Proxy { + proxy := NewProxy(fileManager, proxyStore, tx, registryDao, imageDao, artifactDao, urlProvider) + base.Register(proxy) + return proxy +} + +var WireSet = wire.NewSet(LocalRegistryProvider, ProxyProvider) diff --git a/registry/app/pkg/response/response.go b/registry/app/pkg/response/response.go new file mode 100644 index 000000000..a5ef6c083 --- /dev/null +++ b/registry/app/pkg/response/response.go @@ -0,0 +1,20 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package response + +type Response interface { + GetErrors() []error + SetError(error) +} diff --git a/registry/app/pkg/pypi/types.go b/registry/app/pkg/types/python/types.go similarity index 73% rename from registry/app/pkg/pypi/types.go rename to registry/app/pkg/types/python/types.go index 60f868af9..1a8546c1d 100644 --- a/registry/app/pkg/pypi/types.go +++ b/registry/app/pkg/types/python/types.go @@ -12,18 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pypi +package python import ( - "github.com/harness/gitness/registry/app/metadata/pypi" + "github.com/harness/gitness/registry/app/metadata/python" "github.com/harness/gitness/registry/app/pkg" ) -// Metadata represents the metadata for a PyPI package. - type ArtifactInfo struct { - *pkg.ArtifactInfo - Metadata pypi.Metadata + pkg.ArtifactInfo + Version string + Filename string + Metadata python.Metadata +} + +// BaseArtifactInfo implements pkg.PackageArtifactInfo interface. +func (a ArtifactInfo) BaseArtifactInfo() pkg.ArtifactInfo { + return a.ArtifactInfo } type File struct { diff --git a/registry/app/remote/clients/registry/client.go b/registry/app/remote/clients/registry/client.go index bbd512a3d..7b840be99 100644 --- a/registry/app/remote/clients/registry/client.go +++ b/registry/app/remote/clients/registry/client.go @@ -153,10 +153,10 @@ type Client interface { // Do send generic HTTP requests to the target registry service Do(req *http.Request) (*http.Response, error) - // Download the file + // GetFile Download the file GetFile(filePath string) (*commons.ResponseHeaders, io.ReadCloser, error) - // Check existence of file + // HeadFile Check existence of file HeadFile(filePath string) (*commons.ResponseHeaders, bool, error) } diff --git a/registry/app/remote/controller/proxy/python/controller.go b/registry/app/remote/controller/proxy/python/controller.go new file mode 100644 index 000000000..0c38eb0a3 --- /dev/null +++ b/registry/app/remote/controller/proxy/python/controller.go @@ -0,0 +1,143 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "context" + "io" + "strings" + + "github.com/harness/gitness/app/api/request" + "github.com/harness/gitness/app/services/refcache" + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/app/storage" + cfg "github.com/harness/gitness/registry/config" + "github.com/harness/gitness/registry/types" + "github.com/harness/gitness/secret" + + "github.com/rs/zerolog/log" +) + +type controller struct { + localRegistry registryInterface + secretService secret.Service + spaceFinder refcache.SpaceFinder +} + +type Controller interface { + UseLocalFile(ctx context.Context, info python.ArtifactInfo) ( + responseHeaders *commons.ResponseHeaders, fileReader *storage.FileReader, redirectURL string, useLocal bool, + ) + + ProxyFile( + ctx context.Context, info python.ArtifactInfo, proxy types.UpstreamProxy, serveFile bool, + ) (*commons.ResponseHeaders, io.ReadCloser, error) +} + +// NewProxyController -- get the proxy controller instance. +func NewProxyController( + l registryInterface, secretService secret.Service, + spaceFinder refcache.SpaceFinder, +) Controller { + return &controller{ + localRegistry: l, + secretService: secretService, + spaceFinder: spaceFinder, + } +} + +func (c *controller) UseLocalFile(ctx context.Context, info python.ArtifactInfo) ( + responseHeaders *commons.ResponseHeaders, fileReader *storage.FileReader, redirectURL string, useLocal bool, +) { + responseHeaders, body, _, redirectURL, e := c.localRegistry.GetArtifact(ctx, info) + return responseHeaders, body, redirectURL, len(e) == 0 +} + +func (c *controller) ProxyFile( + ctx context.Context, info python.ArtifactInfo, proxy types.UpstreamProxy, serveFile bool, +) (responseHeaders *commons.ResponseHeaders, body io.ReadCloser, errs error) { + responseHeaders = &commons.ResponseHeaders{ + Headers: make(map[string]string), + } + rHelper, err := NewRemoteHelper(ctx, c.spaceFinder, c.secretService, proxy) + if err != nil { + return responseHeaders, nil, err + } + + filePath := "" + // FIXME:URGENT: + //filePath := utils.GetFilePath(info) + + filePath = strings.Trim(filePath, "/") + + if serveFile { + responseHeaders, body, err = rHelper.GetFile(filePath) + } else { + responseHeaders, _, err = rHelper.HeadFile(filePath) + } + + if err != nil { + return responseHeaders, nil, err + } + + if !serveFile { + return responseHeaders, nil, nil + } + + go func(info python.ArtifactInfo) { + // Cloning Context. + session, ok := request.AuthSessionFrom(ctx) + if !ok { + log.Error().Stack().Err(err).Msg("failed to get auth session from context") + return + } + ctx2 := request.WithAuthSession(ctx, session) + ctx2 = context.WithoutCancel(ctx2) + ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "goRoutine") + err = c.putFileToLocal(ctx2, info, rHelper) + if err != nil { + log.Ctx(ctx2).Error().Stack().Err(err).Msgf("error while putting file to localRegistry, %v", err) + return + } + log.Ctx(ctx2).Info().Msgf("Successfully updated file "+ + "to registry: %s with file path: %s", + info.RegIdentifier, filePath) + }(info) + return responseHeaders, body, nil +} + +func (c *controller) putFileToLocal( + ctx context.Context, + info python.ArtifactInfo, + r RemoteInterface, +) error { + filePath := "" + // FIXME:URGENT: + //filePath := utils.GetFilePath(info) + + filePath = strings.Trim(filePath, "/") + + _, fileReader, err := r.GetFile(filePath) + if err != nil { + return err + } + defer fileReader.Close() + _, errs := c.localRegistry.PutArtifact(ctx, info, fileReader) + if len(errs) > 0 { + return errs[0] + } + return err +} diff --git a/registry/app/remote/controller/proxy/python/local.go b/registry/app/remote/controller/proxy/python/local.go new file mode 100644 index 000000000..c4f12df8f --- /dev/null +++ b/registry/app/remote/controller/proxy/python/local.go @@ -0,0 +1,39 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "context" + "io" + + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/app/storage" +) + +type registryInterface interface { + HeadArtifact(ctx context.Context, info python.ArtifactInfo) ( + responseHeaders *commons.ResponseHeaders, errs []error, + ) + + GetArtifact(ctx context.Context, info python.ArtifactInfo) ( + responseHeaders *commons.ResponseHeaders, body *storage.FileReader, fileReader io.ReadCloser, + redirectURL string, errs []error, + ) + + PutArtifact(ctx context.Context, info python.ArtifactInfo, fileReader io.Reader) ( + responseHeaders *commons.ResponseHeaders, errs []error, + ) +} diff --git a/registry/app/remote/controller/proxy/python/remote.go b/registry/app/remote/controller/proxy/python/remote.go new file mode 100644 index 000000000..129680702 --- /dev/null +++ b/registry/app/remote/controller/proxy/python/remote.go @@ -0,0 +1,96 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package python + +import ( + "context" + "io" + + "github.com/harness/gitness/app/services/refcache" + api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/remote/adapter" + "github.com/harness/gitness/registry/types" + "github.com/harness/gitness/secret" + + "github.com/rs/zerolog/log" + + _ "github.com/harness/gitness/registry/app/remote/adapter/maven" // This is required to init maven adapter +) + +const MavenCentralURL = "https://repo1.maven.org/maven2" + +// RemoteInterface defines operations related to remote repository under proxy. +type RemoteInterface interface { + // Download the file + GetFile(filePath string) (*commons.ResponseHeaders, io.ReadCloser, error) + + // Check existence of file + HeadFile(filePath string) (*commons.ResponseHeaders, bool, error) +} + +type remoteHelper struct { + registry adapter.ArtifactRegistry + upstreamProxy types.UpstreamProxy + URL string + secretService secret.Service +} + +// NewRemoteHelper create a remote interface. +func NewRemoteHelper( + ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, + proxy types.UpstreamProxy, +) (RemoteInterface, error) { + if proxy.Source == string(api.UpstreamConfigSourceMavenCentral) { + proxy.RepoURL = MavenCentralURL + } + r := &remoteHelper{ + upstreamProxy: proxy, + secretService: secretService, + } + if err := r.init(ctx, spaceFinder, string(api.UpstreamConfigSourceMavenCentral)); err != nil { + return nil, err + } + return r, nil +} + +func (r *remoteHelper) init(ctx context.Context, spaceFinder refcache.SpaceFinder, proxyType string) error { + if r.registry != nil { + return nil + } + + factory, err := adapter.GetFactory(proxyType) + if err != nil { + return err + } + adp, err := factory.Create(ctx, spaceFinder, r.upstreamProxy, r.secretService) + if err != nil { + return err + } + reg, ok := adp.(adapter.ArtifactRegistry) + if !ok { + log.Warn().Msgf("Error: adp is not of type adapter.ArtifactRegistry") + } + r.registry = reg + return nil +} + +func (r *remoteHelper) GetFile(filePath string) (*commons.ResponseHeaders, io.ReadCloser, error) { + return r.registry.GetFile(filePath) +} + +func (r *remoteHelper) HeadFile(filePath string) (*commons.ResponseHeaders, bool, error) { + return r.registry.HeadFile(filePath) +} diff --git a/registry/app/store/database/artifact.go b/registry/app/store/database/artifact.go index d5d9cb207..813fa481b 100644 --- a/registry/app/store/database/artifact.go +++ b/registry/app/store/database/artifact.go @@ -24,7 +24,6 @@ import ( "github.com/harness/gitness/app/api/request" "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" - "github.com/harness/gitness/registry/app/metadata/pypi" "github.com/harness/gitness/registry/app/store" "github.com/harness/gitness/registry/app/store/database/util" "github.com/harness/gitness/registry/types" @@ -754,25 +753,3 @@ type nonOCIArtifactMetadataDB struct { ModifiedAt int64 `db:"modified_at"` DownloadCount int64 `db:"download_count"` } - -type GenericMetadata struct { - Files []File `json:"files"` - Description string `json:"desc"` - FileCount int64 `json:"file_count"` -} - -type MavenMetadata struct { - Files []File `json:"files"` - FileCount int64 `json:"file_count"` -} - -type PyPiMetadata struct { - Files []File `json:"files"` - pypi.Metadata -} - -type File struct { - Size int64 `json:"size"` - Filename string `json:"file_name"` - CreatedAt int64 `json:"created_at"` -} diff --git a/registry/request/context.go b/registry/request/context.go index ba3f8b8b6..9f96546fc 100644 --- a/registry/request/context.go +++ b/registry/request/context.go @@ -17,12 +17,15 @@ package request import ( "context" + "github.com/harness/gitness/registry/app/pkg" + "github.com/rs/zerolog/log" ) type contextKey string const OriginalURLKey contextKey = "originalURL" +const ArtifactInfoKey contextKey = "artifactInfo" func OriginalURLFrom(ctx context.Context) string { originalURL, ok := ctx.Value(OriginalURLKey).(string) @@ -35,3 +38,15 @@ func OriginalURLFrom(ctx context.Context) string { func WithOriginalURL(parent context.Context, originalURL string) context.Context { return context.WithValue(parent, OriginalURLKey, originalURL) } + +func ArtifactInfoFrom(ctx context.Context) pkg.PackageArtifactInfo { + baseInfo, ok := ctx.Value(ArtifactInfoKey).(pkg.PackageArtifactInfo) + if !ok { + log.Ctx(ctx).Warn().Msg("Failed to create artifact info") + } + return baseInfo +} + +func WithArtifactInfo(parent context.Context, artifactInfo pkg.PackageArtifactInfo) context.Context { + return context.WithValue(parent, ArtifactInfoKey, artifactInfo) +}