USEFULL
/
drone
mirror of https://github.com/harness/drone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: [AH-1137]: Added cleanup policy; cleaned up error framework; updated upload flows in case of conflicts; misc (#3592) 

* [AH-1137]: PR Review comments
* [AH-1137]: PR Review comments
* [AH-1137]: Fixed auth issue
* [AH-1137]: Added default route:
* [AH-1137]: Added cleanup policy; cleaned up error framework; updated upload flows in case of conflicts; misc
* [AH-1137]: Merge commit
* [AH-993]: Review comments fixed
* [AH-993]: Merge commit
* [AH-993]: Updated upstream creation
* [AH-993]: Cleanup
* [AH-993]: Updated messages
* [AH-993]: Merge commit
* [AH-993]: Upstream flows support for Python Packages
* [AH-993]: Updated local file
* [AH-993]: Added support for local and created arch to support different package types
* Merge branch 'main' of https://git0.harness.io/l7B_kbSEQD2wjrM7PShm5w/PROD/Harness_Commons/gitness into AH-993-upstream-implementation
* [AH-993]: temp commit
* [AH-993]: Merge commit:
* [AH-993]: temp update
main
Arvind Choudhary 2025-03-28 04:57:32 +00:00 committed by Harness
parent 56805e3818
commit 93c3f324ce
32 changed files with 610 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
registry/app/api/controller/pkg/python/controller.go
View File

@ -27,7 +27,7 @@ import (
)
type Controller interface {
GetPackageMetadata(ctx context.Context, info pythontype.ArtifactInfo) (pythontype.PackageMetadata, error)
GetPackageMetadata(ctx context.Context, info pythontype.ArtifactInfo) *GetMetadataResponse
UploadPackageFile(
ctx context.Context,

35
registry/app/api/controller/pkg/python/download.go
View File

@ -33,26 +33,45 @@ func (c *controller) DownloadPackageFile(
f := func(registry registrytypes.Registry, a pkg.Artifact) response.Response {
info.RegIdentifier = registry.Name
info.RegistryID = registry.ID
info.Registry = registry
pythonRegistry, ok := a.(python.Registry)
if !ok {
return &GetArtifactResponse{
[]error{fmt.Errorf("invalid registry type: expected python.Registry")},
nil, "", nil, nil,
BaseResponse{
fmt.Errorf("invalid registry type: expected python.Registry"),
nil,
},
"", nil, nil,
}
}
headers, fileReader, readCloser, redirectURL, errs := pythonRegistry.DownloadPackageFile(ctx, info)
headers, fileReader, readCloser, redirectURL, err := pythonRegistry.DownloadPackageFile(ctx, info)
return &GetArtifactResponse{
errs, headers, redirectURL,
fileReader, readCloser,
BaseResponse{
err,
headers,
},
redirectURL, fileReader, readCloser,
}
}
result := base.ProxyWrapper(ctx, c.registryDao, f, info.BaseArtifactInfo())
result, err := base.ProxyWrapper(ctx, c.registryDao, f, info)
if err != nil {
return &GetArtifactResponse{
BaseResponse{
err,
nil,
},
"", nil, nil,
}
}
getResponse, ok := result.(*GetArtifactResponse)
if !ok {
return &GetArtifactResponse{
[]error{fmt.Errorf("invalid response type: expected GetArtifactResponse")},
nil, "", nil, nil,
BaseResponse{
fmt.Errorf("invalid response type: expected GetArtifactResponse"),
nil,
},
"", nil, nil,
}
}
return getResponse

33
registry/app/api/controller/pkg/python/metadata.go
View File

@ -27,33 +27,42 @@ import (
)
// Metadata represents the metadata of a Python package.
func (c *controller) GetPackageMetadata(ctx context.Context, info pythontype.ArtifactInfo) (
pythontype.PackageMetadata,
error,
) {
func (c *controller) GetPackageMetadata(ctx context.Context, info pythontype.ArtifactInfo) *GetMetadataResponse {
f := func(registry registrytypes.Registry, a pkg.Artifact) response.Response {
info.RegIdentifier = registry.Name
info.RegistryID = registry.ID
info.Registry = registry
pythonRegistry, ok := a.(python.Registry)
if !ok {
return &GetMetadataResponse{
[]error{fmt.Errorf("invalid registry type: expected python.Registry")},
nil, pythontype.PackageMetadata{},
BaseResponse{
fmt.Errorf("invalid registry type: expected python.Registry"),
nil,
},
pythontype.PackageMetadata{},
}
}
metadata, err := pythonRegistry.GetPackageMetadata(ctx, info)
return &GetMetadataResponse{
[]error{err}, nil, metadata,
BaseResponse{
err,
nil,
},
metadata,
}
}
result := base.ProxyWrapper(ctx, c.registryDao, f, info.BaseArtifactInfo())
result, err := base.ProxyWrapper(ctx, c.registryDao, f, info)
metadataResponse, ok := result.(*GetMetadataResponse)
if !ok {
return pythontype.PackageMetadata{},
fmt.Errorf("invalid response type: expected GetMetadataResponse, got %T", result)
return &GetMetadataResponse{
BaseResponse{
err,
nil,
},
pythontype.PackageMetadata{},
}
}
return metadataResponse.PackageMetadata, nil
return metadataResponse
}

51
registry/app/api/controller/pkg/python/response.go
View File

@ -27,46 +27,27 @@ var _ response.Response = (*GetMetadataResponse)(nil)
var _ response.Response = (*GetArtifactResponse)(nil)
var _ response.Response = (*PutArtifactResponse)(nil)
type GetMetadataResponse struct {
Errors []error
type BaseResponse struct {
Error error
ResponseHeaders *commons.ResponseHeaders
}
func (r BaseResponse) GetError() error {
return r.Error
}
type GetMetadataResponse struct {
BaseResponse
PackageMetadata pythontype.PackageMetadata
}
func (r *GetMetadataResponse) GetErrors() []error {
return r.Errors
}
func (r *GetMetadataResponse) SetError(err error) {
r.Errors = make([]error, 1)
r.Errors[0] = err
}
type GetArtifactResponse struct {
Errors []error
ResponseHeaders *commons.ResponseHeaders
RedirectURL string
Body *storage.FileReader
ReadCloser io.ReadCloser
}
func (r *GetArtifactResponse) GetErrors() []error {
return r.Errors
}
func (r *GetArtifactResponse) SetError(err error) {
r.Errors = make([]error, 1)
r.Errors[0] = err
BaseResponse
RedirectURL string
Body *storage.FileReader
ReadCloser io.ReadCloser
}
type PutArtifactResponse struct {
Sha256 string
Errors []error
ResponseHeaders *commons.ResponseHeaders
}
func (r *PutArtifactResponse) GetErrors() []error {
return r.Errors
}
func (r *PutArtifactResponse) SetError(err error) {
r.Errors = make([]error, 1)
r.Errors[0] = err
BaseResponse
Sha256 string
}

31
registry/app/api/controller/pkg/python/upload_package.go
View File

@ -21,7 +21,6 @@ import (
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/base"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/python"
"github.com/harness/gitness/registry/app/pkg/response"
pythontype "github.com/harness/gitness/registry/app/pkg/types/python"
@ -39,33 +38,37 @@ func (c *controller) UploadPackageFile(
f := func(registry registrytypes.Registry, a pkg.Artifact) response.Response {
info.RegIdentifier = registry.Name
info.RegistryID = registry.ID
info.Registry = registry
pythonRegistry, ok := a.(python.Registry)
if !ok {
return &PutArtifactResponse{
BaseResponse{
Error: fmt.Errorf("invalid registry type: expected python.Registry"),
ResponseHeaders: nil,
},
"",
[]error{fmt.Errorf("invalid registry type: expected python.Registry")},
nil,
}
}
headers, sha256, err := pythonRegistry.UploadPackageFile(ctx, info, file, fileHeader.Filename)
if commons.IsEmptyError(err) {
return &PutArtifactResponse{
sha256, []error{}, headers,
}
}
return &PutArtifactResponse{
sha256, []error{err}, headers,
BaseResponse{
Error: err,
ResponseHeaders: headers,
},
sha256,
}
}
result := base.NoProxyWrapper(ctx, c.registryDao, f, info.BaseArtifactInfo())
response, ok := result.(*PutArtifactResponse)
result, err := base.NoProxyWrapper(ctx, c.registryDao, f, info)
putResponse, ok := result.(*PutArtifactResponse)
if !ok {
return &PutArtifactResponse{
BaseResponse{
Error: err,
ResponseHeaders: nil,
},
"",
[]error{fmt.Errorf("invalid response type: expected PutArtifactResponse")},
nil,
}
}
return response
return putResponse
}

8
registry/app/api/handler/generic/base.go
View File

@ -143,7 +143,7 @@ func (h *Handler) GetArtifactInfo(r *http.Request) (pkg.GenericArtifactInfo, err
}
if !commons.IsEmpty(info.Image) && !commons.IsEmpty(info.Version) && !commons.IsEmpty(info.FileName) {
flag, err2 := utils.MatchArtifactFilter(registry.AllowedPattern, registry.BlockedPattern,
flag, err2 := utils.IsPatternAllowed(registry.AllowedPattern, registry.BlockedPattern,
info.Image+":"+info.Version+":"+info.FileName)
if !flag || err2 != nil {
return pkg.GenericArtifactInfo{}, errcode.ErrCodeInvalidRequest.WithDetail(err2)
@ -156,8 +156,10 @@ func (h *Handler) GetArtifactInfo(r *http.Request) (pkg.GenericArtifactInfo, err
// ExtractPathVars extracts registry,image, reference, digest and tag from the path
// Path format: /generic/:rootSpace/:registry/:image/:tag (for ex:
// /generic/myRootSpace/reg1/alpine/v1).
func ExtractPathVars(r *http.Request) (rootIdentifier, registry, artifact,
tag, fileName string, description string, err error) {
func ExtractPathVars(r *http.Request) (
rootIdentifier, registry, artifact,
tag, fileName string, description string, err error,
) {
path := r.URL.Path
// Ensure the path starts with "/generic/"

2
registry/app/api/handler/maven/base.go
View File

@ -139,7 +139,7 @@ func (h *Handler) GetArtifactInfo(r *http.Request, remoteSupport bool) (pkg.Mave
}
if !commons.IsEmpty(info.GroupID) && !commons.IsEmpty(info.ArtifactID) && !commons.IsEmpty(info.Version) {
flag, err2 := utils.MatchArtifactFilter(registry.AllowedPattern, registry.BlockedPattern,
flag, err2 := utils.IsPatternAllowed(registry.AllowedPattern, registry.BlockedPattern,
info.GroupID+":"+info.ArtifactID+":"+info.Version)
if !flag || err2 != nil {
return pkg.MavenArtifactInfo{}, errcode.ErrCodeDenied

2
registry/app/api/handler/oci/base.go
View File

@ -249,7 +249,7 @@ func (h *Handler) GetRegistryInfo(r *http.Request, remoteSupport bool) (pkg.Regi
}
if !commons.IsEmpty(info.Image) && !commons.IsEmpty(info.Tag) {
flag, err2 := utils.MatchArtifactFilter(registry.AllowedPattern, registry.BlockedPattern, info.Image+":"+info.Tag)
flag, err2 := utils.IsPatternAllowed(registry.AllowedPattern, registry.BlockedPattern, info.Image+":"+info.Tag)
if !flag || err2 != nil {
return pkg.RegistryInfo{}, errcode.ErrCodeDenied
}

29
registry/app/api/handler/packages/handler.go
View File

@ -24,6 +24,8 @@ import (
"time"
usercontroller "github.com/harness/gitness/app/api/controller/user"
"github.com/harness/gitness/app/api/render"
"github.com/harness/gitness/app/api/usererror"
"github.com/harness/gitness/app/auth/authn"
"github.com/harness/gitness/app/auth/authz"
corestore "github.com/harness/gitness/app/store"
@ -72,10 +74,11 @@ type Handler interface {
r *http.Request,
reqPermissions ...enum.Permission,
) error
GetArtifactInfo(r *http.Request) (pkg.ArtifactInfo, errcode.Error)
GetArtifactInfo(r *http.Request) (pkg.ArtifactInfo, error)
GetAuthenticator() authn.Authenticator
HandleErrors2(ctx context.Context, errors errcode.Error, w http.ResponseWriter)
HandleErrors(ctx context.Context, errors errcode.Errors, w http.ResponseWriter)
HandleError(ctx context.Context, w http.ResponseWriter, err error)
ServeContent(
w http.ResponseWriter, r *http.Request, fileReader *storage.FileReader, filename string,
)
@ -104,13 +107,16 @@ func (h *handler) GetRegistryCheckAccess(
r *http.Request,
reqPermissions ...enum.Permission,
) error {
info, _ := h.GetArtifactInfo(r)
info, err := h.GetArtifactInfo(r)
if err != nil {
return err
}
return pkg.GetRegistryCheckAccess(ctx, h.RegistryDao, h.Authorizer,
h.SpaceStore,
info.RegIdentifier, info.ParentID, reqPermissions...)
}
func (h *handler) GetArtifactInfo(r *http.Request) (pkg.ArtifactInfo, errcode.Error) {
func (h *handler) GetArtifactInfo(r *http.Request) (pkg.ArtifactInfo, error) {
ctx := r.Context()
rootIdentifier, registryIdentifier, pathPackageType, err := extractPathVars(r)
@ -121,7 +127,7 @@ func (h *handler) GetArtifactInfo(r *http.Request) (pkg.ArtifactInfo, errcode.Er
rootSpace, err := h.SpaceStore.FindByRefCaseInsensitive(ctx, rootIdentifier)
if err != nil {
log.Ctx(ctx).Error().Msgf("Root space not found: %s", rootIdentifier)
return pkg.ArtifactInfo{}, errcode.ErrCodeRootNotFound.WithDetail(err)
return pkg.ArtifactInfo{}, usererror.NotFoundf("Root not found: %s", rootIdentifier)
}
registry, err := h.RegistryDao.GetByRootParentIDAndName(ctx, rootSpace.ID, registryIdentifier)
@ -130,13 +136,13 @@ func (h *handler) GetArtifactInfo(r *http.Request) (pkg.ArtifactInfo, errcode.Er
log.Ctx(ctx).Error().Msgf(
"registry %s not found for root: %s. Reason: %s", registryIdentifier, rootSpace.Identifier, err,
)
return pkg.ArtifactInfo{}, errcode.ErrCodeRegNotFound.WithDetail(err)
return pkg.ArtifactInfo{}, usererror.NotFoundf("Registry not found: %s", registryIdentifier)
}
_, err = h.SpaceStore.Find(r.Context(), registry.ParentID)
if err != nil {
log.Ctx(ctx).Error().Msgf("Parent space not found: %d", registry.ParentID)
return pkg.ArtifactInfo{}, errcode.ErrCodeParentNotFound.WithDetail(err)
return pkg.ArtifactInfo{}, usererror.NotFoundf("Parent not found for registry: %s", registryIdentifier)
}
return pkg.ArtifactInfo{
@ -148,8 +154,9 @@ func (h *handler) GetArtifactInfo(r *http.Request) (pkg.ArtifactInfo, errcode.Er
},
RegIdentifier: registryIdentifier,
RegistryID: registry.ID,
Registry: *registry,
Image: "",
}, errcode.Error{}
}, nil
}
func (h *handler) HandleErrors2(ctx context.Context, err errcode.Error, w http.ResponseWriter) {
@ -182,6 +189,14 @@ func (h *handler) HandleErrors(ctx context.Context, errs errcode.Errors, w http.
}
}
func (h *handler) HandleError(ctx context.Context, w http.ResponseWriter, err error) {
if nil != err {
log.Error().Err(err).Ctx(ctx).Msgf("error: %v", err)
render.TranslatedUserError(ctx, w, err)
return
}
}
func LogError(errList errcode.Errors) {
for _, e1 := range errList {
log.Error().Err(e1).Msgf("error: %v", e1)

6
registry/app/api/handler/python/download.go
View File

@ -55,8 +55,8 @@ func (h *handler) DownloadPackageFile(w http.ResponseWriter, r *http.Request) {
}
}()
if !commons.IsEmpty(response.GetErrors()) {
h.HandleErrors(ctx, response.GetErrors(), w)
if response.GetError() != nil {
h.HandleError(ctx, w, response.GetError())
return
}
@ -69,7 +69,7 @@ func (h *handler) DownloadPackageFile(w http.ResponseWriter, r *http.Request) {
err := commons.ServeContent(w, r, response.Body, info.Filename, response.ReadCloser)
if err != nil {
log.Ctx(ctx).Error().Msgf("Failed to serve content: %v", err)
h.HandleErrors(ctx, []error{err}, w)
h.HandleError(ctx, w, err)
return
}
response.ResponseHeaders.WriteToResponse(w)

3
registry/app/api/handler/python/handler.go
View File

@ -26,7 +26,6 @@ import (
"github.com/harness/gitness/registry/app/api/handler/utils"
python2 "github.com/harness/gitness/registry/app/metadata/python"
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/commons"
pythontype "github.com/harness/gitness/registry/app/pkg/types/python"
"github.com/harness/gitness/registry/validation"
@ -76,7 +75,7 @@ var _ Handler = (*handler)(nil)
func (h *handler) GetPackageArtifactInfo(r *http.Request) (pkg.PackageArtifactInfo, error) {
info, err := h.Handler.GetArtifactInfo(r)
if !commons.IsEmptyError(err) {
if err != nil {
return nil, err
}

26
registry/app/api/handler/python/list.go
View File

@ -15,9 +15,12 @@
package python
import (
"fmt"
"html/template"
"net/http"
"github.com/harness/gitness/app/api/render"
"github.com/harness/gitness/registry/app/common/lib/errors"
pythontype "github.com/harness/gitness/registry/app/pkg/types/python"
"github.com/harness/gitness/registry/request"
)
@ -43,29 +46,30 @@ func (h *handler) PackageMetadata(w http.ResponseWriter, r *http.Request) {
contextInfo := request.ArtifactInfoFrom(r.Context())
info, ok := contextInfo.(*pythontype.ArtifactInfo)
if !ok {
http.Error(w, "Invalid request context", http.StatusBadRequest)
return
}
if info.Image == "" {
http.Error(w, "Package name required", http.StatusBadRequest)
render.TranslatedUserError(r.Context(), w, fmt.Errorf("invalid request context"))
return
}
packageData, err := h.controller.GetPackageMetadata(r.Context(), *info)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
packageData := h.controller.GetPackageMetadata(r.Context(), *info)
if packageData.GetError() != nil {
notFound := errors.IsErr(packageData.GetError(), errors.NotFoundCode)
if notFound {
render.NotFound(r.Context(), w)
return
}
render.TranslatedUserError(r.Context(), w, packageData.GetError())
return
}
// Parse and execute the template
tmpl, err := template.New("simple").Parse(HTMLTemplate)
if err != nil {
http.Error(w, "Template error", http.StatusInternalServerError)
render.TranslatedUserError(r.Context(), w, fmt.Errorf("template error: %w", err))
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
if err := tmpl.Execute(w, packageData); err != nil {
http.Error(w, "Rendering error", http.StatusInternalServerError)
if err := tmpl.Execute(w, packageData.PackageMetadata); err != nil {
render.TranslatedUserError(r.Context(), w, fmt.Errorf("template rendering error: %w", err))
}
}

25
registry/app/api/handler/python/upload.go
View File

@ -18,7 +18,6 @@ import (
"fmt"
"net/http"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
pythontype "github.com/harness/gitness/registry/app/pkg/types/python"
"github.com/harness/gitness/registry/request"
)
@ -27,8 +26,7 @@ func (h *handler) UploadPackageFile(w http.ResponseWriter, r *http.Request) {
file, fileHeader, err := r.FormFile("content")
if err != nil {
h.HandleErrors2(r.Context(), errcode.ErrCodeInvalidRequest.WithMessage(fmt.Sprintf("failed to parse file: %s, "+
"please provide correct file path ", err.Error())), w)
h.HandleError(r.Context(), w, err)
return
}
@ -37,9 +35,10 @@ func (h *handler) UploadPackageFile(w http.ResponseWriter, r *http.Request) {
contextInfo := request.ArtifactInfoFrom(r.Context())
info, ok := contextInfo.(*pythontype.ArtifactInfo)
if !ok {
h.HandleErrors2(r.Context(), errcode.ErrCodeInvalidRequest.WithMessage("failed to fetch info from context"), w)
h.HandleError(r.Context(), w, fmt.Errorf("failed to fetch info from context"))
return
}
// TODO: Can we extract this out to ArtifactInfoProvider
if info.Filename == "" {
info.Filename = fileHeader.Filename
@ -48,13 +47,15 @@ func (h *handler) UploadPackageFile(w http.ResponseWriter, r *http.Request) {
response := h.controller.UploadPackageFile(r.Context(), *info, file, fileHeader)
if len(response.Errors) == 0 {
response.ResponseHeaders.WriteToResponse(w)
_, err := w.Write([]byte(fmt.Sprintf("Pushed.\nSha256: %s", response.Sha256)))
if err != nil {
h.HandleErrors2(r.Context(), errcode.ErrCodeUnknown.WithDetail(err), w)
return
}
if response.GetError() != nil {
h.HandleError(r.Context(), w, response.GetError())
return
}
response.ResponseHeaders.WriteToResponse(w)
_, err = w.Write([]byte(fmt.Sprintf("Pushed.\nSha256: %s", response.Sha256)))
if err != nil {
h.HandleError(r.Context(), w, err)
return
}
h.HandleErrors(r.Context(), response.GetErrors(), w)
}

6
registry/app/api/handler/utils/artifactfilter.go
View File

@ -23,7 +23,7 @@ import (
"github.com/rs/zerolog/log"
)
func MatchArtifactFilter(
func IsPatternAllowed(
allowedPattern pq.StringArray,
blockedPattern pq.StringArray, artifact string,
) (bool, error) {
@ -33,7 +33,7 @@ func MatchArtifactFilter(
if len(blockedPatterns) > 0 {
flag, err := matchPatterns(blockedPatterns, artifact)
if err != nil {
return flag, fmt.Errorf(
return false, fmt.Errorf(
"failed to match blocked patterns for artifact %s: %w",
artifact, err,
)
@ -48,7 +48,7 @@ func MatchArtifactFilter(
if len(allowedPatterns) > 0 {
flag, err := matchPatterns(allowedPatterns, artifact)
if err != nil {
return flag, fmt.Errorf(
return false, fmt.Errorf(
"failed to match allowed patterns for artifact %s: %w",
artifact, err,
)

2
registry/app/api/middleware/artifact_info.go
View File

@ -30,7 +30,7 @@ func StoreArtifactInfo(
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
packageInfo, err := provider.GetPackageArtifactInfo(r)
if err != nil {
render.BadRequestf(r.Context(), w, err.Error())
render.TranslatedUserError(r.Context(), w, err)
return
}
// Store the base artifact info in the context

8
registry/app/api/middleware/request_package_access.go
View File

@ -20,8 +20,6 @@ import (
"github.com/harness/gitness/app/api/render"
"github.com/harness/gitness/registry/app/api/handler/packages"
"github.com/harness/gitness/types/enum"
"github.com/rs/zerolog/log"
)
// StoreOriginalURL stores the original URL in the context.
@ -33,11 +31,7 @@ func RequestPackageAccess(
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
err := packageHandler.GetRegistryCheckAccess(r.Context(), r, reqPermissions...)
if err != nil {
log.Info().Err(err).Msgf("Access denied for path: %s, method: %s, permission: %s", r.URL.Path, r.Method,
reqPermissions)
render.Forbiddenf(r.Context(), w,
"Access denied as permission: %s is required for path: %s, method: %s", reqPermissions, r.URL.Path,
r.Method)
render.TranslatedUserError(r.Context(), w, err)
}
next.ServeHTTP(w, r.WithContext(r.Context()))

18
registry/app/api/router/packages/route.go
View File

@ -15,6 +15,7 @@
package packages
import (
"fmt"
"net/http"
middlewareauthn "github.com/harness/gitness/app/api/middleware/authn"
@ -62,6 +63,7 @@ func NewRouter(
r.Route("/generic", func(r chi.Router) {
r.Use(middlewareauthn.Attempt(packageHandler.GetAuthenticator()))
r.Use(middleware.CheckAuth())
r.Use(middleware.TrackDownloadStatForGenericArtifact(genericHandler))
r.Use(middleware.TrackBandwidthStatForGenericArtifacts(genericHandler))
@ -71,6 +73,7 @@ func NewRouter(
r.Route("/python", func(r chi.Router) {
r.Use(middlewareauthn.Attempt(packageHandler.GetAuthenticator()))
r.Use(middleware.CheckAuth())
// TODO (Arvind): Move this to top layer with total abstraction
r.With(middleware.StoreArtifactInfo(pythonHandler)).
@ -79,12 +82,21 @@ func NewRouter(
r.With(middleware.StoreArtifactInfo(pythonHandler)).
With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)).
Get("/files/{image}/{version}/{filename}", pythonHandler.DownloadPackageFile)
r.With(middleware.StoreArtifactInfo(pythonHandler)).
With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)).
Get("/simple/{image}", pythonHandler.PackageMetadata)
r.With(middleware.StoreArtifactInfo(pythonHandler)).
With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)).
Get("/simple/{image}/", pythonHandler.PackageMetadata)
r.Get("/simple/{image}", func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
})
})
r.Route("/{packageType}", func(r chi.Router) {
r.Use(middlewareauthn.Attempt(packageHandler.GetAuthenticator()))
r.Use(middleware.CheckAuth())
r.HandleFunc("/*", func(w http.ResponseWriter, r *http.Request) {
packageType := chi.URLParam(r, "packageType")
http.Error(w, fmt.Sprintf("Package type '%s' is not supported", packageType), http.StatusNotFound)
})
})
})

1
registry/app/pkg/artifact_info.go
View File

@ -23,6 +23,7 @@ import (
// the base ArtifactInfo type.
type PackageArtifactInfo interface {
BaseArtifactInfo() ArtifactInfo
GetImageVersion() (bool, string)
}
// ArtifactInfoProvider is an interface that must be implemented by package handlers

75
registry/app/pkg/base/base.go
View File

@ -24,6 +24,7 @@ import (
"strings"
"time"
"github.com/harness/gitness/errors"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
"github.com/harness/gitness/registry/app/metadata"
"github.com/harness/gitness/registry/app/pkg"
@ -49,25 +50,19 @@ type LocalBase interface {
path string,
file multipart.File,
metadata metadata.Metadata,
) (
// TODO: Check the scope if we should remove the response message / headers setup here or
// each package implementation should have their own.
headers *commons.ResponseHeaders, sha256 string, err errcode.Error,
)
) (headers *commons.ResponseHeaders, sha256 string, err error)
Upload(
ctx context.Context,
info pkg.ArtifactInfo,
fileName string,
version string,
path string,
fileName, version, path string,
file io.ReadCloser,
metadata metadata.Metadata,
) (*commons.ResponseHeaders, string, errcode.Error)
) (*commons.ResponseHeaders, string, error)
Download(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) (
*commons.ResponseHeaders,
*storage.FileReader,
string,
[]error,
error,
)
Exists(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) bool
@ -104,21 +99,18 @@ func (l *localBase) UploadFile(
version string,
path string,
file multipart.File,
// TODO: Metadata shouldn't be provided as a parameter, it should be fetched or created.
metadata metadata.Metadata,
) (*commons.ResponseHeaders, string, errcode.Error) {
) (*commons.ResponseHeaders, string, error) {
return l.uploadInternal(ctx, info, fileName, version, path, file, nil, metadata)
}
func (l *localBase) Upload(
ctx context.Context,
info pkg.ArtifactInfo,
fileName string,
version string,
path string,
fileName, version, path string,
file io.ReadCloser,
metadata metadata.Metadata,
) (*commons.ResponseHeaders, string, errcode.Error) {
) (*commons.ResponseHeaders, string, error) {
return l.uploadInternal(ctx, info, fileName, version, path, nil, file, metadata)
}
@ -131,7 +123,7 @@ func (l *localBase) uploadInternal(
file multipart.File,
fileReadCloser io.ReadCloser,
metadata metadata.Metadata,
) (*commons.ResponseHeaders, string, errcode.Error) {
) (*commons.ResponseHeaders, string, error) {
responseHeaders := &commons.ResponseHeaders{
Headers: make(map[string]string),
Code: 0,
@ -140,7 +132,16 @@ func (l *localBase) uploadInternal(
err := l.CheckIfFileAlreadyExist(ctx, info, version, metadata, fileName)
if err != nil {
return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(err)
if !errors.IsConflict(err) {
return nil, "", err
}
_, sha256, err2 := l.GetSHA256(ctx, info, version, fileName)
if err2 != nil {
return responseHeaders, "", err2
}
responseHeaders.Code = http.StatusCreated
return responseHeaders, sha256, nil
}
registry, err := l.registryDao.GetByRootParentIDAndName(ctx, info.RootParentID, info.RegIdentifier)
@ -193,18 +194,18 @@ func (l *localBase) uploadInternal(
})
if err != nil {
return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err)
return responseHeaders, "", err
}
responseHeaders.Code = http.StatusCreated
return responseHeaders, fileInfo.Sha256, errcode.Error{}
return responseHeaders, fileInfo.Sha256, nil
}
func (l *localBase) Download(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) (
*commons.ResponseHeaders,
*storage.FileReader,
string,
[]error,
) {
func (l *localBase) Download(
ctx context.Context,
info pkg.ArtifactInfo,
version string,
fileName string,
) (*commons.ResponseHeaders, *storage.FileReader, string, error) {
responseHeaders := &commons.ResponseHeaders{
Headers: make(map[string]string),
Code: 0,
@ -218,17 +219,30 @@ func (l *localBase) Download(ctx context.Context, info pkg.ArtifactInfo, version
Name: info.RegIdentifier,
}, info.RootIdentifier)
if err != nil {
return responseHeaders, nil, "", []error{err}
return responseHeaders, nil, "", err
}
responseHeaders.Code = http.StatusOK
return responseHeaders, fileReader, redirectURL, nil
}
func (l *localBase) Exists(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) bool {
exists, _, _ := l.GetSHA256(ctx, info, version, fileName)
return exists
}
func (l *localBase) GetSHA256(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) (
exists bool,
sha256 string,
err error,
) {
filePath := "/" + info.Image + "/" + version + "/" + fileName
sha256, _ := l.fileManager.HeadFile(ctx, filePath, info.RegistryID)
sha256, err = l.fileManager.HeadFile(ctx, filePath, info.RegistryID)
if err != nil {
return false, "", err
}
//FIXME: err should be checked on if the record doesn't exist or there was DB call issue
return sha256 != ""
return true, sha256, err
}
func (l *localBase) updateMetadata(
@ -302,7 +316,8 @@ func (l *localBase) CheckIfFileAlreadyExist(
for _, file := range metadata.GetFiles() {
if file.Filename == fileName {
return fmt.Errorf("file: [%s] with Artifact: [%s], Version: [%s] and registry: [%s] already exist",
l.Exists(ctx, info, version, fileName)
return errors.Conflict("file: [%s] with Artifact: [%s], Version: [%s] and registry: [%s] already exist",
fileName, info.Image, version, info.RegIdentifier)
}
}

176
registry/app/pkg/base/wrapper.go
View File

@ -1,22 +1,24 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package base
import (
"context"
"github.com/harness/gitness/errors"
"github.com/harness/gitness/registry/app/api/handler/utils"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/response"
@ -42,59 +44,58 @@ func NoProxyWrapper(
ctx context.Context,
registryDao store.RegistryRepository,
f func(registry registrytypes.Registry, a pkg.Artifact) response.Response,
info pkg.ArtifactInfo,
) response.Response {
var result response.Response
registry, err := registryDao.Get(ctx, info.RegistryID)
if err != nil {
log.Ctx(ctx).Error().Err(err).Msgf("Failed to get registry by ID %d", info.RegistryID)
return result
}
log.Ctx(ctx).Info().Msgf("Using Repository: %s, Type: %s", registry.Name, registry.Type)
art, ok := getArtifactRegistry(*registry).(pkg.Artifact)
if !ok {
log.Ctx(ctx).Error().Msgf("artifact %s is not a registry", registry.Name)
return result
}
result = f(*registry, art)
if pkg.IsEmpty(result.GetErrors()) {
return result
}
log.Ctx(ctx).Warn().Msgf("Repository: %s, Type: %s, errors: %v", registry.Name, registry.Type,
result.GetErrors())
return result
info pkg.PackageArtifactInfo,
) (response.Response, error) {
return proxyInternal(ctx, registryDao, f, info, false)
}
func ProxyWrapper(
ctx context.Context,
registryDao store.RegistryRepository,
f func(registry registrytypes.Registry, a pkg.Artifact) response.Response,
info pkg.ArtifactInfo,
) response.Response {
var response response.Response
requestRepoKey := info.RegIdentifier
if repos, err := getOrderedRepos(ctx, registryDao, requestRepoKey, *info.BaseInfo); err == nil {
for _, registry := range repos {
log.Ctx(ctx).Info().Msgf("Using Repository: %s, Type: %s", registry.Name, registry.Type)
artifact, ok := getArtifactRegistry(registry).(pkg.Artifact)
if !ok {
log.Ctx(ctx).Warn().Msgf("artifact %s is not a registry", registry.Name)
continue
}
if artifact != nil {
response = f(registry, artifact)
if pkg.IsEmpty(response.GetErrors()) {
return response
}
log.Ctx(ctx).Warn().Msgf("Repository: %s, Type: %s, errors: %v", registry.Name, registry.Type,
response.GetErrors())
info pkg.PackageArtifactInfo,
) (response.Response, error) {
return proxyInternal(ctx, registryDao, f, info, true)
}
func proxyInternal(
ctx context.Context,
registryDao store.RegistryRepository,
f func(registry registrytypes.Registry, a pkg.Artifact) response.Response,
info pkg.PackageArtifactInfo,
useUpstream bool,
) (response.Response, error) {
var r response.Response
requestRepoKey := info.BaseArtifactInfo().RegIdentifier
registries, skipped, err := filterRegs(ctx, registryDao, requestRepoKey, info, useUpstream)
if err != nil {
return r, err
}
for _, registry := range registries {
log.Ctx(ctx).Info().Msgf("Using Registry: %s, Type: %s", registry.Name, registry.Type)
art := getArtifactRegistry(registry)
if art != nil {
r = f(registry, art)
if r.GetError() == nil {
return r, nil
}
log.Ctx(ctx).Warn().Msgf("Repository: %s, Type: %s, error: %v", registry.Name, registry.Type,
r.GetError())
}
}
return response
if !pkg.IsEmpty(skipped) {
var skippedRegNames []string
for _, registry := range skipped {
skippedRegNames = append(skippedRegNames, registry.Name)
}
return r, errors.NotFound("no matching artifacts found in registry %s, skipped registries: [%s] "+
"due to allowed/blocked policies",
requestRepoKey, pkg.JoinWithSeparator(", ", skippedRegNames...))
}
return r, errors.NotFound("no matching artifacts found in registry %s", requestRepoKey)
}
func factory(key string) pkg.Artifact {
@ -110,23 +111,70 @@ func getArtifactRegistry(registry registrytypes.Registry) pkg.Artifact {
return factory(key)
}
func filterRegs(
ctx context.Context,
registryDao store.RegistryRepository,
repoKey string,
info pkg.PackageArtifactInfo,
upstream bool,
) (included []registrytypes.Registry, skipped []registrytypes.Registry, err error) {
registries, err := getOrderedRepos(ctx, registryDao, repoKey, info.BaseArtifactInfo().ParentID, upstream)
if err != nil {
return nil, nil, err
}
exists, imageVersion := info.GetImageVersion()
if !exists {
log.Debug().Msgf("image version not ready for %s, skipping filter", repoKey)
return registries, nil, nil
}
for _, repo := range registries {
allowedPatterns := repo.AllowedPattern
blockedPatterns := repo.BlockedPattern
isAllowed, err := utils.IsPatternAllowed(allowedPatterns, blockedPatterns, imageVersion)
if !isAllowed || err != nil {
log.Debug().Ctx(ctx).Msgf("Skipping repository %s", repo.Name)
skipped = append(skipped, repo)
continue
}
included = append(included, repo)
}
return included, skipped, nil
}
func getOrderedRepos(
ctx context.Context,
registryDao store.RegistryRepository,
repoKey string,
artInfo pkg.BaseInfo,
parentID int64,
upstream bool,
) ([]registrytypes.Registry, error) {
var result []registrytypes.Registry
if registry, err := registryDao.GetByParentIDAndName(ctx, artInfo.ParentID, repoKey); err == nil {
result = append(result, *registry)
proxies := registry.UpstreamProxies
if len(proxies) > 0 {
upstreamRepos, _ := registryDao.GetByIDIn(ctx, proxies)
result = append(result, *upstreamRepos...)
}
} else {
return result, err
registry, err := registryDao.GetByParentIDAndName(ctx, parentID, repoKey)
if err != nil {
return result, errors.NotFound("registry %s not found", repoKey)
}
result = append(result, *registry)
if !upstream {
return result, nil
}
proxies := registry.UpstreamProxies
if len(proxies) > 0 {
upstreamRepos, err2 := registryDao.GetByIDIn(ctx, proxies)
if err2 != nil {
log.Error().Msgf("Failed to get upstream proxies for %s: %v", repoKey, err2)
return result, err2
}
repoMap := make(map[int64]registrytypes.Registry)
for _, repo := range *upstreamRepos {
repoMap[repo.ID] = repo
}
for _, proxyID := range proxies {
if repo, ok := repoMap[proxyID]; ok {
result = append(result, repo)
}
}
}
return result, nil
}

6
registry/app/pkg/context.go
View File

@ -16,6 +16,7 @@ package pkg
import (
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/types"
v2 "github.com/distribution/distribution/v3/registry/api/v2"
)
@ -32,7 +33,10 @@ type ArtifactInfo struct {
*BaseInfo
RegIdentifier string
RegistryID int64
Image string
// Currently used only for Python packages
// TODO: extend to all package types
Registry types.Registry
Image string
}
type RegistryInfo struct {

28
registry/app/pkg/core_controller.go
View File

@ -68,16 +68,28 @@ func (c *CoreController) GetOrderedRepos(
artInfo BaseInfo,
) ([]types.Registry, error) {
var result []types.Registry
if registry, err := c.RegistryDao.GetByParentIDAndName(ctx, artInfo.ParentID, repoKey); err == nil {
result = append(result, *registry)
proxies := registry.UpstreamProxies
if len(proxies) > 0 {
upstreamRepos, _ := c.RegistryDao.GetByIDIn(ctx, proxies)
result = append(result, *upstreamRepos...)
}
} else {
registry, err := c.RegistryDao.GetByParentIDAndName(ctx, artInfo.ParentID, repoKey)
if err != nil {
return result, err
}
result = append(result, *registry)
proxies := registry.UpstreamProxies
if len(proxies) > 0 {
upstreamRepos, err2 := c.RegistryDao.GetByIDIn(ctx, proxies)
if err2 != nil {
log.Error().Msgf("Failed to get upstream proxies for %s: %v", repoKey, err2)
return result, err2
}
repoMap := make(map[int64]types.Registry)
for _, repo := range *upstreamRepos {
repoMap[repo.ID] = repo
}
for _, proxyID := range proxies {
if repo, ok := repoMap[proxyID]; ok {
result = append(result, repo)
}
}
}
return result, nil
}

26
registry/app/pkg/maven/controller.go
View File

@ -135,9 +135,16 @@ func (c *Controller) GetArtifact(ctx context.Context, info pkg.MavenArtifactInfo
f := func(registry registrytypes.Registry, a Artifact) Response {
info.SetMavenRepoKey(registry.Name)
info.RegistryID = registry.ID
headers, body, fileReader, redirectURL, e := a.(Registry).GetArtifact(ctx, info)
return &GetArtifactResponse{e, headers, redirectURL,
body, fileReader}
r, ok := a.(Registry)
if !ok {
log.Error().Stack().Msgf("Proxy wrapper has invalid registry set")
return nil
}
headers, body, fileReader, redirectURL, e := r.GetArtifact(ctx, info) //nolint:errcheck
return &GetArtifactResponse{
e, headers, redirectURL,
body, fileReader,
}
}
return c.ProxyWrapper(ctx, f, info)
}
@ -156,7 +163,12 @@ func (c *Controller) HeadArtifact(ctx context.Context, info pkg.MavenArtifactInf
f := func(registry registrytypes.Registry, a Artifact) Response {
info.SetMavenRepoKey(registry.Name)
info.RegistryID = registry.ID
headers, e := a.(Registry).HeadArtifact(ctx, info)
r, ok := a.(Registry)
if !ok {
log.Error().Stack().Msgf("Proxy wrapper has invalid registry set")
return nil
}
headers, e := r.HeadArtifact(ctx, info)
return &HeadArtifactResponse{e, headers}
}
return c.ProxyWrapper(ctx, f, info)
@ -224,7 +236,11 @@ func (c *Controller) GetOrderedRepos(
result = append(result, *registry)
proxies := registry.UpstreamProxies
if len(proxies) > 0 {
upstreamRepos, _ := c.DBStore.RegistryDao.GetByIDIn(ctx, proxies)
upstreamRepos, err := c.DBStore.RegistryDao.GetByIDIn(ctx, proxies)
if err != nil {
return result, err
}
result = append(result, *upstreamRepos...)
}
} else {

24
registry/app/pkg/python/local.go
View File

@ -21,12 +21,11 @@ import (
"io"
"mime/multipart"
"sort"
"strconv"
"strings"
urlprovider "github.com/harness/gitness/app/url"
"github.com/harness/gitness/errors"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
pythonmetadata "github.com/harness/gitness/registry/app/metadata/python"
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/base"
@ -92,11 +91,11 @@ func (c *localRegistry) GetPackageTypes() []artifact.PackageType {
func (c *localRegistry) DownloadPackageFile(
ctx context.Context,
info pythontype.ArtifactInfo,
) (*commons.ResponseHeaders, *storage.FileReader, io.ReadCloser, string, []error) {
headers, fileReader, redirectURL, errors := c.localBase.Download(ctx, info.ArtifactInfo, info.Version,
) (*commons.ResponseHeaders, *storage.FileReader, io.ReadCloser, string, error) {
headers, fileReader, redirectURL, err := c.localBase.Download(ctx, info.ArtifactInfo, info.Version,
info.Filename)
if len(errors) > 0 {
return nil, nil, nil, "", errors
if err != nil {
return nil, nil, nil, "", err
}
return headers, fileReader, nil, redirectURL, nil
}
@ -120,6 +119,11 @@ func (c *localRegistry) GetPackageMetadata(
return packageMetadata, err
}
if len(*artifacts) == 0 {
return packageMetadata, errors.NotFound("no artifacts found for registry %s and image %s", info.RegIdentifier,
info.Image)
}
for _, artifact := range *artifacts {
metadata := &pythonmetadata.PythonMetadata{}
err = json.Unmarshal(artifact.Metadata, metadata)
@ -173,9 +177,9 @@ func parseVersion(ctx context.Context, version string) []int {
parts := strings.Split(version, ".")
result := make([]int, len(parts))
for i, part := range parts {
num, err := strconv.Atoi(part)
num, err := pkg.ExtractFirstNumber(part)
if err != nil {
log.Debug().Ctx(ctx).Msgf("failed to parse version %s: %v", part, err)
log.Debug().Ctx(ctx).Msgf("failed to parse version %s, part %s: %v", version, part, err)
continue
}
result[i] = num
@ -188,7 +192,7 @@ func (c *localRegistry) UploadPackageFile(
info pythontype.ArtifactInfo,
file multipart.File,
filename string,
) (headers *commons.ResponseHeaders, sha256 string, err errcode.Error) {
) (headers *commons.ResponseHeaders, sha256 string, err error) {
defer file.Close()
path := pkg.JoinWithSeparator("/", info.Image, info.Metadata.Version, filename)
return c.localBase.UploadFile(ctx, info.ArtifactInfo, filename, info.Metadata.Version, path, file,
@ -202,7 +206,7 @@ func (c *localRegistry) UploadPackageFileReader(
info pythontype.ArtifactInfo,
file io.ReadCloser,
filename string,
) (headers *commons.ResponseHeaders, sha256 string, err errcode.Error) {
) (headers *commons.ResponseHeaders, sha256 string, err error) {
defer file.Close()
path := pkg.JoinWithSeparator("/", info.Image, info.Metadata.Version, filename)
return c.localBase.Upload(ctx, info.ArtifactInfo, filename, info.Metadata.Version, path, file,

9
registry/app/pkg/python/local_helper.go
View File

@ -18,7 +18,6 @@ import (
"context"
"io"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
"github.com/harness/gitness/registry/app/pkg/base"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/types/python"
@ -31,14 +30,14 @@ type LocalRegistryHelper interface {
*commons.ResponseHeaders,
*storage.FileReader,
string,
[]error,
error,
)
UploadPackageFile(
ctx context.Context,
info python.ArtifactInfo,
fileReader io.ReadCloser,
filename string,
) (*commons.ResponseHeaders, string, errcode.Error)
) (*commons.ResponseHeaders, string, error)
}
type localRegistryHelper struct {
@ -61,7 +60,7 @@ func (h *localRegistryHelper) DownloadFile(ctx context.Context, info python.Arti
*commons.ResponseHeaders,
*storage.FileReader,
string,
[]error,
error,
) {
return h.localBase.Download(ctx, info.ArtifactInfo, info.Version, info.Filename)
}
@ -71,6 +70,6 @@ func (h *localRegistryHelper) UploadPackageFile(
info python.ArtifactInfo,
fileReader io.ReadCloser,
filename string,
) (*commons.ResponseHeaders, string, errcode.Error) {
) (*commons.ResponseHeaders, string, error) {
return h.localRegistry.UploadPackageFileReader(ctx, info, fileReader, filename)
}

22
registry/app/pkg/python/proxy.go
View File

@ -100,32 +100,32 @@ func (r *proxy) DownloadPackageFile(ctx context.Context, info pythontype.Artifac
*storage.FileReader,
io.ReadCloser,
string,
[]error,
error,
) {
upstreamProxy, err := r.proxyStore.GetByRegistryIdentifier(ctx, info.ParentID, info.RegIdentifier)
if err != nil {
return nil, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)}
return nil, nil, nil, "", err
}
// TODO: Extract out to Path Utils for all package types
exists := r.localRegistryHelper.FileExists(ctx, info)
if exists {
headers, fileReader, redirectURL, errors := r.localRegistryHelper.DownloadFile(ctx, info)
if len(errors) == 0 {
return headers, fileReader, nil, redirectURL, errors
headers, fileReader, redirectURL, err := r.localRegistryHelper.DownloadFile(ctx, info)
if err == nil {
return headers, fileReader, nil, redirectURL, nil
}
// If file exists in local registry, but download failed, we should try to download from remote
log.Warn().Ctx(ctx).Msgf("failed to pull from local, attempting streaming from remote, %v", errors)
log.Warn().Ctx(ctx).Msgf("failed to pull from local, attempting streaming from remote, %v", err)
}
remote, err := NewRemoteRegistryHelper(ctx, r.spaceFinder, *upstreamProxy, r.service)
if err != nil {
return nil, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)}
return nil, nil, nil, "", err
}
file, err := remote.GetFile(ctx, info.Image, info.Filename)
if err != nil {
return nil, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)}
return nil, nil, nil, "", errcode.ErrCodeUnknown.WithDetail(err)
}
go func(info pythontype.ArtifactInfo) {
@ -202,7 +202,7 @@ func (r *proxy) putFileToLocal(ctx context.Context, pkg string, filename string,
info.Filename = filename
_, sha256, err2 := r.localRegistryHelper.UploadPackageFile(ctx, *info, file, filename)
if !commons.IsEmptyError(err2) {
if err2 != nil {
log.Ctx(ctx).Error().Stack().Err(err2).Msgf("uploading file %s failed, %v", filename, err)
return err2
}
@ -217,7 +217,7 @@ func (r *proxy) UploadPackageFile(
_ pythontype.ArtifactInfo,
_ multipart.File,
_ string,
) (*commons.ResponseHeaders, string, errcode.Error) {
) (*commons.ResponseHeaders, string, error) {
log.Error().Ctx(ctx).Msg("Not implemented")
return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(fmt.Errorf("not implemented"))
}
@ -229,7 +229,7 @@ func (r *proxy) UploadPackageFileReader(
_ pythontype.ArtifactInfo,
_ io.ReadCloser,
_ string,
) (*commons.ResponseHeaders, string, errcode.Error) {
) (*commons.ResponseHeaders, string, error) {
log.Error().Ctx(ctx).Msg("Not implemented")
return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(fmt.Errorf("not implemented"))
}

7
registry/app/pkg/python/registry.go
View File

@ -19,7 +19,6 @@ import (
"io"
"mime/multipart"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/types/python"
@ -36,20 +35,20 @@ type Registry interface {
info python.ArtifactInfo,
file multipart.File,
filename string,
) (*commons.ResponseHeaders, string, errcode.Error)
) (*commons.ResponseHeaders, string, error)
UploadPackageFileReader(
ctx context.Context,
info python.ArtifactInfo,
file io.ReadCloser,
filename string,
) (*commons.ResponseHeaders, string, errcode.Error)
) (*commons.ResponseHeaders, string, error)
DownloadPackageFile(ctx context.Context, info python.ArtifactInfo) (
*commons.ResponseHeaders,
*storage.FileReader,
io.ReadCloser,
string,
[]error,
error,
)
}

3
registry/app/pkg/response/response.go
View File

@ -15,6 +15,5 @@
package response
type Response interface {
GetErrors() []error
SetError(error)
GetError() error
}

27
registry/app/pkg/types/python/types.go
View File

@ -1,16 +1,16 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package python
@ -28,11 +28,18 @@ type ArtifactInfo struct {
Metadata python.Metadata
}
// BaseArtifactInfo implements pkg.PackageArtifactInfo interface
// BaseArtifactInfo implements pkg.PackageArtifactInfo interface.
func (a ArtifactInfo) BaseArtifactInfo() pkg.ArtifactInfo {
return a.ArtifactInfo
}
func (a ArtifactInfo) GetImageVersion() (exists bool, imageVersion string) {
if a.Image != "" && a.Version != "" {
return true, pkg.JoinWithSeparator(":", a.Image, a.Version)
}
return false, ""
}
type File struct {
FileURL string
Name string

22
registry/app/pkg/utils.go
View File

@ -15,10 +15,17 @@
package pkg
import (
"fmt"
"reflect"
"regexp"
"strconv"
"strings"
)
var (
numberRegex = regexp.MustCompile(`\d+`)
)
func IsEmpty(slice interface{}) bool {
if slice == nil {
return true
@ -29,3 +36,18 @@ func IsEmpty(slice interface{}) bool {
func JoinWithSeparator(sep string, args ...string) string {
return strings.Join(args, sep)
}
func ExtractFirstNumber(input string) (int, error) {
match := numberRegex.FindString(input)
if match == "" {
return 0, fmt.Errorf("no number found in input: %s", input)
}
result, err := strconv.Atoi(match)
if err != nil {
return 0, fmt.Errorf("failed to convert string '%s' to number: %w", match, err)
}
return result, nil
}

161
registry/app/pkg/utils_test.go Normal file
View File

@ -0,0 +1,161 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pkg
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestIsEmpty(t *testing.T) {
tests := []struct {
name string
input interface{}
expected bool
}{
{
name: "nil slice",
input: nil,
expected: true,
},
{
name: "empty string slice",
input: []string{},
expected: true,
},
{
name: "non-empty string slice",
input: []string{"a", "b"},
expected: false,
},
{
name: "empty int slice",
input: []int{},
expected: true,
},
{
name: "non-empty int slice",
input: []int{1, 2, 3},
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := IsEmpty(tt.input)
assert.Equal(t, tt.expected, result)
})
}
}
func TestJoinWithSeparator(t *testing.T) {
tests := []struct {
name string
sep string
args []string
expected string
}{
{
name: "join with comma",
sep: ",",
args: []string{"a", "b", "c"},
expected: "a,b,c",
},
{
name: "join with space",
sep: " ",
args: []string{"hello", "world"},
expected: "hello world",
},
{
name: "empty strings",
sep: "-",
args: []string{"", "", ""},
expected: "--",
},
{
name: "single string",
sep: ".",
args: []string{"single"},
expected: "single",
},
{
name: "no strings",
sep: "+",
args: []string{},
expected: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := JoinWithSeparator(tt.sep, tt.args...)
assert.Equal(t, tt.expected, result)
})
}
}
func TestExtractFirstNumber(t *testing.T) {
tests := []struct {
name string
input string
expected int
expectError bool
}{
{
name: "simple number",
input: "123",
expected: 123,
expectError: false,
},
{
name: "number with text",
input: "abc123def",
expected: 123,
expectError: false,
},
{
name: "multiple numbers",
input: "123abc456",
expected: 123,
expectError: false,
},
{
name: "no numbers",
input: "abc",
expected: 0,
expectError: true,
},
{
name: "empty string",
input: "",
expected: 0,
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := ExtractFirstNumber(tt.input)
if tt.expectError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, tt.expected, result)
}
})
}
}

11
registry/app/remote/adapter/commons/pypi/dto.go
View File

@ -17,6 +17,7 @@ package pypi
import (
"errors"
"fmt"
"regexp"
"strings"
"golang.org/x/net/html"
@ -35,6 +36,8 @@ var (
".app",
".dmg",
}
exeRegex = regexp.MustCompile(`(\d+(?:\.\d+)+)`)
)
type SimpleMetadata struct {
@ -90,7 +93,13 @@ func GetPyPIVersion(filename string) string {
switch ext {
case ".whl", ".egg":
return splits[1]
case ".tar.gz", ".tar.bz2", ".tar.xz", ".zip", ".dmg", ".app", ".exe":
case ".tar.gz", ".tar.bz2", ".tar.xz", ".zip", ".dmg", ".app":
return splits[len(splits)-1]
case ".exe":
match := exeRegex.FindStringSubmatch(filename)
if len(match) > 1 {
return match[1]
}
return splits[len(splits)-1]
default:
return ""