USEFULL
/
drone
mirror of https://github.com/harness/drone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: [AH-987]: New Package Routing implementation done (#3490) 

* [AH-987]: Updated lint issues
* [AH-987]: Formatting fixes
* [AH-987]: Base changes
* [AH-987]: Base changes
try-new-ui
Arvind Choudhary 2025-03-03 22:07:31 +00:00 committed by Harness
parent 1d73543050
commit 6d739f624a
20 changed files with 860 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/gitness/wire_gen.go
View File

@ -124,6 +124,7 @@ import (
"github.com/harness/gitness/registry/app/pkg/filemanager"
"github.com/harness/gitness/registry/app/pkg/generic"
"github.com/harness/gitness/registry/app/pkg/maven"
"github.com/harness/gitness/registry/app/pkg/pypi"
database2 "github.com/harness/gitness/registry/app/store/database"
"github.com/harness/gitness/registry/gc"
"github.com/harness/gitness/ssh"
@ -482,7 +483,11 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
genericController := generic.ControllerProvider(spaceStore, authorizer, fileManager, genericDBStore, transactor)
genericHandler := api2.NewGenericHandlerProvider(spaceStore, genericController, tokenStore, controller, authenticator, provider, authorizer)
handler3 := router.GenericHandlerProvider(genericHandler)
appRouter := router.AppRouterProvider(registryOCIHandler, apiHandler, handler2, handler3)
packagesHandler := api2.NewPackageHandlerProvider(registryRepository, spaceStore, tokenStore, controller, authenticator, provider, authorizer)
pypiController := pypi.ControllerProvider(artifactRepository, upstreamProxyConfigRepository)
pypiHandler := api2.NewPypiHandlerProvider(pypiController, packagesHandler)
handler4 := router.PackageHandlerProvider(packagesHandler, mavenHandler, genericHandler, pypiHandler)
appRouter := router.AppRouterProvider(registryOCIHandler, apiHandler, handler2, handler3, handler4)
sender := usage.ProvideMediator(ctx, config, spaceFinder, usageMetricStore)
routerRouter := router2.ProvideRouter(ctx, config, authenticator, repoController, reposettingsController, executionController, logsController, spaceController, pipelineController, secretController, triggerController, connectorController, templateController, pluginController, pullreqController, webhookController, githookController, gitInterface, serviceaccountController, controller, principalController, usergroupController, checkController, systemController, uploadController, keywordsearchController, infraproviderController, gitspaceController, migrateController, provider, openapiService, appRouter, sender)
serverServer := server2.ProvideServer(config, routerRouter)

2
registry/app/api/controller/metadata/artifact_mapper.go
View File

@ -121,6 +121,8 @@ func toPackageType(packageTypeStr string) (artifactapi.PackageType, error) {
return artifactapi.PackageTypeHELM, nil
case string(artifactapi.PackageTypeMAVEN):
return artifactapi.PackageTypeMAVEN, nil
case string(artifactapi.PackageTypePYPI):
return artifactapi.PackageTypePYPI, nil
default:
return "", errors.New("invalid package type")
}

143
registry/app/api/controller/metadata/get_client_setup_details.go
View File

@ -131,6 +131,8 @@ func (c *APIController) GenerateClientSetupDetails(
loginPasswordLabel, username, registryRef, image, tag)
case string(artifact.PackageTypeGENERIC):
return c.generateGenericClientSetupDetail(ctx, blankString, registryRef, image, tag)
case string(artifact.PackageTypePYPI):
return c.generatePyPIClientSetupDetail(ctx, registryRef, username, image, tag)
}
header1 := "Login to Docker"
section1step1Header := "Run this Docker command in your terminal to authenticate the client."
@ -230,8 +232,10 @@ func (c *APIController) GenerateClientSetupDetails(
}
//nolint:lll
func (c *APIController) generateGenericClientSetupDetail(ctx context.Context, blankString string,
registryRef string, image *artifact.ArtifactParam, tag *artifact.VersionParam) *artifact.ClientSetupDetailsResponseJSONResponse {
func (c *APIController) generateGenericClientSetupDetail(
ctx context.Context, blankString string,
registryRef string, image *artifact.ArtifactParam, tag *artifact.VersionParam,
) *artifact.ClientSetupDetailsResponseJSONResponse {
header1 := "Generate identity token"
section1Header := "An identity token will serve as the password for uploading and downloading artifact."
section1Type := artifact.ClientSetupStepTypeGenerateToken
@ -302,7 +306,8 @@ func (c *APIController) generateGenericClientSetupDetail(ctx context.Context, bl
},
}
//nolint:lll
c.replacePlaceholders(ctx, &clientSetupDetails.Sections, "", registryRef, image, tag, "", "", string(artifact.PackageTypeGENERIC))
c.replacePlaceholders(ctx, &clientSetupDetails.Sections, "", registryRef, image, tag, "", "",
string(artifact.PackageTypeGENERIC))
return &artifact.ClientSetupDetailsResponseJSONResponse{
Data: clientSetupDetails,
Status: artifact.StatusSUCCESS,
@ -310,8 +315,17 @@ func (c *APIController) generateGenericClientSetupDetail(ctx context.Context, bl
}
//nolint:lll
func (c *APIController) generateHelmClientSetupDetail(ctx context.Context, blankString string,
loginUsernameLabel string, loginUsernameValue string, loginPasswordLabel string, username string, registryRef string, image *artifact.ArtifactParam, tag *artifact.VersionParam) *artifact.ClientSetupDetailsResponseJSONResponse {
func (c *APIController) generateHelmClientSetupDetail(
ctx context.Context,
blankString string,
loginUsernameLabel string,
loginUsernameValue string,
loginPasswordLabel string,
username string,
registryRef string,
image *artifact.ArtifactParam,
tag *artifact.VersionParam,
) *artifact.ClientSetupDetailsResponseJSONResponse {
header1 := "Login to Helm"
section1step1Header := "Run this Helm command in your terminal to authenticate the client."
helmLoginValue := "helm registry login <LOGIN_HOSTNAME>"
@ -569,12 +583,12 @@ func (c *APIController) generateMavenClientSetupDetail(
_ = gradleSection2.FromClientSetupStepConfig(artifact.ClientSetupStepConfig{
Steps: &[]artifact.ClientSetupStep{
{
Header: stringPtr("Add a maven publish plugin configuration to the project’s build.gradle."),
Header: stringPtr("Add a maven publish plugin configuration to the project's build.gradle."),
Type: &staticStepType,
Commands: &[]artifact.ClientSetupStepCommand{
{
//nolint:lll
Value: stringPtr("publishing {\n publications {\n maven(MavenPublication) {\n groupId = '<GROUP_ID>'\n artifactId = '<ARTIFACT_ID>'\n version = '<VERSION>'\n\n from components.java\n }\n }\n}"),
Value: stringPtr("publishing {\n publications {\n maven(MavenPublication) {\n groupId = 'GROUP_ID'\n artifactId = 'ARTIFACT_ID'\n version = 'VERSION'\n\n from components.java\n }\n }\n}"),
},
},
},
@ -714,7 +728,105 @@ func (c *APIController) generateMavenClientSetupDetail(
registryURL := c.URLProvider.RegistryURL(ctx, "maven", rootSpace)
//nolint:lll
c.replacePlaceholders(ctx, &clientSetupDetails.Sections, username, registryRef, artifactName, version, registryURL, groupID, "")
c.replacePlaceholders(ctx, &clientSetupDetails.Sections, username, registryRef, artifactName, version, registryURL,
groupID, "")
return &artifact.ClientSetupDetailsResponseJSONResponse{
Data: clientSetupDetails,
Status: artifact.StatusSUCCESS,
}
}
func (c *APIController) generatePyPIClientSetupDetail(
ctx context.Context,
registryRef string,
username string,
image *artifact.ArtifactParam,
tag *artifact.VersionParam,
) *artifact.ClientSetupDetailsResponseJSONResponse {
staticStepType := artifact.ClientSetupStepTypeStatic
generateTokenType := artifact.ClientSetupStepTypeGenerateToken
// Authentication section
section1 := artifact.ClientSetupSection{
Header: stringPtr("1. Configure Authentication"),
}
_ = section1.FromClientSetupStepConfig(artifact.ClientSetupStepConfig{
Steps: &[]artifact.ClientSetupStep{
{
Header: stringPtr("Create or update your ~/.pypirc file with the following content:"),
Type: &staticStepType,
Commands: &[]artifact.ClientSetupStepCommand{
{
Value: stringPtr("[distutils]\n" +
"index-servers = harness\n\n" +
"[harness]\n" +
"repository: <REGISTRY_URL>/<REGISTRY_NAME>\n" +
"username: <USERNAME>\n" +
"password: {{identity-token}}"),
},
},
},
{
Header: stringPtr("Generate an identity token for authentication"),
Type: &generateTokenType,
},
},
})
// Install section
section2 := artifact.ClientSetupSection{
Header: stringPtr("2. Install Package"),
}
_ = section2.FromClientSetupStepConfig(artifact.ClientSetupStepConfig{
Steps: &[]artifact.ClientSetupStep{
{
Header: stringPtr("Install a package using pip:"),
Type: &staticStepType,
Commands: &[]artifact.ClientSetupStepCommand{
{
Value: stringPtr("pip install --index-url <REGISTRY_URL>/<REGISTRY_NAME> <PACKAGE_NAME>==<VERSION>"),
},
},
},
},
})
// Publish section
section3 := artifact.ClientSetupSection{
Header: stringPtr("3. Publish Package"),
}
_ = section3.FromClientSetupStepConfig(artifact.ClientSetupStepConfig{
Steps: &[]artifact.ClientSetupStep{
{
Header: stringPtr("Build and publish your package:"),
Type: &staticStepType,
Commands: &[]artifact.ClientSetupStepCommand{
{
Value: stringPtr("python -m build"),
},
{
Value: stringPtr("python -m twine upload --repository harness dist/*"),
},
},
},
},
})
clientSetupDetails := artifact.ClientSetupDetails{
MainHeader: "PyPI Client Setup",
SecHeader: "Follow these instructions to install/use Python packages from this registry.",
Sections: []artifact.ClientSetupSection{
section1,
section2,
section3,
},
}
rootSpace, _, _ := paths.DisectRoot(registryRef)
registryURL := c.URLProvider.RegistryURL(ctx, "pypi", rootSpace)
c.replacePlaceholders(ctx, &clientSetupDetails.Sections, username, registryRef, image, tag, registryURL, "", "pypi")
return &artifact.ClientSetupDetailsResponseJSONResponse{
Data: clientSetupDetails,
@ -737,10 +849,12 @@ func (c *APIController) replacePlaceholders(
tab, err := (*clientSetupSections)[i].AsTabSetupStepConfig()
if err != nil || tab.Tabs == nil {
//nolint:lll
c.replacePlaceholdersInSection(ctx, &(*clientSetupSections)[i], username, regRef, image, tag, pkgType, groupID, registryURL)
c.replacePlaceholdersInSection(ctx, &(*clientSetupSections)[i], username, regRef, image, tag, pkgType,
groupID, registryURL)
} else {
for j := range *tab.Tabs {
c.replacePlaceholders(ctx, (*tab.Tabs)[j].Sections, username, regRef, image, tag, groupID, registryURL, pkgType)
c.replacePlaceholders(ctx, (*tab.Tabs)[j].Sections, username, regRef, image, tag, groupID, registryURL,
pkgType)
}
_ = (*clientSetupSections)[i].FromTabSetupStepConfig(tab)
}
@ -816,9 +930,12 @@ func replaceText(
(*st.Commands)[i].Value = stringPtr(strings.ReplaceAll(*(*st.Commands)[i].Value, "<REGISTRY_NAME>", repoName))
}
if image != nil {
(*st.Commands)[i].Value = stringPtr(strings.ReplaceAll(*(*st.Commands)[i].Value, "<IMAGE_NAME>", string(*image)))
(*st.Commands)[i].Value = stringPtr(strings.ReplaceAll(*(*st.Commands)[i].Value, "<ARTIFACT_ID>", string(*image)))
(*st.Commands)[i].Value = stringPtr(strings.ReplaceAll(*(*st.Commands)[i].Value, "<ARTIFACT_NAME>", string(*image)))
(*st.Commands)[i].Value = stringPtr(strings.ReplaceAll(*(*st.Commands)[i].Value, "<IMAGE_NAME>",
string(*image)))
(*st.Commands)[i].Value = stringPtr(strings.ReplaceAll(*(*st.Commands)[i].Value, "<ARTIFACT_ID>",
string(*image)))
(*st.Commands)[i].Value = stringPtr(strings.ReplaceAll(*(*st.Commands)[i].Value, "<ARTIFACT_NAME>",
string(*image)))
}
if tag != nil {
(*st.Commands)[i].Value = stringPtr(strings.ReplaceAll(*(*st.Commands)[i].Value, "<TAG>", string(*tag)))

264
registry/app/api/handler/packages/handler.go Normal file
View File

@ -0,0 +1,264 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package packages
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
usercontroller "github.com/harness/gitness/app/api/controller/user"
"github.com/harness/gitness/app/auth/authn"
"github.com/harness/gitness/app/auth/authz"
corestore "github.com/harness/gitness/app/store"
urlprovider "github.com/harness/gitness/app/url"
"github.com/harness/gitness/registry/app/api/controller/metadata"
"github.com/harness/gitness/registry/app/api/handler/utils"
artifact2 "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/types/enum"
"github.com/rs/zerolog/log"
)
const (
packageNameRegex = `^[a-zA-Z0-9][a-zA-Z0-9._-]*[a-zA-Z0-9]$`
versionRegex = `^[a-z0-9][a-z0-9.-]*[a-z0-9]$`
filenameRegex = `^[a-zA-Z0-9][a-zA-Z0-9._~@,/-]*[a-zA-Z0-9]$`
// Add other route types here.
)
func NewHandler(
registryDao store.RegistryRepository,
spaceStore corestore.SpaceStore, tokenStore corestore.TokenStore,
userCtrl *usercontroller.Controller, authenticator authn.Authenticator,
urlProvider urlprovider.Provider, authorizer authz.Authorizer,
) Handler {
return &handler{
RegistryDao: registryDao,
SpaceStore: spaceStore,
TokenStore: tokenStore,
UserCtrl: userCtrl,
Authenticator: authenticator,
URLProvider: urlProvider,
Authorizer: authorizer,
}
}
type handler struct {
RegistryDao store.RegistryRepository
SpaceStore corestore.SpaceStore
TokenStore corestore.TokenStore
UserCtrl *usercontroller.Controller
Authenticator authn.Authenticator
URLProvider urlprovider.Provider
Authorizer authz.Authorizer
}
type Handler interface {
GetRegistryCheckAccess(
ctx context.Context,
r *http.Request,
reqPermissions ...enum.Permission,
) error
GetArtifactInfo(r *http.Request) (pkg.GenericArtifactInfo, errcode.Error)
GetAuthenticator() authn.Authenticator
}
func (h *handler) GetAuthenticator() authn.Authenticator {
return h.Authenticator
}
func (h *handler) GetRegistryCheckAccess(
ctx context.Context,
r *http.Request,
reqPermissions ...enum.Permission,
) error {
info, _ := h.GetArtifactInfo(r)
return pkg.GetRegistryCheckAccess(ctx, h.RegistryDao, h.Authorizer,
h.SpaceStore,
info.RegIdentifier, info.ParentID, reqPermissions...)
}
func (h *handler) GetArtifactInfo(r *http.Request) (pkg.GenericArtifactInfo, errcode.Error) {
ctx := r.Context()
path := r.URL.Path
rootIdentifier, registryIdentifier, artifact, tag, fileName, description, err := ExtractPathVars(r)
if err != nil {
return pkg.GenericArtifactInfo{}, errcode.ErrCodeInvalidRequest.WithDetail(err)
}
if err := metadata.ValidateIdentifier(registryIdentifier); err != nil {
return pkg.GenericArtifactInfo{}, errcode.ErrCodeInvalidRequest.WithDetail(err)
}
if err := validatePackageVersionAndFileName(artifact, tag, fileName); err != nil {
return pkg.GenericArtifactInfo{}, errcode.ErrCodeInvalidRequest.WithDetail(err)
}
rootSpace, err := h.SpaceStore.FindByRefCaseInsensitive(ctx, rootIdentifier)
if err != nil {
log.Ctx(ctx).Error().Msgf("Root space not found: %s", rootIdentifier)
return pkg.GenericArtifactInfo{}, errcode.ErrCodeRootNotFound.WithDetail(err)
}
registry, err := h.RegistryDao.GetByRootParentIDAndName(ctx, rootSpace.ID,
registryIdentifier)
if err != nil {
log.Ctx(ctx).Error().Msgf(
"registry %s not found for root: %s. Reason: %s", registryIdentifier, rootSpace.Identifier, err,
)
return pkg.GenericArtifactInfo{}, errcode.ErrCodeRegNotFound.WithDetail(err)
}
if registry.PackageType != artifact2.PackageTypeGENERIC {
log.Ctx(ctx).Error().Msgf(
"registry %s is not a generic artifact registry for root: %s", registryIdentifier, rootSpace.Identifier,
)
return pkg.GenericArtifactInfo{}, errcode.ErrCodeInvalidRequest.WithDetail(fmt.Errorf("registry %s is"+
" not a generic artifact registry", registryIdentifier))
}
_, err = h.SpaceStore.Find(r.Context(), registry.ParentID)
if err != nil {
log.Ctx(ctx).Error().Msgf("Parent space not found: %d", registry.ParentID)
return pkg.GenericArtifactInfo{}, errcode.ErrCodeParentNotFound.WithDetail(err)
}
info := &pkg.GenericArtifactInfo{
ArtifactInfo: &pkg.ArtifactInfo{
BaseInfo: &pkg.BaseInfo{
RootIdentifier: rootIdentifier,
RootParentID: rootSpace.ID,
ParentID: registry.ParentID,
},
RegIdentifier: registryIdentifier,
Image: artifact,
},
RegistryID: registry.ID,
Version: tag,
FileName: fileName,
Description: description,
}
log.Ctx(ctx).Info().Msgf("Dispatch: URI: %s", path)
if commons.IsEmpty(rootSpace.Identifier) {
log.Ctx(ctx).Error().Msgf("ParentRef not found in context")
return pkg.GenericArtifactInfo{}, errcode.ErrCodeParentNotFound.WithDetail(err)
}
if commons.IsEmpty(registryIdentifier) {
log.Ctx(ctx).Warn().Msgf("registry not found in context")
return pkg.GenericArtifactInfo{}, errcode.ErrCodeRegNotFound.WithDetail(err)
}
if !commons.IsEmpty(info.Image) && !commons.IsEmpty(info.Version) && !commons.IsEmpty(info.FileName) {
flag, err2 := utils.MatchArtifactFilter(registry.AllowedPattern, registry.BlockedPattern,
info.Image+":"+info.Version+":"+info.FileName)
if !flag || err2 != nil {
return pkg.GenericArtifactInfo{}, errcode.ErrCodeInvalidRequest.WithDetail(err2)
}
}
return *info, errcode.Error{}
}
// ExtractPathVars extracts registry,image, reference, digest and tag from the path
// Path format: /generic/:rootSpace/:registry/:image/:tag (for ex:
// /generic/myRootSpace/reg1/alpine/v1).
func ExtractPathVars(r *http.Request) (
rootIdentifier, registry, artifact,
tag, fileName string, description string, err error,
) {
path := r.URL.Path
// Ensure the path starts with "/generic/"
if !strings.HasPrefix(path, "/generic/") {
return "", "", "", "", "", "", fmt.Errorf("invalid path: must start with /generic/")
}
trimmedPath := strings.TrimPrefix(path, "/generic/")
firstSlashIndex := strings.Index(trimmedPath, "/")
if firstSlashIndex == -1 {
return "", "", "", "", "", "", fmt.Errorf("invalid path format: missing rootIdentifier or registry")
}
rootIdentifier = trimmedPath[:firstSlashIndex]
remainingPath := trimmedPath[firstSlashIndex+1:]
secondSlashIndex := strings.Index(remainingPath, "/")
if secondSlashIndex == -1 {
return "", "", "", "", "", "", fmt.Errorf("invalid path format: missing registry")
}
registry = remainingPath[:secondSlashIndex]
// Extract the artifact and tag from the remaining path
artifactPath := remainingPath[secondSlashIndex+1:]
// Check if the artifactPath contains a ":" for tag and filename
if strings.Contains(artifactPath, ":") {
segments := strings.SplitN(artifactPath, ":", 3)
if len(segments) < 3 {
return "", "", "", "", "", "", fmt.Errorf("invalid artifact format: %s", artifactPath)
}
artifact = segments[0]
tag = segments[1]
fileName = segments[2]
} else {
segments := strings.SplitN(artifactPath, "/", 2)
if len(segments) < 2 {
return "", "", "", "", "", "", fmt.Errorf("invalid artifact format: %s", artifactPath)
}
artifact = segments[0]
tag = segments[1]
fileName = r.FormValue("filename")
if fileName == "" {
return "", "", "", "", "", "", fmt.Errorf("filename not provided in path or form parameter")
}
}
description = r.FormValue("description")
return rootIdentifier, registry, artifact, tag, fileName, description, nil
}
func validatePackageVersionAndFileName(packageName, version, filename string) error {
// Compile the regular expressions
packageNameRe := regexp.MustCompile(packageNameRegex)
versionRe := regexp.MustCompile(versionRegex)
filenameRe := regexp.MustCompile(filenameRegex)
// Validate package name
if !packageNameRe.MatchString(packageName) {
return fmt.Errorf("invalid package name: %s", packageName)
}
// Validate version
if !versionRe.MatchString(version) {
return fmt.Errorf("invalid version: %s", version)
}
// Validate filename
if !filenameRe.MatchString(filename) {
return fmt.Errorf("invalid filename: %s", filename)
}
return nil
}

23
registry/app/api/handler/pypi/download.go Normal file
View File

@ -0,0 +1,23 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pypi
import (
"net/http"
)
func (h *handler) DownloadPackageFile(_ http.ResponseWriter, _ *http.Request) {
}

45
registry/app/api/handler/pypi/handler.go Normal file
View File

@ -0,0 +1,45 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pypi
import (
"net/http"
"github.com/harness/gitness/registry/app/api/handler/packages"
"github.com/harness/gitness/registry/app/pkg/pypi"
)
type Handler interface {
DownloadPackageFile(http.ResponseWriter, *http.Request)
UploadPackageFile(writer http.ResponseWriter, request *http.Request)
PackageMetadata(writer http.ResponseWriter, request *http.Request)
}
type handler struct {
packages.Handler
controller pypi.Controller
}
func NewHandler(
controller pypi.Controller,
packageHandler packages.Handler,
) Handler {
return &handler{
Handler: packageHandler,
controller: controller,
}
}
var _ Handler = (*handler)(nil)

22
registry/app/api/handler/pypi/list.go Normal file
View File

@ -0,0 +1,22 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pypi
import (
"net/http"
)
func (h *handler) PackageMetadata(_ http.ResponseWriter, _ *http.Request) {
}

23
registry/app/api/handler/pypi/upload.go Normal file
View File

@ -0,0 +1,23 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pypi
import (
"net/http"
)
func (h *handler) UploadPackageFile(_ http.ResponseWriter, _ *http.Request) {
}

46
registry/app/api/middleware/request_package_access.go Normal file
View File

@ -0,0 +1,46 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package middleware
import (
"net/http"
"github.com/harness/gitness/app/api/render"
"github.com/harness/gitness/registry/app/api/handler/packages"
"github.com/harness/gitness/types/enum"
"github.com/rs/zerolog/log"
)
// StoreOriginalURL stores the original URL in the context.
func RequestPackageAccess(
packageHandler packages.Handler,
reqPermissions ...enum.Permission,
) func(next http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
err := packageHandler.GetRegistryCheckAccess(r.Context(), r, reqPermissions...)
if err != nil {
log.Info().Err(err).Msgf("Access denied for path: %s, method: %s, permission: %s", r.URL.Path, r.Method,
reqPermissions)
render.Forbiddenf(r.Context(), w,
"Access denied as permission: %s is required for path: %s, method: %s", reqPermissions, r.URL.Path,
r.Method)
}
next.ServeHTTP(w, r.WithContext(r.Context()))
})
}
}

1
registry/app/api/openapi/contracts/artifact/types.gen.go
View File

@ -28,6 +28,7 @@ const (
const (
PackageTypeDOCKER PackageType = "DOCKER"
PackageTypeGENERIC PackageType = "GENERIC"
PackageTypePYPI PackageType = "PYPI"
PackageTypeHELM PackageType = "HELM"
PackageTypeMAVEN PackageType = "MAVEN"
)

1
registry/app/api/router/generic/route.go
View File

@ -37,6 +37,7 @@ func NewGenericArtifactHandler(handler *generic.Handler) Handler {
http.MethodGet: handler.PullArtifact,
}
r.Route("/generic", func(r chi.Router) {
r.Use(middleware.StoreOriginalURL)
r.Use(middlewareauthn.Attempt(handler.Authenticator))
r.Use(middleware.TrackDownloadStatForGenericArtifact(handler))
r.Use(middleware.TrackBandwidthStatForGenericArtifacts(handler))

84
registry/app/api/router/packages/route.go Normal file
View File

@ -0,0 +1,84 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package packages
import (
"net/http"
middlewareauthn "github.com/harness/gitness/app/api/middleware/authn"
"github.com/harness/gitness/registry/app/api/handler/generic"
"github.com/harness/gitness/registry/app/api/handler/maven"
"github.com/harness/gitness/registry/app/api/handler/packages"
"github.com/harness/gitness/registry/app/api/handler/pypi"
"github.com/harness/gitness/registry/app/api/middleware"
"github.com/harness/gitness/types/enum"
"github.com/go-chi/chi/v5"
)
type Handler interface {
http.Handler
}
/**
* NewRouter creates a new router for the package API.
* It sets up the routes and middleware for handling package-related requests.
* Paths look like:
* For all packages: /{rootIdentifier}/{registryName}/<package_type>/<package specific routes> .
*/
func NewRouter(
packageHandler packages.Handler,
mavenHandler *maven.Handler,
genericHandler *generic.Handler,
pypiHandler pypi.Handler,
) Handler {
r := chi.NewRouter()
r.Route("/{rootIdentifier}/{registryIdentifier}", func(r chi.Router) {
r.Use(middleware.StoreOriginalURL)
r.Route("/maven/", func(r chi.Router) {
r.Use(middleware.CheckMavenAuthHeader())
r.Use(middlewareauthn.Attempt(packageHandler.GetAuthenticator()))
r.Use(middleware.CheckMavenAuth())
r.Use(middleware.TrackDownloadStatForMavenArtifact(mavenHandler))
r.Use(middleware.TrackBandwidthStatForMavenArtifacts(mavenHandler))
r.Get("/*", mavenHandler.GetArtifact)
r.Head("/*", mavenHandler.HeadArtifact)
r.Put("/*", mavenHandler.PutArtifact)
})
r.Route("/generic/", func(r chi.Router) {
r.Use(middlewareauthn.Attempt(packageHandler.GetAuthenticator()))
r.Use(middleware.TrackDownloadStatForGenericArtifact(genericHandler))
r.Use(middleware.TrackBandwidthStatForGenericArtifacts(genericHandler))
r.Get("/*", genericHandler.PullArtifact)
r.Put("/*", genericHandler.PushArtifact)
})
r.Route("/pypi/", func(r chi.Router) {
r.Use(middlewareauthn.Attempt(packageHandler.GetAuthenticator()))
r.With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsUpload)).
Post("/*", pypiHandler.UploadPackageFile)
r.With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)).
Get("/files/{id}/{version}/{filename}", pypiHandler.DownloadPackageFile)
r.With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)).
Get("/simple/{id}", pypiHandler.PackageMetadata)
})
})
return r
}

2
registry/app/api/router/registry_router.go
View File

@ -41,7 +41,7 @@ func (r *RegistryRouter) IsEligibleTraffic(req *http.Request) bool {
if req.URL.RawPath != "" {
urlPath = req.URL.RawPath
}
if utils.HasAnyPrefix(urlPath, []string{RegistryMount, "/v2/", "/registry/", "/maven/", "/generic/"}) ||
if utils.HasAnyPrefix(urlPath, []string{RegistryMount, "/v2/", "/registry/", "/maven/", "/generic/", "/pkg/"}) ||
(strings.HasPrefix(urlPath, APIMount+"/v1/spaces/") &&
utils.HasAnySuffix(urlPath, []string{"/artifacts", "/registries"})) {
return true

16
registry/app/api/router/router.go
View File

@ -16,6 +16,7 @@ package router
import (
"fmt"
"log"
"net/http"
"github.com/harness/gitness/app/api/middleware/address"
@ -25,6 +26,7 @@ import (
"github.com/harness/gitness/registry/app/api/router/harness"
"github.com/harness/gitness/registry/app/api/router/maven"
"github.com/harness/gitness/registry/app/api/router/oci"
"github.com/harness/gitness/registry/app/api/router/packages"
"github.com/go-chi/chi/v5"
"github.com/rs/zerolog/hlog"
@ -40,6 +42,7 @@ func GetAppRouter(
baseURL string,
mavenHandler maven.Handler,
genericHandler generic2.Handler,
packageHandler packages.Handler,
) AppRouter {
r := chi.NewRouter()
r.Use(hlog.URLHandler("http.url"))
@ -51,10 +54,23 @@ func GetAppRouter(
r.Group(func(r chi.Router) {
r.Handle(fmt.Sprintf("%s/*", baseURL), appHandler)
r.Handle("/v2/*", ociHandler)
// deprecated
r.Handle("/maven/*", mavenHandler)
// deprecated
r.Handle("/generic/*", genericHandler)
r.Mount("/pkg/", packageHandler)
r.Handle("/registry/swagger*", swagger.GetSwaggerHandler("/registry"))
})
// Walk through all routes and print them
if err := chi.Walk(r,
func(method string, route string, _ http.Handler, _ ...func(http.Handler) http.Handler) error {
log.Printf("%-7s %s", method, route)
return nil
}); err != nil {
log.Fatalf("Error walking router: %v", err)
}
return r
}

17
registry/app/api/router/wire.go
View File

@ -24,10 +24,13 @@ import (
"github.com/harness/gitness/registry/app/api/handler/generic"
"github.com/harness/gitness/registry/app/api/handler/maven"
hoci "github.com/harness/gitness/registry/app/api/handler/oci"
"github.com/harness/gitness/registry/app/api/handler/packages"
"github.com/harness/gitness/registry/app/api/handler/pypi"
generic2 "github.com/harness/gitness/registry/app/api/router/generic"
"github.com/harness/gitness/registry/app/api/router/harness"
mavenRouter "github.com/harness/gitness/registry/app/api/router/maven"
"github.com/harness/gitness/registry/app/api/router/oci"
packagerrouter "github.com/harness/gitness/registry/app/api/router/packages"
storagedriver "github.com/harness/gitness/registry/app/driver"
"github.com/harness/gitness/registry/app/pkg/filemanager"
"github.com/harness/gitness/registry/app/store"
@ -41,8 +44,9 @@ func AppRouterProvider(
appHandler harness.APIHandler,
mavenHandler mavenRouter.Handler,
genericHandler generic2.Handler,
handler packagerrouter.Handler,
) AppRouter {
return GetAppRouter(ocir, appHandler, config.APIURL, mavenHandler, genericHandler)
return GetAppRouter(ocir, appHandler, config.APIURL, mavenHandler, genericHandler, handler)
}
func APIHandlerProvider(
@ -96,5 +100,14 @@ func GenericHandlerProvider(handler *generic.Handler) generic2.Handler {
return generic2.NewGenericArtifactHandler(handler)
}
func PackageHandlerProvider(
handler packages.Handler,
mavenHandler *maven.Handler,
genericHandler *generic.Handler,
pypiHandler pypi.Handler,
) packagerrouter.Handler {
return packagerrouter.NewRouter(handler, mavenHandler, genericHandler, pypiHandler)
}
var WireSet = wire.NewSet(APIHandlerProvider, OCIHandlerProvider, AppRouterProvider,
MavenHandlerProvider, GenericHandlerProvider)
MavenHandlerProvider, GenericHandlerProvider, PackageHandlerProvider)

30
registry/app/api/wire.go
View File

@ -24,6 +24,8 @@ import (
"github.com/harness/gitness/registry/app/api/handler/generic"
mavenhandler "github.com/harness/gitness/registry/app/api/handler/maven"
ocihandler "github.com/harness/gitness/registry/app/api/handler/oci"
"github.com/harness/gitness/registry/app/api/handler/packages"
pypi2 "github.com/harness/gitness/registry/app/api/handler/pypi"
"github.com/harness/gitness/registry/app/api/router"
storagedriver "github.com/harness/gitness/registry/app/driver"
"github.com/harness/gitness/registry/app/driver/factory"
@ -34,6 +36,8 @@ import (
"github.com/harness/gitness/registry/app/pkg/filemanager"
generic2 "github.com/harness/gitness/registry/app/pkg/generic"
"github.com/harness/gitness/registry/app/pkg/maven"
"github.com/harness/gitness/registry/app/pkg/pypi"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/registry/app/store/database"
"github.com/harness/gitness/registry/config"
"github.com/harness/gitness/registry/gc"
@ -104,6 +108,29 @@ func NewMavenHandlerProvider(
)
}
func NewPackageHandlerProvider(
registryDao store.RegistryRepository, spaceStore corestore.SpaceStore, tokenStore corestore.TokenStore,
userCtrl *usercontroller.Controller, authenticator authn.Authenticator,
urlProvider urlprovider.Provider, authorizer authz.Authorizer,
) packages.Handler {
return packages.NewHandler(
registryDao,
spaceStore,
tokenStore,
userCtrl,
authenticator,
urlProvider,
authorizer,
)
}
func NewPypiHandlerProvider(
controller pypi.Controller,
packageHandler packages.Handler,
) pypi2.Handler {
return pypi2.NewHandler(controller, packageHandler)
}
func NewGenericHandlerProvider(
spaceStore corestore.SpaceStore, controller *generic2.Controller, tokenStore corestore.TokenStore,
userCtrl *usercontroller.Controller, authenticator authn.Authenticator, urlProvider urlprovider.Provider,
@ -125,11 +152,14 @@ var WireSet = wire.NewSet(
NewHandlerProvider,
NewMavenHandlerProvider,
NewGenericHandlerProvider,
NewPackageHandlerProvider,
NewPypiHandlerProvider,
database.WireSet,
pkg.WireSet,
docker.WireSet,
filemanager.WireSet,
maven.WireSet,
pypi.WireSet,
router.WireSet,
gc.WireSet,
generic2.WireSet,

45
registry/app/pkg/generic/controller.go
View File

@ -38,8 +38,8 @@ import (
)
type Controller struct {
spaceStore corestore.SpaceStore
authorizer authz.Authorizer
SpaceStore corestore.SpaceStore
Authorizer authz.Authorizer
DBStore *DBStore
fileManager filemanager.FileManager
tx dbtx.Transactor
@ -62,8 +62,8 @@ func NewController(
tx dbtx.Transactor,
) *Controller {
return &Controller{
spaceStore: spaceStore,
authorizer: authorizer,
SpaceStore: spaceStore,
Authorizer: authorizer,
fileManager: fileManager,
DBStore: dBStore,
tx: tx,
@ -88,14 +88,16 @@ func NewDBStore(
const regNameFormat = "registry : [%s]"
func (c Controller) UploadArtifact(ctx context.Context, info pkg.GenericArtifactInfo,
file multipart.File) (*commons.ResponseHeaders, string, errcode.Error) {
func (c Controller) UploadArtifact(
ctx context.Context, info pkg.GenericArtifactInfo,
file multipart.File,
) (*commons.ResponseHeaders, string, errcode.Error) {
responseHeaders := &commons.ResponseHeaders{
Headers: make(map[string]string),
Code: 0,
}
err := pkg.GetRegistryCheckAccess(
ctx, c.DBStore.RegistryDao, c.authorizer, c.spaceStore, info.RegIdentifier, info.ParentID,
ctx, c.DBStore.RegistryDao, c.Authorizer, c.SpaceStore, info.RegIdentifier, info.ParentID,
enum.PermissionArtifactsUpload,
)
if err != nil {
@ -169,13 +171,16 @@ func (c Controller) UploadArtifact(ctx context.Context, info pkg.GenericArtifact
return responseHeaders, fileInfo.Sha256, errcode.Error{}
}
func (c Controller) updateMetadata(dbArtifact *types.Artifact, metadata *database.GenericMetadata,
info pkg.GenericArtifactInfo, fileInfo pkg.FileInfo) error {
func (c Controller) updateMetadata(
dbArtifact *types.Artifact, metadata *database.GenericMetadata,
info pkg.GenericArtifactInfo, fileInfo pkg.FileInfo,
) error {
var files []database.File
if dbArtifact != nil {
err := json.Unmarshal(dbArtifact.Metadata, metadata)
if err != nil {
return fmt.Errorf("failed to get metadata for artifact : [%s] with registry : [%s]", info.Image, info.RegIdentifier)
return fmt.Errorf("failed to get metadata for artifact : [%s] with registry : [%s]", info.Image,
info.RegIdentifier)
}
fileExist := false
files = metadata.Files
@ -185,28 +190,34 @@ func (c Controller) updateMetadata(dbArtifact *types.Artifact, metadata *databas
}
}
if !fileExist {
files = append(files, database.File{Size: fileInfo.Size, Filename: fileInfo.Filename,
CreatedAt: time.Now().UnixMilli()})
files = append(files, database.File{
Size: fileInfo.Size, Filename: fileInfo.Filename,
CreatedAt: time.Now().UnixMilli(),
})
metadata.Files = files
metadata.FileCount++
}
} else {
files = append(files, database.File{Size: fileInfo.Size, Filename: fileInfo.Filename,
CreatedAt: time.Now().UnixMilli()})
files = append(files, database.File{
Size: fileInfo.Size, Filename: fileInfo.Filename,
CreatedAt: time.Now().UnixMilli(),
})
metadata.Files = files
metadata.FileCount++
}
return nil
}
func (c Controller) PullArtifact(ctx context.Context, info pkg.GenericArtifactInfo) (*commons.ResponseHeaders,
*storage.FileReader, string, errcode.Error) {
func (c Controller) PullArtifact(ctx context.Context, info pkg.GenericArtifactInfo) (
*commons.ResponseHeaders,
*storage.FileReader, string, errcode.Error,
) {
responseHeaders := &commons.ResponseHeaders{
Headers: make(map[string]string),
Code: 0,
}
err := pkg.GetRegistryCheckAccess(
ctx, c.DBStore.RegistryDao, c.authorizer, c.spaceStore, info.RegIdentifier, info.ParentID,
ctx, c.DBStore.RegistryDao, c.Authorizer, c.SpaceStore, info.RegIdentifier, info.ParentID,
enum.PermissionArtifactsDownload,
)
if err != nil {

50
registry/app/pkg/pypi/controller.go Normal file
View File

@ -0,0 +1,50 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pypi
import (
"context"
"io"
"github.com/harness/gitness/registry/app/store"
)
// Controller handles PyPI package operations.
type controller struct {
artifactStore store.ArtifactRepository
proxyStore store.UpstreamProxyConfigRepository
_ FileManager
}
type Controller interface {
}
// FileManager interface for managing PyPI package files.
type FileManager interface {
Upload(ctx context.Context, registryID int64, path string, content io.Reader) error
Download(ctx context.Context, registryID int64, path string) (io.ReadCloser, error)
Delete(ctx context.Context, registryID int64, path string) error
}
// NewController creates a new PyPI controller.
func NewController(
artifactStore store.ArtifactRepository,
proxyStore store.UpstreamProxyConfigRepository,
) Controller {
return &controller{
artifactStore: artifactStore,
proxyStore: proxyStore,
}
}

42
registry/app/pkg/pypi/types.go Normal file
View File

@ -0,0 +1,42 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pypi
// Metadata represents the metadata for a PyPI package.
type Metadata struct {
Name string `json:"name"`
Version string `json:"version"`
Summary string `json:"summary"`
Description string `json:"description"`
Author string `json:"author"`
AuthorEmail string `json:"author_email,omitempty"`
License string `json:"license"`
Keywords []string `json:"keywords,omitempty"`
Platform string `json:"platform,omitempty"`
RequiresPython string `json:"requires_python,omitempty"`
Dependencies []string `json:"dependencies,omitempty"`
}
// PackageFile represents a PyPI package file.
type PackageFile struct {
Filename string `json:"filename"`
ContentType string `json:"content_type"`
Size int64 `json:"size"`
MD5 string `json:"md5_digest"`
SHA256 string `json:"sha256_digest"`
PackageType string `json:"package_type"` // e.g., "sdist", "bdist_wheel"
PythonVersion string `json:"python_version"`
UploadTime int64 `json:"upload_time_ms"`
}

31
registry/app/pkg/pypi/wire.go Normal file
View File

@ -0,0 +1,31 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pypi
import (
"github.com/harness/gitness/registry/app/store"
"github.com/google/wire"
)
func ControllerProvider(
artifactStore store.ArtifactRepository,
proxyStore store.UpstreamProxyConfigRepository,
) Controller {
return NewController(artifactStore, proxyStore)
}
var ControllerSet = wire.NewSet(ControllerProvider)
var WireSet = wire.NewSet(ControllerSet)