feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID (#3443)

* feat: [AH-925]: fix lint errors
* feat: [AH-925]: keep original context and also async flow. also keep the linter
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID - fix lint errors by ignoring problematic contextcheck
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID - lint errors
* feat: [AH-925]: rebase main
* feat: [AH-925]: rebase main
* feat: [AH-925]: rebase main
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID - fix lint
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID
* feat: [AH-925]: rebase main
jobatzil/login/xforwardedfor
Shivakumar Ningappa 2025-02-19 21:03:21 +00:00 committed by Harness
parent bab50ed39e
commit 5f7808a2da
22 changed files with 164 additions and 133 deletions

View File

@ -449,19 +449,19 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
layerRepository := database2.ProvideLayerDao(db, mediaTypesRepository) layerRepository := database2.ProvideLayerDao(db, mediaTypesRepository)
eventReporter := docker.ProvideReporter() eventReporter := docker.ProvideReporter()
ociImageIndexMappingRepository := database2.ProvideOCIImageIndexMappingDao(db) ociImageIndexMappingRepository := database2.ProvideOCIImageIndexMappingDao(db)
manifestService := docker.ManifestServiceProvider(registryRepository, manifestRepository, blobRepository, mediaTypesRepository, manifestReferenceRepository, tagRepository, imageRepository, artifactRepository, layerRepository, gcService, transactor, eventReporter, spacePathStore, ociImageIndexMappingRepository) manifestService := docker.ManifestServiceProvider(registryRepository, manifestRepository, blobRepository, mediaTypesRepository, manifestReferenceRepository, tagRepository, imageRepository, artifactRepository, layerRepository, gcService, transactor, eventReporter, spaceFinder, ociImageIndexMappingRepository)
registryBlobRepository := database2.ProvideRegistryBlobDao(db) registryBlobRepository := database2.ProvideRegistryBlobDao(db)
bandwidthStatRepository := database2.ProvideBandwidthStatDao(db) bandwidthStatRepository := database2.ProvideBandwidthStatDao(db)
downloadStatRepository := database2.ProvideDownloadStatDao(db) downloadStatRepository := database2.ProvideDownloadStatDao(db)
localRegistry := docker.LocalRegistryProvider(app, manifestService, blobRepository, registryRepository, manifestRepository, registryBlobRepository, mediaTypesRepository, tagRepository, imageRepository, artifactRepository, bandwidthStatRepository, downloadStatRepository, gcService, transactor) localRegistry := docker.LocalRegistryProvider(app, manifestService, blobRepository, registryRepository, manifestRepository, registryBlobRepository, mediaTypesRepository, tagRepository, imageRepository, artifactRepository, bandwidthStatRepository, downloadStatRepository, gcService, transactor)
upstreamProxyConfigRepository := database2.ProvideUpstreamDao(db, registryRepository, spacePathStore) upstreamProxyConfigRepository := database2.ProvideUpstreamDao(db, registryRepository, spaceFinder)
secretService := secret3.ProvideSecretService(secretStore, encrypter, spacePathStore) secretService := secret3.ProvideSecretService(secretStore, encrypter, spacePathStore)
proxyController := docker.ProvideProxyController(localRegistry, manifestService, secretService, spacePathStore) proxyController := docker.ProvideProxyController(localRegistry, manifestService, secretService, spaceFinder)
remoteRegistry := docker.RemoteRegistryProvider(localRegistry, app, upstreamProxyConfigRepository, spacePathStore, secretService, proxyController) remoteRegistry := docker.RemoteRegistryProvider(localRegistry, app, upstreamProxyConfigRepository, spaceFinder, secretService, proxyController)
coreController := pkg.CoreControllerProvider(registryRepository) coreController := pkg.CoreControllerProvider(registryRepository)
dbStore := docker.DBStoreProvider(blobRepository, imageRepository, artifactRepository, bandwidthStatRepository, downloadStatRepository) dbStore := docker.DBStoreProvider(blobRepository, imageRepository, artifactRepository, bandwidthStatRepository, downloadStatRepository)
dockerController := docker.ControllerProvider(localRegistry, remoteRegistry, coreController, spaceStore, authorizer, dbStore) dockerController := docker.ControllerProvider(localRegistry, remoteRegistry, coreController, spaceStore, authorizer, dbStore)
handler := api2.NewHandlerProvider(dockerController, spaceStore, tokenStore, controller, authenticator, provider, authorizer, config) handler := api2.NewHandlerProvider(dockerController, spaceFinder, spaceStore, tokenStore, controller, authenticator, provider, authorizer, config)
registryOCIHandler := router.OCIHandlerProvider(handler) registryOCIHandler := router.OCIHandlerProvider(handler)
filemanagerApp := filemanager.NewApp(ctx, config, storageService) filemanagerApp := filemanager.NewApp(ctx, config, storageService)
genericBlobRepository := database2.ProvideGenericBlobDao(db) genericBlobRepository := database2.ProvideGenericBlobDao(db)
@ -472,7 +472,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
apiHandler := router.APIHandlerProvider(registryRepository, upstreamProxyConfigRepository, fileManager, tagRepository, manifestRepository, cleanupPolicyRepository, imageRepository, storageDriver, spaceFinder, transactor, authenticator, provider, authorizer, auditService, spacePathStore, artifactRepository, webhooksRepository) apiHandler := router.APIHandlerProvider(registryRepository, upstreamProxyConfigRepository, fileManager, tagRepository, manifestRepository, cleanupPolicyRepository, imageRepository, storageDriver, spaceFinder, transactor, authenticator, provider, authorizer, auditService, spacePathStore, artifactRepository, webhooksRepository)
mavenDBStore := maven.DBStoreProvider(registryRepository, imageRepository, artifactRepository, spaceStore, bandwidthStatRepository, downloadStatRepository, nodesRepository, upstreamProxyConfigRepository) mavenDBStore := maven.DBStoreProvider(registryRepository, imageRepository, artifactRepository, spaceStore, bandwidthStatRepository, downloadStatRepository, nodesRepository, upstreamProxyConfigRepository)
mavenLocalRegistry := maven.LocalRegistryProvider(mavenDBStore, transactor, fileManager) mavenLocalRegistry := maven.LocalRegistryProvider(mavenDBStore, transactor, fileManager)
mavenController := maven.ProvideProxyController(mavenLocalRegistry, secretService, spacePathStore) mavenController := maven.ProvideProxyController(mavenLocalRegistry, secretService, spaceFinder)
mavenRemoteRegistry := maven.RemoteRegistryProvider(mavenDBStore, transactor, mavenLocalRegistry, mavenController) mavenRemoteRegistry := maven.RemoteRegistryProvider(mavenDBStore, transactor, mavenLocalRegistry, mavenController)
controller2 := maven.ControllerProvider(mavenLocalRegistry, mavenRemoteRegistry, authorizer, mavenDBStore) controller2 := maven.ControllerProvider(mavenLocalRegistry, mavenRemoteRegistry, authorizer, mavenDBStore)
mavenHandler := api2.NewMavenHandlerProvider(controller2, spaceStore, tokenStore, controller, authenticator, authorizer) mavenHandler := api2.NewMavenHandlerProvider(controller2, spaceStore, tokenStore, controller, authenticator, authorizer)

View File

@ -445,11 +445,11 @@ func (c *APIController) mapToWebhookResponseEntity(
} }
secretSpacePath := "" secretSpacePath := ""
if createdWebhook.SecretSpaceID > 0 { if createdWebhook.SecretSpaceID > 0 {
primary, err := c.spacePathStore.FindPrimaryBySpaceID(ctx, int64(createdWebhook.SecretSpaceID)) primary, err := c.SpaceFinder.FindByID(ctx, int64(createdWebhook.SecretSpaceID))
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get secret space path: %w", err) return nil, fmt.Errorf("failed to get secret space path: %w", err)
} }
secretSpacePath = primary.Value secretSpacePath = primary.Path
} }
if createdWebhook.SecretIdentifier != "" { if createdWebhook.SecretIdentifier != "" {
webhookResponseEntity.SecretIdentifier = &createdWebhook.SecretIdentifier webhookResponseEntity.SecretIdentifier = &createdWebhook.SecretIdentifier

View File

@ -22,6 +22,7 @@ import (
usercontroller "github.com/harness/gitness/app/api/controller/user" usercontroller "github.com/harness/gitness/app/api/controller/user"
"github.com/harness/gitness/app/auth/authn" "github.com/harness/gitness/app/auth/authn"
"github.com/harness/gitness/app/auth/authz" "github.com/harness/gitness/app/auth/authz"
"github.com/harness/gitness/app/services/refcache"
corestore "github.com/harness/gitness/app/store" corestore "github.com/harness/gitness/app/store"
urlprovider "github.com/harness/gitness/app/url" urlprovider "github.com/harness/gitness/app/url"
"github.com/harness/gitness/registry/app/api/controller/metadata" "github.com/harness/gitness/registry/app/api/controller/metadata"
@ -41,12 +42,14 @@ import (
) )
func NewHandler( func NewHandler(
controller *docker.Controller, spaceStore corestore.SpaceStore, tokenStore corestore.TokenStore, controller *docker.Controller, spaceFinder refcache.SpaceFinder, spaceStore corestore.SpaceStore,
tokenStore corestore.TokenStore,
userCtrl *usercontroller.Controller, authenticator authn.Authenticator, urlProvider urlprovider.Provider, userCtrl *usercontroller.Controller, authenticator authn.Authenticator, urlProvider urlprovider.Provider,
authorizer authz.Authorizer, ociRelativeURL bool, authorizer authz.Authorizer, ociRelativeURL bool,
) *Handler { ) *Handler {
return &Handler{ return &Handler{
Controller: controller, Controller: controller,
SpaceFinder: spaceFinder,
SpaceStore: spaceStore, SpaceStore: spaceStore,
TokenStore: tokenStore, TokenStore: tokenStore,
UserCtrl: userCtrl, UserCtrl: userCtrl,
@ -59,6 +62,7 @@ func NewHandler(
type Handler struct { type Handler struct {
Controller *docker.Controller Controller *docker.Controller
SpaceFinder refcache.SpaceFinder
SpaceStore corestore.SpaceStore SpaceStore corestore.SpaceStore
TokenStore corestore.TokenStore TokenStore corestore.TokenStore
UserCtrl *usercontroller.Controller UserCtrl *usercontroller.Controller

View File

@ -96,14 +96,14 @@ func (h *Handler) GetToken(w http.ResponseWriter, r *http.Request) {
} }
} }
func (h *Handler) getSpace(ctx context.Context, name string) (*types.Space, error) { func (h *Handler) getSpace(ctx context.Context, name string) (*types.SpaceCore, error) {
spaceRef, _, _ := paths.DisectRoot(name) spaceRef, _, _ := paths.DisectRoot(name)
space, err := h.SpaceStore.FindByRef(ctx, spaceRef) space, err := h.SpaceFinder.FindByRef(ctx, spaceRef)
return space, err return space, err
} }
func (h *Handler) getAccessPermissionList( func (h *Handler) getAccessPermissionList(
ctx context.Context, space *types.Space, ra *ResourceActions, session *auth.Session, ctx context.Context, space *types.SpaceCore, ra *ResourceActions, session *auth.Session,
accessPermissionsList []jwt.AccessPermissions, accessPermissionsList []jwt.AccessPermissions,
) []jwt.AccessPermissions { ) []jwt.AccessPermissions {
accessPermissions := &jwt.AccessPermissions{SpaceID: space.ID, Permissions: []enum.Permission{}} accessPermissions := &jwt.AccessPermissions{SpaceID: space.ID, Permissions: []enum.Permission{}}
@ -118,7 +118,7 @@ func (h *Handler) getAccessPermissionList(
ctx, ctx,
h.Authorizer, h.Authorizer,
session, session,
space.Core(), space,
enum.ResourceTypeRegistry, enum.ResourceTypeRegistry,
permission, permission,
) )

View File

@ -18,6 +18,7 @@ import (
usercontroller "github.com/harness/gitness/app/api/controller/user" usercontroller "github.com/harness/gitness/app/api/controller/user"
"github.com/harness/gitness/app/auth/authn" "github.com/harness/gitness/app/auth/authn"
"github.com/harness/gitness/app/auth/authz" "github.com/harness/gitness/app/auth/authz"
"github.com/harness/gitness/app/services/refcache"
corestore "github.com/harness/gitness/app/store" corestore "github.com/harness/gitness/app/store"
urlprovider "github.com/harness/gitness/app/url" urlprovider "github.com/harness/gitness/app/url"
"github.com/harness/gitness/registry/app/api/handler/generic" "github.com/harness/gitness/registry/app/api/handler/generic"
@ -71,12 +72,13 @@ func BlobStorageProvider(c *types.Config) (storagedriver.StorageDriver, error) {
} }
func NewHandlerProvider( func NewHandlerProvider(
controller *docker.Controller, spaceStore corestore.SpaceStore, controller *docker.Controller, spaceFinder refcache.SpaceFinder, spaceStore corestore.SpaceStore,
tokenStore corestore.TokenStore, userCtrl *usercontroller.Controller, authenticator authn.Authenticator, tokenStore corestore.TokenStore, userCtrl *usercontroller.Controller, authenticator authn.Authenticator,
urlProvider urlprovider.Provider, authorizer authz.Authorizer, config *types.Config, urlProvider urlprovider.Provider, authorizer authz.Authorizer, config *types.Config,
) *ocihandler.Handler { ) *ocihandler.Handler {
return ocihandler.NewHandler( return ocihandler.NewHandler(
controller, controller,
spaceFinder,
spaceStore, spaceStore,
tokenStore, tokenStore,
userCtrl, userCtrl,

View File

@ -23,7 +23,7 @@ import (
"fmt" "fmt"
"time" "time"
gas "github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/event" "github.com/harness/gitness/registry/app/event"
"github.com/harness/gitness/registry/app/manifest" "github.com/harness/gitness/registry/app/manifest"
@ -58,7 +58,7 @@ type manifestService struct {
artifactDao store.ArtifactRepository artifactDao store.ArtifactRepository
manifestRefDao store.ManifestReferenceRepository manifestRefDao store.ManifestReferenceRepository
ociImageIndexMappingDao store.OCIImageIndexMappingRepository ociImageIndexMappingDao store.OCIImageIndexMappingRepository
spacePathStore gas.SpacePathStore spaceFinder refcache.SpaceFinder
gcService gc.Service gcService gc.Service
tx dbtx.Transactor tx dbtx.Transactor
reporter event.Reporter reporter event.Reporter
@ -69,7 +69,7 @@ func NewManifestService(
blobRepo store.BlobRepository, mtRepository store.MediaTypesRepository, tagDao store.TagRepository, blobRepo store.BlobRepository, mtRepository store.MediaTypesRepository, tagDao store.TagRepository,
imageDao store.ImageRepository, artifactDao store.ArtifactRepository, imageDao store.ImageRepository, artifactDao store.ArtifactRepository,
layerDao store.LayerRepository, manifestRefDao store.ManifestReferenceRepository, layerDao store.LayerRepository, manifestRefDao store.ManifestReferenceRepository,
tx dbtx.Transactor, gcService gc.Service, reporter event.Reporter, spacePathStore gas.SpacePathStore, tx dbtx.Transactor, gcService gc.Service, reporter event.Reporter, spaceFinder refcache.SpaceFinder,
ociImageIndexMappingDao store.OCIImageIndexMappingRepository, ociImageIndexMappingDao store.OCIImageIndexMappingRepository,
) ManifestService { ) ManifestService {
return &manifestService{ return &manifestService{
@ -85,7 +85,7 @@ func NewManifestService(
gcService: gcService, gcService: gcService,
tx: tx, tx: tx,
reporter: reporter, reporter: reporter,
spacePathStore: spacePathStore, spaceFinder: spaceFinder,
ociImageIndexMappingDao: ociImageIndexMappingDao, ociImageIndexMappingDao: ociImageIndexMappingDao,
} }
} }
@ -279,7 +279,7 @@ func (l *manifestService) getSpacePathAndPackageType(
ctx context.Context, ctx context.Context,
dbRepo *types.Registry, dbRepo *types.Registry,
) (string, event.PackageType, error) { ) (string, event.PackageType, error) {
spacePath, err := l.spacePathStore.FindPrimaryBySpaceID(ctx, dbRepo.ParentID) spacePath, err := l.spaceFinder.FindByID(ctx, dbRepo.ParentID)
if err != nil { if err != nil {
log.Ctx(ctx).Err(err).Msg("Failed to find spacePath") log.Ctx(ctx).Err(err).Msg("Failed to find spacePath")
return "", event.PackageType(0), err return "", event.PackageType(0), err
@ -291,7 +291,7 @@ func (l *manifestService) getSpacePathAndPackageType(
return "", event.PackageType(0), err return "", event.PackageType(0), err
} }
return spacePath.Value, packageType, nil return spacePath.Path, packageType, nil
} }
// Reports event asynchronously. // Reports event asynchronously.

View File

@ -24,7 +24,7 @@ import (
"time" "time"
"github.com/harness/gitness/app/api/request" "github.com/harness/gitness/app/api/request"
store2 "github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/common/lib/errors" "github.com/harness/gitness/registry/app/common/lib/errors"
"github.com/harness/gitness/registry/app/manifest" "github.com/harness/gitness/registry/app/manifest"
"github.com/harness/gitness/registry/app/manifest/manifestlist" "github.com/harness/gitness/registry/app/manifest/manifestlist"
@ -34,6 +34,7 @@ import (
proxy2 "github.com/harness/gitness/registry/app/remote/controller/proxy" proxy2 "github.com/harness/gitness/registry/app/remote/controller/proxy"
"github.com/harness/gitness/registry/app/storage" "github.com/harness/gitness/registry/app/storage"
"github.com/harness/gitness/registry/app/store" "github.com/harness/gitness/registry/app/store"
cfg "github.com/harness/gitness/registry/config"
"github.com/harness/gitness/secret" "github.com/harness/gitness/secret"
v1 "github.com/opencontainers/image-spec/specs-go/v1" v1 "github.com/opencontainers/image-spec/specs-go/v1"
@ -51,7 +52,7 @@ const (
func NewRemoteRegistry( func NewRemoteRegistry(
local *LocalRegistry, app *App, upstreamProxyConfigRepo store.UpstreamProxyConfigRepository, local *LocalRegistry, app *App, upstreamProxyConfigRepo store.UpstreamProxyConfigRepository,
spacePathStore store2.SpacePathStore, secretService secret.Service, proxyCtl proxy2.Controller, spaceFinder refcache.SpaceFinder, secretService secret.Service, proxyCtl proxy2.Controller,
) Registry { ) Registry {
cache := proxy2.GetManifestCache(local, local.ms) cache := proxy2.GetManifestCache(local, local.ms)
listCache := proxy2.GetManifestListCache(local) listCache := proxy2.GetManifestListCache(local)
@ -67,7 +68,7 @@ func NewRemoteRegistry(
local: local, local: local,
App: app, App: app,
upstreamProxyConfigRepo: upstreamProxyConfigRepo, upstreamProxyConfigRepo: upstreamProxyConfigRepo,
spacePathStore: spacePathStore, spaceFinder: spaceFinder,
secretService: secretService, secretService: secretService,
manifestCacheHandlerMap: registry, manifestCacheHandlerMap: registry,
proxyCtl: proxyCtl, proxyCtl: proxyCtl,
@ -82,7 +83,7 @@ type RemoteRegistry struct {
local *LocalRegistry local *LocalRegistry
App *App App *App
upstreamProxyConfigRepo store.UpstreamProxyConfigRepository upstreamProxyConfigRepo store.UpstreamProxyConfigRepository
spacePathStore store2.SpacePathStore spaceFinder refcache.SpaceFinder
secretService secret.Service secretService secret.Service
proxyCtl proxy2.Controller proxyCtl proxy2.Controller
manifestCacheHandlerMap map[string]proxy2.ManifestCacheHandler manifestCacheHandlerMap map[string]proxy2.ManifestCacheHandler
@ -116,7 +117,9 @@ func proxyManifestHead(
go func(art pkg.RegistryInfo) { go func(art pkg.RegistryInfo) {
// Write function to update local storage. // Write function to update local storage.
session, _ := request.AuthSessionFrom(ctx) session, _ := request.AuthSessionFrom(ctx)
ctx2 := request.WithAuthSession(context.Background(), session) ctx2 := request.WithAuthSession(ctx, session)
ctx2 = context.WithoutCancel(ctx2)
ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "EnsureTag")
tag := art.Tag tag := art.Tag
art.Tag = "" art.Tag = ""
art.Digest = desc.Digest.String() art.Digest = desc.Digest.String()
@ -125,16 +128,15 @@ func proxyManifestHead(
for i := 0; i < ensureTagMaxRetry; i++ { for i := 0; i < ensureTagMaxRetry; i++ {
time.Sleep(ensureTagInterval) time.Sleep(ensureTagInterval)
count++ count++
log.Ctx(ctx2).Info().Str("goRoutine", "EnsureTag").Msgf("Tag %s for image: %s, retry: %d", tag, log.Ctx(ctx2).Info().Msgf("Tag %s for image: %s, retry: %d", tag,
info.Image, info.Image,
count) count)
e := ctl.EnsureTag(ctx2, responseHeaders, art, acceptHeaders, ifNoneMatchHeader) e := ctl.EnsureTag(ctx2, responseHeaders, art, acceptHeaders, ifNoneMatchHeader)
if e != nil { if e != nil {
log.Ctx(ctx2).Warn().Str("goRoutine", log.Ctx(ctx2).Warn().Err(e).Msgf("Failed to update tag: %s for image: %s",
"EnsureTag").Err(e).Msgf("Failed to update tag: %s for image: %s",
tag, info.Image) tag, info.Image)
} else { } else {
log.Ctx(ctx2).Info().Str("goRoutine", "EnsureTag").Msgf("Tag updated: %s for image: %s", tag, log.Ctx(ctx2).Info().Msgf("Tag updated: %s for image: %s", tag,
info.Image) info.Image)
return return
} }
@ -175,7 +177,7 @@ func (r *RemoteRegistry) ManifestExist(
errs = append(errs, err) errs = append(errs, err)
return responseHeaders, descriptor, manifestResult, errs return responseHeaders, descriptor, manifestResult, errs
} }
remoteHelper, err := proxy2.NewRemoteHelper(ctx, r.spacePathStore, r.secretService, artInfo.RegIdentifier, remoteHelper, err := proxy2.NewRemoteHelper(ctx, r.spaceFinder, r.secretService, artInfo.RegIdentifier,
*upstreamProxy) *upstreamProxy)
if err != nil { if err != nil {
errs = append(errs, errors.New("Proxy is down")) errs = append(errs, errors.New("Proxy is down"))
@ -251,7 +253,7 @@ func (r *RemoteRegistry) PullManifest(
errs = append(errs, err) errs = append(errs, err)
return responseHeaders, descriptor, manifestResult, errs return responseHeaders, descriptor, manifestResult, errs
} }
remoteHelper, err := proxy2.NewRemoteHelper(ctx, r.spacePathStore, r.secretService, artInfo.RegIdentifier, remoteHelper, err := proxy2.NewRemoteHelper(ctx, r.spaceFinder, r.secretService, artInfo.RegIdentifier,
*upstreamProxy) *upstreamProxy)
if err != nil { if err != nil {
errs = append(errs, errors.New("Proxy is down")) errs = append(errs, errors.New("Proxy is down"))

View File

@ -16,6 +16,7 @@ package docker
import ( import (
"github.com/harness/gitness/app/auth/authz" "github.com/harness/gitness/app/auth/authz"
"github.com/harness/gitness/app/services/refcache"
gitnessstore "github.com/harness/gitness/app/store" gitnessstore "github.com/harness/gitness/app/store"
storagedriver "github.com/harness/gitness/registry/app/driver" storagedriver "github.com/harness/gitness/registry/app/driver"
"github.com/harness/gitness/registry/app/event" "github.com/harness/gitness/registry/app/event"
@ -43,6 +44,7 @@ func LocalRegistryProvider(
bandwidthStatDao store.BandwidthStatRepository, downloadStatDao store.DownloadStatRepository, bandwidthStatDao store.BandwidthStatRepository, downloadStatDao store.DownloadStatRepository,
gcService gc.Service, tx dbtx.Transactor, gcService gc.Service, tx dbtx.Transactor,
) *LocalRegistry { ) *LocalRegistry {
//nolint:errcheck
return NewLocalRegistry( return NewLocalRegistry(
app, ms, manifestDao, registryDao, registryBlobDao, blobRepo, app, ms, manifestDao, registryDao, registryBlobDao, blobRepo,
mtRepository, tagDao, imageDao, artifactDao, bandwidthStatDao, downloadStatDao, gcService, tx, mtRepository, tagDao, imageDao, artifactDao, bandwidthStatDao, downloadStatDao, gcService, tx,
@ -54,21 +56,22 @@ func ManifestServiceProvider(
manifestDao store.ManifestRepository, blobRepo store.BlobRepository, mtRepository store.MediaTypesRepository, manifestDao store.ManifestRepository, blobRepo store.BlobRepository, mtRepository store.MediaTypesRepository,
manifestRefDao store.ManifestReferenceRepository, tagDao store.TagRepository, imageDao store.ImageRepository, manifestRefDao store.ManifestReferenceRepository, tagDao store.TagRepository, imageDao store.ImageRepository,
artifactDao store.ArtifactRepository, layerDao store.LayerRepository, artifactDao store.ArtifactRepository, layerDao store.LayerRepository,
gcService gc.Service, tx dbtx.Transactor, reporter event.Reporter, spacePathStore gitnessstore.SpacePathStore, gcService gc.Service, tx dbtx.Transactor, reporter event.Reporter, spaceFinder refcache.SpaceFinder,
ociImageIndexMappingDao store.OCIImageIndexMappingRepository, ociImageIndexMappingDao store.OCIImageIndexMappingRepository,
) ManifestService { ) ManifestService {
return NewManifestService( return NewManifestService(
registryDao, manifestDao, blobRepo, mtRepository, tagDao, imageDao, registryDao, manifestDao, blobRepo, mtRepository, tagDao, imageDao,
artifactDao, layerDao, manifestRefDao, tx, gcService, reporter, spacePathStore, artifactDao, layerDao, manifestRefDao, tx, gcService, reporter, spaceFinder,
ociImageIndexMappingDao, ociImageIndexMappingDao,
) )
} }
func RemoteRegistryProvider( func RemoteRegistryProvider(
local *LocalRegistry, app *App, upstreamProxyConfigRepo store.UpstreamProxyConfigRepository, local *LocalRegistry, app *App, upstreamProxyConfigRepo store.UpstreamProxyConfigRepository,
spacePathStore gitnessstore.SpacePathStore, secretService secret.Service, proxyCtrl proxy2.Controller, spaceFinder refcache.SpaceFinder, secretService secret.Service, proxyCtrl proxy2.Controller,
) *RemoteRegistry { ) *RemoteRegistry {
return NewRemoteRegistry(local, app, upstreamProxyConfigRepo, spacePathStore, secretService, //nolint:errcheck
return NewRemoteRegistry(local, app, upstreamProxyConfigRepo, spaceFinder, secretService,
proxyCtrl).(*RemoteRegistry) proxyCtrl).(*RemoteRegistry)
} }
@ -103,10 +106,10 @@ func ProvideReporter() event.Reporter {
func ProvideProxyController( func ProvideProxyController(
registry *LocalRegistry, ms ManifestService, secretService secret.Service, registry *LocalRegistry, ms ManifestService, secretService secret.Service,
spacePathStore gitnessstore.SpacePathStore, spaceFinder refcache.SpaceFinder,
) proxy2.Controller { ) proxy2.Controller {
manifestCacheHandler := getManifestCacheHandler(registry, ms) manifestCacheHandler := getManifestCacheHandler(registry, ms)
return proxy2.NewProxyController(registry, ms, secretService, spacePathStore, manifestCacheHandler) return proxy2.NewProxyController(registry, ms, secretService, spaceFinder, manifestCacheHandler)
} }
func getManifestCacheHandler( func getManifestCacheHandler(

View File

@ -16,6 +16,7 @@ package maven
import ( import (
"github.com/harness/gitness/app/auth/authz" "github.com/harness/gitness/app/auth/authz"
"github.com/harness/gitness/app/services/refcache"
corestore "github.com/harness/gitness/app/store" corestore "github.com/harness/gitness/app/store"
"github.com/harness/gitness/registry/app/pkg/filemanager" "github.com/harness/gitness/registry/app/pkg/filemanager"
"github.com/harness/gitness/registry/app/remote/controller/proxy/maven" "github.com/harness/gitness/registry/app/remote/controller/proxy/maven"
@ -31,6 +32,7 @@ func LocalRegistryProvider(
tx dbtx.Transactor, tx dbtx.Transactor,
fileManager filemanager.FileManager, fileManager filemanager.FileManager,
) *LocalRegistry { ) *LocalRegistry {
//nolint:errcheck
return NewLocalRegistry(dBStore, return NewLocalRegistry(dBStore,
tx, tx,
fileManager, fileManager,
@ -43,6 +45,7 @@ func RemoteRegistryProvider(
local *LocalRegistry, local *LocalRegistry,
proxyController maven.Controller, proxyController maven.Controller,
) *RemoteRegistry { ) *RemoteRegistry {
//nolint:errcheck
return NewRemoteRegistry(dBStore, tx, local, proxyController).(*RemoteRegistry) return NewRemoteRegistry(dBStore, tx, local, proxyController).(*RemoteRegistry)
} }
@ -65,6 +68,7 @@ func DBStoreProvider(
nodeDao store.NodesRepository, nodeDao store.NodesRepository,
upstreamProxyDao store.UpstreamProxyConfigRepository, upstreamProxyDao store.UpstreamProxyConfigRepository,
) *DBStore { ) *DBStore {
//nolint:errcheck
return NewDBStore(registryDao, imageDao, artifactDao, spaceStore, bandwidthStatDao, return NewDBStore(registryDao, imageDao, artifactDao, spaceStore, bandwidthStatDao,
downloadStatDao, downloadStatDao,
nodeDao, nodeDao,
@ -73,9 +77,9 @@ func DBStoreProvider(
func ProvideProxyController( func ProvideProxyController(
registry *LocalRegistry, secretService secret.Service, registry *LocalRegistry, secretService secret.Service,
spacePathStore corestore.SpacePathStore, spaceFinder refcache.SpaceFinder,
) maven.Controller { ) maven.Controller {
return maven.NewProxyController(registry, secretService, spacePathStore) return maven.NewProxyController(registry, secretService, spaceFinder)
} }
var ControllerSet = wire.NewSet(ControllerProvider) var ControllerSet = wire.NewSet(ControllerProvider)

View File

@ -22,7 +22,7 @@ import (
"fmt" "fmt"
"io" "io"
store2 "github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/manifest" "github.com/harness/gitness/registry/app/manifest"
"github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/types" "github.com/harness/gitness/registry/types"
@ -40,7 +40,7 @@ var registryKeys = []string{}
// Factory creates a specific Adapter according to the params. // Factory creates a specific Adapter according to the params.
type Factory interface { type Factory interface {
Create( Create(
ctx context.Context, spacePathStore store2.SpacePathStore, record types.UpstreamProxy, service secret.Service, ctx context.Context, spaceFinder refcache.SpaceFinder, record types.UpstreamProxy, service secret.Service,
) (Adapter, error) ) (Adapter, error)
} }

View File

@ -20,7 +20,7 @@ import (
"context" "context"
"regexp" "regexp"
store2 "github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
adp "github.com/harness/gitness/registry/app/remote/adapter" adp "github.com/harness/gitness/registry/app/remote/adapter"
"github.com/harness/gitness/registry/app/remote/adapter/native" "github.com/harness/gitness/registry/app/remote/adapter/native"
@ -49,9 +49,9 @@ func init() {
} }
func newAdapter( func newAdapter(
ctx context.Context, spacePathStore store2.SpacePathStore, service secret.Service, registry types.UpstreamProxy, ctx context.Context, spaceFinder refcache.SpaceFinder, service secret.Service, registry types.UpstreamProxy,
) (adp.Adapter, error) { ) (adp.Adapter, error) {
accessKey, secretKey, isPublic, err := getCreds(ctx, spacePathStore, service, registry) accessKey, secretKey, isPublic, err := getCreds(ctx, spaceFinder, service, registry)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -73,9 +73,9 @@ func newAdapter(
// Create ... // Create ...
func (f *factory) Create( func (f *factory) Create(
ctx context.Context, spacePathStore store2.SpacePathStore, record types.UpstreamProxy, service secret.Service, ctx context.Context, spaceFinder refcache.SpaceFinder, record types.UpstreamProxy, service secret.Service,
) (adp.Adapter, error) { ) (adp.Adapter, error) {
return newAdapter(ctx, spacePathStore, service, record) return newAdapter(ctx, spaceFinder, service, record)
} }
type factory struct { type factory struct {

View File

@ -28,7 +28,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
commonhttp "github.com/harness/gitness/registry/app/common/http" commonhttp "github.com/harness/gitness/registry/app/common/http"
"github.com/harness/gitness/registry/app/common/http/modifier" "github.com/harness/gitness/registry/app/common/http/modifier"
@ -135,14 +135,14 @@ func getAwsSvc(accessKey, secretKey string, reg types.UpstreamProxy) (*awsecrapi
func parseAccountRegion(url string) (string, string, error) { func parseAccountRegion(url string) (string, string, error) {
rs := ecrRegexp.FindStringSubmatch(url) rs := ecrRegexp.FindStringSubmatch(url)
if rs == nil || len(rs) < 4 { if len(rs) < 4 {
return "", "", errors.New("bad aws url") return "", "", errors.New("bad aws url")
} }
return rs[1], rs[3], nil return rs[1], rs[3], nil
} }
func getCreds( func getCreds(
ctx context.Context, spacePathStore store.SpacePathStore, secretService secret.Service, reg types.UpstreamProxy, ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, reg types.UpstreamProxy,
) (string, string, bool, error) { ) (string, string, bool, error) {
if api.AuthType(reg.RepoAuthType) == api.AuthTypeAnonymous { if api.AuthType(reg.RepoAuthType) == api.AuthTypeAnonymous {
return "", "", true, nil return "", "", true, nil
@ -151,7 +151,7 @@ func getCreds(
log.Debug().Msgf("invalid auth type: %s", reg.RepoAuthType) log.Debug().Msgf("invalid auth type: %s", reg.RepoAuthType)
return "", "", false, nil return "", "", false, nil
} }
secretKey, err := getSecretValue(ctx, spacePathStore, secretService, reg.SecretSpaceID, secretKey, err := getSecretValue(ctx, spaceFinder, secretService, reg.SecretSpaceID,
reg.SecretIdentifier) reg.SecretIdentifier)
if err != nil { if err != nil {
return "", "", false, err return "", "", false, err
@ -159,7 +159,7 @@ func getCreds(
if reg.UserName != "" { if reg.UserName != "" {
return reg.UserName, secretKey, false, nil return reg.UserName, secretKey, false, nil
} }
accessKey, err := getSecretValue(ctx, spacePathStore, secretService, reg.UserNameSecretSpaceID, accessKey, err := getSecretValue(ctx, spaceFinder, secretService, reg.UserNameSecretSpaceID,
reg.UserNameSecretIdentifier) reg.UserNameSecretIdentifier)
if err != nil { if err != nil {
return "", "", false, err return "", "", false, err
@ -167,14 +167,14 @@ func getCreds(
return accessKey, secretKey, false, nil return accessKey, secretKey, false, nil
} }
func getSecretValue(ctx context.Context, spacePathStore store.SpacePathStore, secretService secret.Service, func getSecretValue(ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service,
secretSpaceID int64, secretSpacePath string) (string, error) { secretSpaceID int64, secretSpacePath string) (string, error) {
spacePath, err := spacePathStore.FindPrimaryBySpaceID(ctx, secretSpaceID) spacePath, err := spaceFinder.FindByID(ctx, secretSpaceID)
if err != nil { if err != nil {
log.Error().Msgf("failed to find space path: %v", err) log.Error().Msgf("failed to find space path: %v", err)
return "", err return "", err
} }
decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Value, secretSpacePath) decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Path, secretSpacePath)
if err != nil { if err != nil {
log.Error().Msgf("failed to decrypt secret: %v", err) log.Error().Msgf("failed to decrypt secret: %v", err)
return "", err return "", err

View File

@ -19,7 +19,7 @@ package dockerhub
import ( import (
"context" "context"
store2 "github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
adp "github.com/harness/gitness/registry/app/remote/adapter" adp "github.com/harness/gitness/registry/app/remote/adapter"
"github.com/harness/gitness/registry/app/remote/adapter/native" "github.com/harness/gitness/registry/app/remote/adapter/native"
@ -38,7 +38,7 @@ func init() {
} }
func newAdapter( func newAdapter(
ctx context.Context, spacePathStore store2.SpacePathStore, service secret.Service, registry types.UpstreamProxy, ctx context.Context, spaceFinder refcache.SpaceFinder, service secret.Service, registry types.UpstreamProxy,
) (adp.Adapter, error) { ) (adp.Adapter, error) {
client, err := NewClient(registry) client, err := NewClient(registry)
if err != nil { if err != nil {
@ -48,7 +48,7 @@ func newAdapter(
// TODO: get Upstream Credentials // TODO: get Upstream Credentials
return &adapter{ return &adapter{
client: client, client: client,
Adapter: native.NewAdapter(ctx, spacePathStore, service, registry), Adapter: native.NewAdapter(ctx, spaceFinder, service, registry),
}, nil }, nil
} }
@ -57,9 +57,9 @@ type factory struct {
// Create ... // Create ...
func (f *factory) Create( func (f *factory) Create(
ctx context.Context, spacePathStore store2.SpacePathStore, record types.UpstreamProxy, service secret.Service, ctx context.Context, spaceFinder refcache.SpaceFinder, record types.UpstreamProxy, service secret.Service,
) (adp.Adapter, error) { ) (adp.Adapter, error) {
return newAdapter(ctx, spacePathStore, service, record) return newAdapter(ctx, spaceFinder, service, record)
} }
var ( var (

View File

@ -20,7 +20,7 @@ import (
"context" "context"
"net/http" "net/http"
store2 "github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
commonhttp "github.com/harness/gitness/registry/app/common/http" commonhttp "github.com/harness/gitness/registry/app/common/http"
adp "github.com/harness/gitness/registry/app/remote/adapter" adp "github.com/harness/gitness/registry/app/remote/adapter"
@ -44,7 +44,7 @@ func init() {
} }
func newAdapter( func newAdapter(
ctx context.Context, spacePathStore store2.SpacePathStore, service secret.Service, registry types.UpstreamProxy, ctx context.Context, spaceFinder refcache.SpaceFinder, service secret.Service, registry types.UpstreamProxy,
) (adp.Adapter, error) { ) (adp.Adapter, error) {
client, err := NewClient(registry) client, err := NewClient(registry)
if err != nil { if err != nil {
@ -53,7 +53,7 @@ func newAdapter(
return &adapter{ return &adapter{
client: client, client: client,
Adapter: native.NewAdapter(ctx, spacePathStore, service, registry), Adapter: native.NewAdapter(ctx, spaceFinder, service, registry),
}, nil }, nil
} }
@ -62,9 +62,9 @@ type factory struct {
// Create ... // Create ...
func (f *factory) Create( func (f *factory) Create(
ctx context.Context, spacePathStore store2.SpacePathStore, record types.UpstreamProxy, service secret.Service, ctx context.Context, spaceFinder refcache.SpaceFinder, record types.UpstreamProxy, service secret.Service,
) (adp.Adapter, error) { ) (adp.Adapter, error) {
return newAdapter(ctx, spacePathStore, service, record) return newAdapter(ctx, spaceFinder, service, record)
} }
var ( var (

View File

@ -19,7 +19,7 @@ package native
import ( import (
"context" "context"
"github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/common/lib" "github.com/harness/gitness/registry/app/common/lib"
"github.com/harness/gitness/registry/app/common/lib/errors" "github.com/harness/gitness/registry/app/common/lib/errors"
@ -47,13 +47,13 @@ type Adapter struct {
// NewAdapter returns an instance of the Adapter. // NewAdapter returns an instance of the Adapter.
func NewAdapter( func NewAdapter(
ctx context.Context, spacePathStore store.SpacePathStore, service secret.Service, reg types.UpstreamProxy, ctx context.Context, spaceFinder refcache.SpaceFinder, service secret.Service, reg types.UpstreamProxy,
) *Adapter { ) *Adapter {
adapter := &Adapter{ adapter := &Adapter{
proxy: reg, proxy: reg,
} }
// Get the password: lookup secrets.secret_data using secret_identifier & secret_space_id. // Get the password: lookup secrets.secret_data using secret_identifier & secret_space_id.
password := getPwd(ctx, spacePathStore, service, reg) password := getPwd(ctx, spaceFinder, service, reg)
username, password, url := reg.UserName, password, reg.RepoURL username, password, url := reg.UserName, password, reg.RepoURL
adapter.Client = registry.NewClient(url, username, password, false) adapter.Client = registry.NewClient(url, username, password, false)
return adapter return adapter
@ -69,18 +69,18 @@ func NewAdapterWithAuthorizer(reg types.UpstreamProxy, authorizer lib.Authorizer
// getPwd: lookup secrets.secret_data using secret_identifier & secret_space_id. // getPwd: lookup secrets.secret_data using secret_identifier & secret_space_id.
func getPwd( func getPwd(
ctx context.Context, spacePathStore store.SpacePathStore, secretService secret.Service, reg types.UpstreamProxy, ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, reg types.UpstreamProxy,
) string { ) string {
if api.AuthType(reg.RepoAuthType) == api.AuthTypeUserPassword { if api.AuthType(reg.RepoAuthType) == api.AuthTypeUserPassword {
secretSpaceID := reg.SecretSpaceID secretSpaceID := reg.SecretSpaceID
secretIdentifier := reg.SecretIdentifier secretIdentifier := reg.SecretIdentifier
spacePath, err := spacePathStore.FindPrimaryBySpaceID(ctx, secretSpaceID) spacePath, err := spaceFinder.FindByID(ctx, secretSpaceID)
if err != nil { if err != nil {
log.Error().Msgf("failed to find space path: %v", err) log.Error().Msgf("failed to find space path: %v", err)
return "" return ""
} }
decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Value, secretIdentifier) decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Path, secretIdentifier)
if err != nil { if err != nil {
log.Error().Msgf("failed to decrypt secret: %v", err) log.Error().Msgf("failed to decrypt secret: %v", err)
return "" return ""

View File

@ -25,11 +25,12 @@ import (
"time" "time"
"github.com/harness/gitness/app/api/request" "github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/common/lib/errors" "github.com/harness/gitness/registry/app/common/lib/errors"
"github.com/harness/gitness/registry/app/manifest" "github.com/harness/gitness/registry/app/manifest"
"github.com/harness/gitness/registry/app/pkg" "github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/commons"
cfg "github.com/harness/gitness/registry/config"
"github.com/harness/gitness/registry/types" "github.com/harness/gitness/registry/types"
"github.com/harness/gitness/secret" "github.com/harness/gitness/secret"
@ -91,20 +92,20 @@ type controller struct {
localRegistry registryInterface localRegistry registryInterface
localManifestRegistry registryManifestInterface localManifestRegistry registryManifestInterface
secretService secret.Service secretService secret.Service
spacePathStore store.SpacePathStore spaceFinder refcache.SpaceFinder
manifestCacheHandlerMap map[string]ManifestCacheHandler manifestCacheHandlerMap map[string]ManifestCacheHandler
} }
// NewProxyController -- get the proxy controller instance. // NewProxyController -- get the proxy controller instance.
func NewProxyController( func NewProxyController(
l registryInterface, lm registryManifestInterface, secretService secret.Service, l registryInterface, lm registryManifestInterface, secretService secret.Service,
spacePathStore store.SpacePathStore, manifestCacheHandlerMap map[string]ManifestCacheHandler, spaceFinder refcache.SpaceFinder, manifestCacheHandlerMap map[string]ManifestCacheHandler,
) Controller { ) Controller {
return &controller{ return &controller{
localRegistry: l, localRegistry: l,
localManifestRegistry: lm, localManifestRegistry: lm,
secretService: secretService, secretService: secretService,
spacePathStore: spacePathStore, spaceFinder: spaceFinder,
manifestCacheHandlerMap: manifestCacheHandlerMap, manifestCacheHandlerMap: manifestCacheHandlerMap,
} }
} }
@ -229,18 +230,19 @@ func (c *controller) ProxyManifest(
// This GoRoutine is to push the manifest from Remote to Local registry. // This GoRoutine is to push the manifest from Remote to Local registry.
go func(_, ct string) { go func(_, ct string) {
session, _ := request.AuthSessionFrom(ctx) session, _ := request.AuthSessionFrom(ctx)
ctx2 := request.WithAuthSession(context.Background(), session) ctx2 := request.WithAuthSession(ctx, session)
ctx2 = context.WithoutCancel(ctx2)
ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "UpdateManifest")
var count = 0 var count = 0
for n := 0; n < maxManifestWait; n++ { for n := 0; n < maxManifestWait; n++ {
time.Sleep(sleepIntervalSec * time.Second) time.Sleep(sleepIntervalSec * time.Second)
count++ count++
log.Ctx(ctx2).Info().Str("goRoutine", "UpdateManifest").Msgf("Current retry=%v artifact: %v:%v, digest: %s", log.Ctx(ctx2).Info().Msgf("Current retry=%v artifact: %v:%v, digest: %s",
count, repoKey, imageName, count, repoKey, imageName,
art.Digest) art.Digest)
_, des, _, e := c.localRegistry.PullManifest(ctx2, art, acceptHeader, ifNoneMatchHeader) _, des, _, e := c.localRegistry.PullManifest(ctx2, art, acceptHeader, ifNoneMatchHeader)
if len(e) > 0 { if len(e) > 0 {
log.Ctx(ctx2).Info().Str("goRoutine", log.Ctx(ctx2).Info().Stack().Err(err).Msgf("Local manifest doesn't exist, error %v", e[0])
"UpdateManifest").Stack().Err(err).Msgf("Local manifest doesn't exist, error %v", e[0])
} }
// Push manifest to localRegistry when pull with digest, or artifact not found, or digest mismatch. // Push manifest to localRegistry when pull with digest, or artifact not found, or digest mismatch.
errs := []error{} errs := []error{}
@ -249,7 +251,6 @@ func (c *controller) ProxyManifest(
if len(artInfo.Digest) == 0 { if len(artInfo.Digest) == 0 {
artInfo.Digest = dig artInfo.Digest = dig
} }
err = c.waitAndPushManifest(ctx2, art, ct, man) err = c.waitAndPushManifest(ctx2, art, ct, man)
if err != nil { if err != nil {
continue continue
@ -260,10 +261,9 @@ func (c *controller) ProxyManifest(
if e == nil || commons.IsEmpty(errs) { if e == nil || commons.IsEmpty(errs) {
_, _, _, err := c.localRegistry.PullManifest(ctx2, art, acceptHeader, ifNoneMatchHeader) _, _, _, err := c.localRegistry.PullManifest(ctx2, art, acceptHeader, ifNoneMatchHeader)
if err != nil { if err != nil {
log.Ctx(ctx2).Error().Str("goRoutine", log.Ctx(ctx2).Error().Stack().Msgf("failed to get manifest, error %v", err)
"UpdateManifest").Stack().Msgf("failed to get manifest, error %v", err)
} else { } else {
log.Ctx(ctx2).Info().Str("goRoutine", "UpdateManifest").Msgf( log.Ctx(ctx2).Info().Msgf(
"Completed manifest push to localRegistry registry. Image: %s, Tag: %s, Digest: %s", "Completed manifest push to localRegistry registry. Image: %s, Tag: %s, Digest: %s",
art.Image, art.Tag, art.Digest, art.Image, art.Tag, art.Digest,
) )
@ -296,7 +296,7 @@ func (c *controller) ProxyBlob(
remoteImage := getRemoteRepo(art) remoteImage := getRemoteRepo(art)
log.Debug().Msgf("The blob doesn't exist, proxy the request to the target server, url:%v", remoteImage) log.Debug().Msgf("The blob doesn't exist, proxy the request to the target server, url:%v", remoteImage)
rHelper, err := NewRemoteHelper(ctx, c.spacePathStore, c.secretService, repoKey, proxy) rHelper, err := NewRemoteHelper(ctx, c.spaceFinder, c.secretService, repoKey, proxy)
if err != nil { if err != nil {
return 0, nil, err return 0, nil, err
} }
@ -316,18 +316,17 @@ func (c *controller) ProxyBlob(
log.Error().Stack().Err(err).Msg("failed to get auth session from context") log.Error().Stack().Err(err).Msg("failed to get auth session from context")
return return
} }
ctx2 := request.WithAuthSession(context.Background(), session) ctx2 := request.WithAuthSession(ctx, session)
ctx2 = log.Ctx(ctx2).With(). ctx2 = context.WithoutCancel(ctx2)
Str("goRoutine", "AddBlob"). ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "AddBlob")
Logger().WithContext(ctx2) ctx2 = log.Ctx(ctx2).With().Logger().WithContext(ctx2)
err := c.putBlobToLocal(ctx2, art, remoteImage, repoKey, desc, rHelper) err := c.putBlobToLocal(ctx2, art, remoteImage, repoKey, desc, rHelper)
if err != nil { if err != nil {
log.Ctx(ctx2).Error().Str("goRoutine", log.Ctx(ctx2).Error().Stack().Err(err).
"AddBlob").Stack().Err(err).Msgf("error while putting blob to localRegistry registry, %v", err) Msgf("error while putting blob to localRegistry registry, %v", err)
return return
} }
log.Ctx(ctx2).Info().Str("goRoutine", "AddBlob").Msgf("Successfully updated the cache for digest %s", log.Ctx(ctx2).Info().Msgf("Successfully updated the cache for digest %s", art.Digest)
art.Digest)
}(art) }(art)
return size, bReader, nil return size, bReader, nil
} }

View File

@ -20,11 +20,12 @@ import (
"strings" "strings"
"github.com/harness/gitness/app/api/request" "github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/pkg" "github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/maven/utils" "github.com/harness/gitness/registry/app/pkg/maven/utils"
"github.com/harness/gitness/registry/app/storage" "github.com/harness/gitness/registry/app/storage"
cfg "github.com/harness/gitness/registry/config"
"github.com/harness/gitness/registry/types" "github.com/harness/gitness/registry/types"
"github.com/harness/gitness/secret" "github.com/harness/gitness/secret"
@ -32,9 +33,9 @@ import (
) )
type controller struct { type controller struct {
localRegistry registryInterface localRegistry registryInterface
secretService secret.Service secretService secret.Service
spacePathStore store.SpacePathStore spaceFinder refcache.SpaceFinder
} }
type Controller interface { type Controller interface {
@ -49,12 +50,12 @@ type Controller interface {
// NewProxyController -- get the proxy controller instance. // NewProxyController -- get the proxy controller instance.
func NewProxyController( func NewProxyController(
l registryInterface, secretService secret.Service, l registryInterface, secretService secret.Service,
spacePathStore store.SpacePathStore, spaceFinder refcache.SpaceFinder,
) Controller { ) Controller {
return &controller{ return &controller{
localRegistry: l, localRegistry: l,
secretService: secretService, secretService: secretService,
spacePathStore: spacePathStore, spaceFinder: spaceFinder,
} }
} }
@ -70,7 +71,7 @@ func (c *controller) ProxyFile(
responseHeaders = &commons.ResponseHeaders{ responseHeaders = &commons.ResponseHeaders{
Headers: make(map[string]string), Headers: make(map[string]string),
} }
rHelper, err := NewRemoteHelper(ctx, c.spacePathStore, c.secretService, proxy) rHelper, err := NewRemoteHelper(ctx, c.spaceFinder, c.secretService, proxy)
if err != nil { if err != nil {
return responseHeaders, nil, err return responseHeaders, nil, err
} }
@ -100,15 +101,15 @@ func (c *controller) ProxyFile(
log.Error().Stack().Err(err).Msg("failed to get auth session from context") log.Error().Stack().Err(err).Msg("failed to get auth session from context")
return return
} }
ctx2 := request.WithAuthSession(context.Background(), session) ctx2 := request.WithAuthSession(ctx, session)
ctx2 = context.WithoutCancel(ctx2)
ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "goRoutine")
err = c.putFileToLocal(ctx2, info, rHelper) err = c.putFileToLocal(ctx2, info, rHelper)
if err != nil { if err != nil {
log.Ctx(ctx2).Error().Str("goRoutine", log.Ctx(ctx2).Error().Stack().Err(err).Msgf("error while putting file to localRegistry, %v", err)
"AddMavenFile").Stack().Err(err).Msgf("error while putting file to localRegistry, %v", err)
return return
} }
log.Ctx(ctx2).Info().Str("goRoutine", "AddMavenFile").Msgf("Successfully updated file "+ log.Ctx(ctx2).Info().Msgf("Successfully updated file "+
"to registry: %s with file path: %s", "to registry: %s with file path: %s",
info.RegIdentifier, filePath) info.RegIdentifier, filePath)
}(info) }(info)

View File

@ -18,7 +18,7 @@ import (
"context" "context"
"io" "io"
"github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/remote/adapter" "github.com/harness/gitness/registry/app/remote/adapter"
@ -50,7 +50,7 @@ type remoteHelper struct {
// NewRemoteHelper create a remote interface. // NewRemoteHelper create a remote interface.
func NewRemoteHelper( func NewRemoteHelper(
ctx context.Context, spacePathStore store.SpacePathStore, secretService secret.Service, ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service,
proxy types.UpstreamProxy, proxy types.UpstreamProxy,
) (RemoteInterface, error) { ) (RemoteInterface, error) {
if proxy.Source == string(api.UpstreamConfigSourceMavenCentral) { if proxy.Source == string(api.UpstreamConfigSourceMavenCentral) {
@ -60,13 +60,13 @@ func NewRemoteHelper(
upstreamProxy: proxy, upstreamProxy: proxy,
secretService: secretService, secretService: secretService,
} }
if err := r.init(ctx, spacePathStore, string(api.UpstreamConfigSourceMavenCentral)); err != nil { if err := r.init(ctx, spaceFinder, string(api.UpstreamConfigSourceMavenCentral)); err != nil {
return nil, err return nil, err
} }
return r, nil return r, nil
} }
func (r *remoteHelper) init(ctx context.Context, spacePathStore store.SpacePathStore, proxyType string) error { func (r *remoteHelper) init(ctx context.Context, spaceFinder refcache.SpaceFinder, proxyType string) error {
if r.registry != nil { if r.registry != nil {
return nil return nil
} }
@ -75,7 +75,7 @@ func (r *remoteHelper) init(ctx context.Context, spacePathStore store.SpacePathS
if err != nil { if err != nil {
return err return err
} }
adp, err := factory.Create(ctx, spacePathStore, r.upstreamProxy, r.secretService) adp, err := factory.Create(ctx, spaceFinder, r.upstreamProxy, r.secretService)
if err != nil { if err != nil {
return err return err
} }

View File

@ -19,7 +19,7 @@ package proxy
import ( import (
"io" "io"
"github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/manifest" "github.com/harness/gitness/registry/app/manifest"
"github.com/harness/gitness/registry/app/remote/adapter" "github.com/harness/gitness/registry/app/remote/adapter"
@ -58,7 +58,7 @@ type remoteHelper struct {
// NewRemoteHelper create a remote interface. // NewRemoteHelper create a remote interface.
func NewRemoteHelper( func NewRemoteHelper(
ctx context.Context, spacePathStore store.SpacePathStore, secretService secret.Service, repoKey string, ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, repoKey string,
proxy types.UpstreamProxy, proxy types.UpstreamProxy,
) (RemoteInterface, error) { ) (RemoteInterface, error) {
if proxy.Source == string(api.UpstreamConfigSourceDockerhub) { if proxy.Source == string(api.UpstreamConfigSourceDockerhub) {
@ -73,13 +73,13 @@ func NewRemoteHelper(
if proxy.Source == string(api.UpstreamConfigSourceCustom) { if proxy.Source == string(api.UpstreamConfigSourceCustom) {
adapterType = string(api.UpstreamConfigSourceDockerhub) adapterType = string(api.UpstreamConfigSourceDockerhub)
} }
if err := r.init(ctx, spacePathStore, adapterType); err != nil { if err := r.init(ctx, spaceFinder, adapterType); err != nil {
return nil, err return nil, err
} }
return r, nil return r, nil
} }
func (r *remoteHelper) init(ctx context.Context, spacePathStore store.SpacePathStore, proxyType string) error { func (r *remoteHelper) init(ctx context.Context, spaceFinder refcache.SpaceFinder, proxyType string) error {
if r.registry != nil { if r.registry != nil {
return nil return nil
} }
@ -89,7 +89,7 @@ func (r *remoteHelper) init(ctx context.Context, spacePathStore store.SpacePathS
if err != nil { if err != nil {
return err return err
} }
adp, err := factory.Create(ctx, spacePathStore, r.upstreamProxy, r.secretService) adp, err := factory.Create(ctx, spaceFinder, r.upstreamProxy, r.secretService)
if err != nil { if err != nil {
return err return err
} }

View File

@ -21,7 +21,7 @@ import (
"time" "time"
"github.com/harness/gitness/app/api/request" "github.com/harness/gitness/app/api/request"
corestore "github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/store" "github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/registry/app/store/database/util" "github.com/harness/gitness/registry/app/store/database/util"
@ -35,19 +35,24 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
) )
var (
ulimit = uint64(0)
uoffset = uint64(0)
)
type UpstreamproxyDao struct { type UpstreamproxyDao struct {
registryDao store.RegistryRepository registryDao store.RegistryRepository
db *sqlx.DB db *sqlx.DB
spacePathStore corestore.SpacePathStore spaceFinder refcache.SpaceFinder
} }
func NewUpstreamproxyDao( func NewUpstreamproxyDao(
db *sqlx.DB, registryDao store.RegistryRepository, spacePathStore corestore.SpacePathStore, db *sqlx.DB, registryDao store.RegistryRepository, spaceFinder refcache.SpaceFinder,
) store.UpstreamProxyConfigRepository { ) store.UpstreamProxyConfigRepository {
return &UpstreamproxyDao{ return &UpstreamproxyDao{
registryDao: registryDao, registryDao: registryDao,
db: db, db: db,
spacePathStore: spacePathStore, spaceFinder: spaceFinder,
} }
} }
@ -307,8 +312,16 @@ func (r UpstreamproxyDao) GetAll(
q = q.Where(" AND r.registry_package_type in ? ", packageTypes) q = q.Where(" AND r.registry_package_type in ? ", packageTypes)
} }
q = q.OrderBy(" r.registry_" + sortByField + " " + sortByOrder).Limit(uint64(limit)).Offset(uint64(offset)) if limit > 0 {
ulimit = uint64(limit)
}
if offset > 0 {
uoffset = uint64(offset)
}
q = q.OrderBy(" r.registry_" + sortByField + " " + sortByOrder).
Limit(ulimit).
Offset(uoffset)
sql, args, err := q.ToSql() sql, args, err := q.ToSql()
if err != nil { if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql") return nil, errors.Wrap(err, "Failed to convert query to sql")
@ -411,11 +424,11 @@ func (r UpstreamproxyDao) mapToUpstreamProxy(
secretSpacePath := "" secretSpacePath := ""
if dst.SecretSpaceID.Valid { if dst.SecretSpaceID.Valid {
primary, err := r.spacePathStore.FindPrimaryBySpaceID(ctx, int64(dst.SecretSpaceID.Int32)) primary, err := r.spaceFinder.FindByID(ctx, int64(dst.SecretSpaceID.Int32))
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get secret space path: %w", err) return nil, fmt.Errorf("failed to get secret space path: %w", err)
} }
secretSpacePath = primary.Value secretSpacePath = primary.Path
} }
userNameSecretIdentifier := "" userNameSecretIdentifier := ""
@ -429,11 +442,11 @@ func (r UpstreamproxyDao) mapToUpstreamProxy(
userNameSecretSpacePath := "" userNameSecretSpacePath := ""
if dst.UserNameSecretSpaceID.Valid { if dst.UserNameSecretSpaceID.Valid {
primary, err := r.spacePathStore.FindPrimaryBySpaceID(ctx, int64(dst.UserNameSecretSpaceID.Int32)) primary, err := r.spaceFinder.FindByID(ctx, int64(dst.UserNameSecretSpaceID.Int32))
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get secret space path: %w", err) return nil, fmt.Errorf("failed to get secret space path: %w", err)
} }
userNameSecretSpacePath = primary.Value userNameSecretSpacePath = primary.Path
} }
return &types.UpstreamProxy{ return &types.UpstreamProxy{

View File

@ -15,7 +15,7 @@
package database package database
import ( import (
corestore "github.com/harness/gitness/app/store" "github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/store" "github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/store/database/dbtx" "github.com/harness/gitness/store/database/dbtx"
@ -26,9 +26,9 @@ import (
func ProvideUpstreamDao( func ProvideUpstreamDao(
db *sqlx.DB, db *sqlx.DB,
registryDao store.RegistryRepository, registryDao store.RegistryRepository,
spacePathStore corestore.SpacePathStore, spaceFinder refcache.SpaceFinder,
) store.UpstreamProxyConfigRepository { ) store.UpstreamProxyConfigRepository {
return NewUpstreamproxyDao(db, registryDao, spacePathStore) return NewUpstreamproxyDao(db, registryDao, spaceFinder)
} }
func ProvideRepoDao(db *sqlx.DB, mtRepository store.MediaTypesRepository) store.RegistryRepository { func ProvideRepoDao(db *sqlx.DB, mtRepository store.MediaTypesRepository) store.RegistryRepository {

View File

@ -14,8 +14,11 @@
package config package config
type contextKey string
// const variables. // const variables.
const ( const (
PostgresqlDatabase = "postgres" PostgresqlDatabase = "postgres"
Sqlite = "sqlite3" Sqlite = "sqlite3"
GoRoutineKey contextKey = "goRoutine"
) )