feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID (#3443)

* feat: [AH-925]: fix lint errors
* feat: [AH-925]: keep original context and also async flow. also keep the linter
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID - fix lint errors by ignoring problematic contextcheck
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID - lint errors
* feat: [AH-925]: rebase main
* feat: [AH-925]: rebase main
* feat: [AH-925]: rebase main
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID - fix lint
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID
* feat: [AH-925]: add cache for space_paths: replace DB calls for look up by spaceID
* feat: [AH-925]: rebase main
jobatzil/login/xforwardedfor
Shivakumar Ningappa 2025-02-19 21:03:21 +00:00 committed by Harness
parent bab50ed39e
commit 5f7808a2da
22 changed files with 164 additions and 133 deletions

View File

@ -449,19 +449,19 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
layerRepository := database2.ProvideLayerDao(db, mediaTypesRepository)
eventReporter := docker.ProvideReporter()
ociImageIndexMappingRepository := database2.ProvideOCIImageIndexMappingDao(db)
manifestService := docker.ManifestServiceProvider(registryRepository, manifestRepository, blobRepository, mediaTypesRepository, manifestReferenceRepository, tagRepository, imageRepository, artifactRepository, layerRepository, gcService, transactor, eventReporter, spacePathStore, ociImageIndexMappingRepository)
manifestService := docker.ManifestServiceProvider(registryRepository, manifestRepository, blobRepository, mediaTypesRepository, manifestReferenceRepository, tagRepository, imageRepository, artifactRepository, layerRepository, gcService, transactor, eventReporter, spaceFinder, ociImageIndexMappingRepository)
registryBlobRepository := database2.ProvideRegistryBlobDao(db)
bandwidthStatRepository := database2.ProvideBandwidthStatDao(db)
downloadStatRepository := database2.ProvideDownloadStatDao(db)
localRegistry := docker.LocalRegistryProvider(app, manifestService, blobRepository, registryRepository, manifestRepository, registryBlobRepository, mediaTypesRepository, tagRepository, imageRepository, artifactRepository, bandwidthStatRepository, downloadStatRepository, gcService, transactor)
upstreamProxyConfigRepository := database2.ProvideUpstreamDao(db, registryRepository, spacePathStore)
upstreamProxyConfigRepository := database2.ProvideUpstreamDao(db, registryRepository, spaceFinder)
secretService := secret3.ProvideSecretService(secretStore, encrypter, spacePathStore)
proxyController := docker.ProvideProxyController(localRegistry, manifestService, secretService, spacePathStore)
remoteRegistry := docker.RemoteRegistryProvider(localRegistry, app, upstreamProxyConfigRepository, spacePathStore, secretService, proxyController)
proxyController := docker.ProvideProxyController(localRegistry, manifestService, secretService, spaceFinder)
remoteRegistry := docker.RemoteRegistryProvider(localRegistry, app, upstreamProxyConfigRepository, spaceFinder, secretService, proxyController)
coreController := pkg.CoreControllerProvider(registryRepository)
dbStore := docker.DBStoreProvider(blobRepository, imageRepository, artifactRepository, bandwidthStatRepository, downloadStatRepository)
dockerController := docker.ControllerProvider(localRegistry, remoteRegistry, coreController, spaceStore, authorizer, dbStore)
handler := api2.NewHandlerProvider(dockerController, spaceStore, tokenStore, controller, authenticator, provider, authorizer, config)
handler := api2.NewHandlerProvider(dockerController, spaceFinder, spaceStore, tokenStore, controller, authenticator, provider, authorizer, config)
registryOCIHandler := router.OCIHandlerProvider(handler)
filemanagerApp := filemanager.NewApp(ctx, config, storageService)
genericBlobRepository := database2.ProvideGenericBlobDao(db)
@ -472,7 +472,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
apiHandler := router.APIHandlerProvider(registryRepository, upstreamProxyConfigRepository, fileManager, tagRepository, manifestRepository, cleanupPolicyRepository, imageRepository, storageDriver, spaceFinder, transactor, authenticator, provider, authorizer, auditService, spacePathStore, artifactRepository, webhooksRepository)
mavenDBStore := maven.DBStoreProvider(registryRepository, imageRepository, artifactRepository, spaceStore, bandwidthStatRepository, downloadStatRepository, nodesRepository, upstreamProxyConfigRepository)
mavenLocalRegistry := maven.LocalRegistryProvider(mavenDBStore, transactor, fileManager)
mavenController := maven.ProvideProxyController(mavenLocalRegistry, secretService, spacePathStore)
mavenController := maven.ProvideProxyController(mavenLocalRegistry, secretService, spaceFinder)
mavenRemoteRegistry := maven.RemoteRegistryProvider(mavenDBStore, transactor, mavenLocalRegistry, mavenController)
controller2 := maven.ControllerProvider(mavenLocalRegistry, mavenRemoteRegistry, authorizer, mavenDBStore)
mavenHandler := api2.NewMavenHandlerProvider(controller2, spaceStore, tokenStore, controller, authenticator, authorizer)

View File

@ -445,11 +445,11 @@ func (c *APIController) mapToWebhookResponseEntity(
}
secretSpacePath := ""
if createdWebhook.SecretSpaceID > 0 {
primary, err := c.spacePathStore.FindPrimaryBySpaceID(ctx, int64(createdWebhook.SecretSpaceID))
primary, err := c.SpaceFinder.FindByID(ctx, int64(createdWebhook.SecretSpaceID))
if err != nil {
return nil, fmt.Errorf("failed to get secret space path: %w", err)
}
secretSpacePath = primary.Value
secretSpacePath = primary.Path
}
if createdWebhook.SecretIdentifier != "" {
webhookResponseEntity.SecretIdentifier = &createdWebhook.SecretIdentifier

View File

@ -22,6 +22,7 @@ import (
usercontroller "github.com/harness/gitness/app/api/controller/user"
"github.com/harness/gitness/app/auth/authn"
"github.com/harness/gitness/app/auth/authz"
"github.com/harness/gitness/app/services/refcache"
corestore "github.com/harness/gitness/app/store"
urlprovider "github.com/harness/gitness/app/url"
"github.com/harness/gitness/registry/app/api/controller/metadata"
@ -41,12 +42,14 @@ import (
)
func NewHandler(
controller *docker.Controller, spaceStore corestore.SpaceStore, tokenStore corestore.TokenStore,
controller *docker.Controller, spaceFinder refcache.SpaceFinder, spaceStore corestore.SpaceStore,
tokenStore corestore.TokenStore,
userCtrl *usercontroller.Controller, authenticator authn.Authenticator, urlProvider urlprovider.Provider,
authorizer authz.Authorizer, ociRelativeURL bool,
) *Handler {
return &Handler{
Controller: controller,
SpaceFinder: spaceFinder,
SpaceStore: spaceStore,
TokenStore: tokenStore,
UserCtrl: userCtrl,
@ -59,6 +62,7 @@ func NewHandler(
type Handler struct {
Controller *docker.Controller
SpaceFinder refcache.SpaceFinder
SpaceStore corestore.SpaceStore
TokenStore corestore.TokenStore
UserCtrl *usercontroller.Controller

View File

@ -96,14 +96,14 @@ func (h *Handler) GetToken(w http.ResponseWriter, r *http.Request) {
}
}
func (h *Handler) getSpace(ctx context.Context, name string) (*types.Space, error) {
func (h *Handler) getSpace(ctx context.Context, name string) (*types.SpaceCore, error) {
spaceRef, _, _ := paths.DisectRoot(name)
space, err := h.SpaceStore.FindByRef(ctx, spaceRef)
space, err := h.SpaceFinder.FindByRef(ctx, spaceRef)
return space, err
}
func (h *Handler) getAccessPermissionList(
ctx context.Context, space *types.Space, ra *ResourceActions, session *auth.Session,
ctx context.Context, space *types.SpaceCore, ra *ResourceActions, session *auth.Session,
accessPermissionsList []jwt.AccessPermissions,
) []jwt.AccessPermissions {
accessPermissions := &jwt.AccessPermissions{SpaceID: space.ID, Permissions: []enum.Permission{}}
@ -118,7 +118,7 @@ func (h *Handler) getAccessPermissionList(
ctx,
h.Authorizer,
session,
space.Core(),
space,
enum.ResourceTypeRegistry,
permission,
)

View File

@ -18,6 +18,7 @@ import (
usercontroller "github.com/harness/gitness/app/api/controller/user"
"github.com/harness/gitness/app/auth/authn"
"github.com/harness/gitness/app/auth/authz"
"github.com/harness/gitness/app/services/refcache"
corestore "github.com/harness/gitness/app/store"
urlprovider "github.com/harness/gitness/app/url"
"github.com/harness/gitness/registry/app/api/handler/generic"
@ -71,12 +72,13 @@ func BlobStorageProvider(c *types.Config) (storagedriver.StorageDriver, error) {
}
func NewHandlerProvider(
controller *docker.Controller, spaceStore corestore.SpaceStore,
controller *docker.Controller, spaceFinder refcache.SpaceFinder, spaceStore corestore.SpaceStore,
tokenStore corestore.TokenStore, userCtrl *usercontroller.Controller, authenticator authn.Authenticator,
urlProvider urlprovider.Provider, authorizer authz.Authorizer, config *types.Config,
) *ocihandler.Handler {
return ocihandler.NewHandler(
controller,
spaceFinder,
spaceStore,
tokenStore,
userCtrl,

View File

@ -23,7 +23,7 @@ import (
"fmt"
"time"
gas "github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/event"
"github.com/harness/gitness/registry/app/manifest"
@ -58,7 +58,7 @@ type manifestService struct {
artifactDao store.ArtifactRepository
manifestRefDao store.ManifestReferenceRepository
ociImageIndexMappingDao store.OCIImageIndexMappingRepository
spacePathStore gas.SpacePathStore
spaceFinder refcache.SpaceFinder
gcService gc.Service
tx dbtx.Transactor
reporter event.Reporter
@ -69,7 +69,7 @@ func NewManifestService(
blobRepo store.BlobRepository, mtRepository store.MediaTypesRepository, tagDao store.TagRepository,
imageDao store.ImageRepository, artifactDao store.ArtifactRepository,
layerDao store.LayerRepository, manifestRefDao store.ManifestReferenceRepository,
tx dbtx.Transactor, gcService gc.Service, reporter event.Reporter, spacePathStore gas.SpacePathStore,
tx dbtx.Transactor, gcService gc.Service, reporter event.Reporter, spaceFinder refcache.SpaceFinder,
ociImageIndexMappingDao store.OCIImageIndexMappingRepository,
) ManifestService {
return &manifestService{
@ -85,7 +85,7 @@ func NewManifestService(
gcService: gcService,
tx: tx,
reporter: reporter,
spacePathStore: spacePathStore,
spaceFinder: spaceFinder,
ociImageIndexMappingDao: ociImageIndexMappingDao,
}
}
@ -279,7 +279,7 @@ func (l *manifestService) getSpacePathAndPackageType(
ctx context.Context,
dbRepo *types.Registry,
) (string, event.PackageType, error) {
spacePath, err := l.spacePathStore.FindPrimaryBySpaceID(ctx, dbRepo.ParentID)
spacePath, err := l.spaceFinder.FindByID(ctx, dbRepo.ParentID)
if err != nil {
log.Ctx(ctx).Err(err).Msg("Failed to find spacePath")
return "", event.PackageType(0), err
@ -291,7 +291,7 @@ func (l *manifestService) getSpacePathAndPackageType(
return "", event.PackageType(0), err
}
return spacePath.Value, packageType, nil
return spacePath.Path, packageType, nil
}
// Reports event asynchronously.

View File

@ -24,7 +24,7 @@ import (
"time"
"github.com/harness/gitness/app/api/request"
store2 "github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/common/lib/errors"
"github.com/harness/gitness/registry/app/manifest"
"github.com/harness/gitness/registry/app/manifest/manifestlist"
@ -34,6 +34,7 @@ import (
proxy2 "github.com/harness/gitness/registry/app/remote/controller/proxy"
"github.com/harness/gitness/registry/app/storage"
"github.com/harness/gitness/registry/app/store"
cfg "github.com/harness/gitness/registry/config"
"github.com/harness/gitness/secret"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
@ -51,7 +52,7 @@ const (
func NewRemoteRegistry(
local *LocalRegistry, app *App, upstreamProxyConfigRepo store.UpstreamProxyConfigRepository,
spacePathStore store2.SpacePathStore, secretService secret.Service, proxyCtl proxy2.Controller,
spaceFinder refcache.SpaceFinder, secretService secret.Service, proxyCtl proxy2.Controller,
) Registry {
cache := proxy2.GetManifestCache(local, local.ms)
listCache := proxy2.GetManifestListCache(local)
@ -67,7 +68,7 @@ func NewRemoteRegistry(
local: local,
App: app,
upstreamProxyConfigRepo: upstreamProxyConfigRepo,
spacePathStore: spacePathStore,
spaceFinder: spaceFinder,
secretService: secretService,
manifestCacheHandlerMap: registry,
proxyCtl: proxyCtl,
@ -82,7 +83,7 @@ type RemoteRegistry struct {
local *LocalRegistry
App *App
upstreamProxyConfigRepo store.UpstreamProxyConfigRepository
spacePathStore store2.SpacePathStore
spaceFinder refcache.SpaceFinder
secretService secret.Service
proxyCtl proxy2.Controller
manifestCacheHandlerMap map[string]proxy2.ManifestCacheHandler
@ -116,7 +117,9 @@ func proxyManifestHead(
go func(art pkg.RegistryInfo) {
// Write function to update local storage.
session, _ := request.AuthSessionFrom(ctx)
ctx2 := request.WithAuthSession(context.Background(), session)
ctx2 := request.WithAuthSession(ctx, session)
ctx2 = context.WithoutCancel(ctx2)
ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "EnsureTag")
tag := art.Tag
art.Tag = ""
art.Digest = desc.Digest.String()
@ -125,16 +128,15 @@ func proxyManifestHead(
for i := 0; i < ensureTagMaxRetry; i++ {
time.Sleep(ensureTagInterval)
count++
log.Ctx(ctx2).Info().Str("goRoutine", "EnsureTag").Msgf("Tag %s for image: %s, retry: %d", tag,
log.Ctx(ctx2).Info().Msgf("Tag %s for image: %s, retry: %d", tag,
info.Image,
count)
e := ctl.EnsureTag(ctx2, responseHeaders, art, acceptHeaders, ifNoneMatchHeader)
if e != nil {
log.Ctx(ctx2).Warn().Str("goRoutine",
"EnsureTag").Err(e).Msgf("Failed to update tag: %s for image: %s",
log.Ctx(ctx2).Warn().Err(e).Msgf("Failed to update tag: %s for image: %s",
tag, info.Image)
} else {
log.Ctx(ctx2).Info().Str("goRoutine", "EnsureTag").Msgf("Tag updated: %s for image: %s", tag,
log.Ctx(ctx2).Info().Msgf("Tag updated: %s for image: %s", tag,
info.Image)
return
}
@ -175,7 +177,7 @@ func (r *RemoteRegistry) ManifestExist(
errs = append(errs, err)
return responseHeaders, descriptor, manifestResult, errs
}
remoteHelper, err := proxy2.NewRemoteHelper(ctx, r.spacePathStore, r.secretService, artInfo.RegIdentifier,
remoteHelper, err := proxy2.NewRemoteHelper(ctx, r.spaceFinder, r.secretService, artInfo.RegIdentifier,
*upstreamProxy)
if err != nil {
errs = append(errs, errors.New("Proxy is down"))
@ -251,7 +253,7 @@ func (r *RemoteRegistry) PullManifest(
errs = append(errs, err)
return responseHeaders, descriptor, manifestResult, errs
}
remoteHelper, err := proxy2.NewRemoteHelper(ctx, r.spacePathStore, r.secretService, artInfo.RegIdentifier,
remoteHelper, err := proxy2.NewRemoteHelper(ctx, r.spaceFinder, r.secretService, artInfo.RegIdentifier,
*upstreamProxy)
if err != nil {
errs = append(errs, errors.New("Proxy is down"))

View File

@ -16,6 +16,7 @@ package docker
import (
"github.com/harness/gitness/app/auth/authz"
"github.com/harness/gitness/app/services/refcache"
gitnessstore "github.com/harness/gitness/app/store"
storagedriver "github.com/harness/gitness/registry/app/driver"
"github.com/harness/gitness/registry/app/event"
@ -43,6 +44,7 @@ func LocalRegistryProvider(
bandwidthStatDao store.BandwidthStatRepository, downloadStatDao store.DownloadStatRepository,
gcService gc.Service, tx dbtx.Transactor,
) *LocalRegistry {
//nolint:errcheck
return NewLocalRegistry(
app, ms, manifestDao, registryDao, registryBlobDao, blobRepo,
mtRepository, tagDao, imageDao, artifactDao, bandwidthStatDao, downloadStatDao, gcService, tx,
@ -54,21 +56,22 @@ func ManifestServiceProvider(
manifestDao store.ManifestRepository, blobRepo store.BlobRepository, mtRepository store.MediaTypesRepository,
manifestRefDao store.ManifestReferenceRepository, tagDao store.TagRepository, imageDao store.ImageRepository,
artifactDao store.ArtifactRepository, layerDao store.LayerRepository,
gcService gc.Service, tx dbtx.Transactor, reporter event.Reporter, spacePathStore gitnessstore.SpacePathStore,
gcService gc.Service, tx dbtx.Transactor, reporter event.Reporter, spaceFinder refcache.SpaceFinder,
ociImageIndexMappingDao store.OCIImageIndexMappingRepository,
) ManifestService {
return NewManifestService(
registryDao, manifestDao, blobRepo, mtRepository, tagDao, imageDao,
artifactDao, layerDao, manifestRefDao, tx, gcService, reporter, spacePathStore,
artifactDao, layerDao, manifestRefDao, tx, gcService, reporter, spaceFinder,
ociImageIndexMappingDao,
)
}
func RemoteRegistryProvider(
local *LocalRegistry, app *App, upstreamProxyConfigRepo store.UpstreamProxyConfigRepository,
spacePathStore gitnessstore.SpacePathStore, secretService secret.Service, proxyCtrl proxy2.Controller,
spaceFinder refcache.SpaceFinder, secretService secret.Service, proxyCtrl proxy2.Controller,
) *RemoteRegistry {
return NewRemoteRegistry(local, app, upstreamProxyConfigRepo, spacePathStore, secretService,
//nolint:errcheck
return NewRemoteRegistry(local, app, upstreamProxyConfigRepo, spaceFinder, secretService,
proxyCtrl).(*RemoteRegistry)
}
@ -103,10 +106,10 @@ func ProvideReporter() event.Reporter {
func ProvideProxyController(
registry *LocalRegistry, ms ManifestService, secretService secret.Service,
spacePathStore gitnessstore.SpacePathStore,
spaceFinder refcache.SpaceFinder,
) proxy2.Controller {
manifestCacheHandler := getManifestCacheHandler(registry, ms)
return proxy2.NewProxyController(registry, ms, secretService, spacePathStore, manifestCacheHandler)
return proxy2.NewProxyController(registry, ms, secretService, spaceFinder, manifestCacheHandler)
}
func getManifestCacheHandler(

View File

@ -16,6 +16,7 @@ package maven
import (
"github.com/harness/gitness/app/auth/authz"
"github.com/harness/gitness/app/services/refcache"
corestore "github.com/harness/gitness/app/store"
"github.com/harness/gitness/registry/app/pkg/filemanager"
"github.com/harness/gitness/registry/app/remote/controller/proxy/maven"
@ -31,6 +32,7 @@ func LocalRegistryProvider(
tx dbtx.Transactor,
fileManager filemanager.FileManager,
) *LocalRegistry {
//nolint:errcheck
return NewLocalRegistry(dBStore,
tx,
fileManager,
@ -43,6 +45,7 @@ func RemoteRegistryProvider(
local *LocalRegistry,
proxyController maven.Controller,
) *RemoteRegistry {
//nolint:errcheck
return NewRemoteRegistry(dBStore, tx, local, proxyController).(*RemoteRegistry)
}
@ -65,6 +68,7 @@ func DBStoreProvider(
nodeDao store.NodesRepository,
upstreamProxyDao store.UpstreamProxyConfigRepository,
) *DBStore {
//nolint:errcheck
return NewDBStore(registryDao, imageDao, artifactDao, spaceStore, bandwidthStatDao,
downloadStatDao,
nodeDao,
@ -73,9 +77,9 @@ func DBStoreProvider(
func ProvideProxyController(
registry *LocalRegistry, secretService secret.Service,
spacePathStore corestore.SpacePathStore,
spaceFinder refcache.SpaceFinder,
) maven.Controller {
return maven.NewProxyController(registry, secretService, spacePathStore)
return maven.NewProxyController(registry, secretService, spaceFinder)
}
var ControllerSet = wire.NewSet(ControllerProvider)

View File

@ -22,7 +22,7 @@ import (
"fmt"
"io"
store2 "github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/manifest"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/types"
@ -40,7 +40,7 @@ var registryKeys = []string{}
// Factory creates a specific Adapter according to the params.
type Factory interface {
Create(
ctx context.Context, spacePathStore store2.SpacePathStore, record types.UpstreamProxy, service secret.Service,
ctx context.Context, spaceFinder refcache.SpaceFinder, record types.UpstreamProxy, service secret.Service,
) (Adapter, error)
}

View File

@ -20,7 +20,7 @@ import (
"context"
"regexp"
store2 "github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
adp "github.com/harness/gitness/registry/app/remote/adapter"
"github.com/harness/gitness/registry/app/remote/adapter/native"
@ -49,9 +49,9 @@ func init() {
}
func newAdapter(
ctx context.Context, spacePathStore store2.SpacePathStore, service secret.Service, registry types.UpstreamProxy,
ctx context.Context, spaceFinder refcache.SpaceFinder, service secret.Service, registry types.UpstreamProxy,
) (adp.Adapter, error) {
accessKey, secretKey, isPublic, err := getCreds(ctx, spacePathStore, service, registry)
accessKey, secretKey, isPublic, err := getCreds(ctx, spaceFinder, service, registry)
if err != nil {
return nil, err
}
@ -73,9 +73,9 @@ func newAdapter(
// Create ...
func (f *factory) Create(
ctx context.Context, spacePathStore store2.SpacePathStore, record types.UpstreamProxy, service secret.Service,
ctx context.Context, spaceFinder refcache.SpaceFinder, record types.UpstreamProxy, service secret.Service,
) (adp.Adapter, error) {
return newAdapter(ctx, spacePathStore, service, record)
return newAdapter(ctx, spaceFinder, service, record)
}
type factory struct {

View File

@ -28,7 +28,7 @@ import (
"strings"
"time"
"github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
commonhttp "github.com/harness/gitness/registry/app/common/http"
"github.com/harness/gitness/registry/app/common/http/modifier"
@ -135,14 +135,14 @@ func getAwsSvc(accessKey, secretKey string, reg types.UpstreamProxy) (*awsecrapi
func parseAccountRegion(url string) (string, string, error) {
rs := ecrRegexp.FindStringSubmatch(url)
if rs == nil || len(rs) < 4 {
if len(rs) < 4 {
return "", "", errors.New("bad aws url")
}
return rs[1], rs[3], nil
}
func getCreds(
ctx context.Context, spacePathStore store.SpacePathStore, secretService secret.Service, reg types.UpstreamProxy,
ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, reg types.UpstreamProxy,
) (string, string, bool, error) {
if api.AuthType(reg.RepoAuthType) == api.AuthTypeAnonymous {
return "", "", true, nil
@ -151,7 +151,7 @@ func getCreds(
log.Debug().Msgf("invalid auth type: %s", reg.RepoAuthType)
return "", "", false, nil
}
secretKey, err := getSecretValue(ctx, spacePathStore, secretService, reg.SecretSpaceID,
secretKey, err := getSecretValue(ctx, spaceFinder, secretService, reg.SecretSpaceID,
reg.SecretIdentifier)
if err != nil {
return "", "", false, err
@ -159,7 +159,7 @@ func getCreds(
if reg.UserName != "" {
return reg.UserName, secretKey, false, nil
}
accessKey, err := getSecretValue(ctx, spacePathStore, secretService, reg.UserNameSecretSpaceID,
accessKey, err := getSecretValue(ctx, spaceFinder, secretService, reg.UserNameSecretSpaceID,
reg.UserNameSecretIdentifier)
if err != nil {
return "", "", false, err
@ -167,14 +167,14 @@ func getCreds(
return accessKey, secretKey, false, nil
}
func getSecretValue(ctx context.Context, spacePathStore store.SpacePathStore, secretService secret.Service,
func getSecretValue(ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service,
secretSpaceID int64, secretSpacePath string) (string, error) {
spacePath, err := spacePathStore.FindPrimaryBySpaceID(ctx, secretSpaceID)
spacePath, err := spaceFinder.FindByID(ctx, secretSpaceID)
if err != nil {
log.Error().Msgf("failed to find space path: %v", err)
return "", err
}
decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Value, secretSpacePath)
decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Path, secretSpacePath)
if err != nil {
log.Error().Msgf("failed to decrypt secret: %v", err)
return "", err

View File

@ -19,7 +19,7 @@ package dockerhub
import (
"context"
store2 "github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
adp "github.com/harness/gitness/registry/app/remote/adapter"
"github.com/harness/gitness/registry/app/remote/adapter/native"
@ -38,7 +38,7 @@ func init() {
}
func newAdapter(
ctx context.Context, spacePathStore store2.SpacePathStore, service secret.Service, registry types.UpstreamProxy,
ctx context.Context, spaceFinder refcache.SpaceFinder, service secret.Service, registry types.UpstreamProxy,
) (adp.Adapter, error) {
client, err := NewClient(registry)
if err != nil {
@ -48,7 +48,7 @@ func newAdapter(
// TODO: get Upstream Credentials
return &adapter{
client: client,
Adapter: native.NewAdapter(ctx, spacePathStore, service, registry),
Adapter: native.NewAdapter(ctx, spaceFinder, service, registry),
}, nil
}
@ -57,9 +57,9 @@ type factory struct {
// Create ...
func (f *factory) Create(
ctx context.Context, spacePathStore store2.SpacePathStore, record types.UpstreamProxy, service secret.Service,
ctx context.Context, spaceFinder refcache.SpaceFinder, record types.UpstreamProxy, service secret.Service,
) (adp.Adapter, error) {
return newAdapter(ctx, spacePathStore, service, record)
return newAdapter(ctx, spaceFinder, service, record)
}
var (

View File

@ -20,7 +20,7 @@ import (
"context"
"net/http"
store2 "github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
commonhttp "github.com/harness/gitness/registry/app/common/http"
adp "github.com/harness/gitness/registry/app/remote/adapter"
@ -44,7 +44,7 @@ func init() {
}
func newAdapter(
ctx context.Context, spacePathStore store2.SpacePathStore, service secret.Service, registry types.UpstreamProxy,
ctx context.Context, spaceFinder refcache.SpaceFinder, service secret.Service, registry types.UpstreamProxy,
) (adp.Adapter, error) {
client, err := NewClient(registry)
if err != nil {
@ -53,7 +53,7 @@ func newAdapter(
return &adapter{
client: client,
Adapter: native.NewAdapter(ctx, spacePathStore, service, registry),
Adapter: native.NewAdapter(ctx, spaceFinder, service, registry),
}, nil
}
@ -62,9 +62,9 @@ type factory struct {
// Create ...
func (f *factory) Create(
ctx context.Context, spacePathStore store2.SpacePathStore, record types.UpstreamProxy, service secret.Service,
ctx context.Context, spaceFinder refcache.SpaceFinder, record types.UpstreamProxy, service secret.Service,
) (adp.Adapter, error) {
return newAdapter(ctx, spacePathStore, service, record)
return newAdapter(ctx, spaceFinder, service, record)
}
var (

View File

@ -19,7 +19,7 @@ package native
import (
"context"
"github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/common/lib"
"github.com/harness/gitness/registry/app/common/lib/errors"
@ -47,13 +47,13 @@ type Adapter struct {
// NewAdapter returns an instance of the Adapter.
func NewAdapter(
ctx context.Context, spacePathStore store.SpacePathStore, service secret.Service, reg types.UpstreamProxy,
ctx context.Context, spaceFinder refcache.SpaceFinder, service secret.Service, reg types.UpstreamProxy,
) *Adapter {
adapter := &Adapter{
proxy: reg,
}
// Get the password: lookup secrets.secret_data using secret_identifier & secret_space_id.
password := getPwd(ctx, spacePathStore, service, reg)
password := getPwd(ctx, spaceFinder, service, reg)
username, password, url := reg.UserName, password, reg.RepoURL
adapter.Client = registry.NewClient(url, username, password, false)
return adapter
@ -69,18 +69,18 @@ func NewAdapterWithAuthorizer(reg types.UpstreamProxy, authorizer lib.Authorizer
// getPwd: lookup secrets.secret_data using secret_identifier & secret_space_id.
func getPwd(
ctx context.Context, spacePathStore store.SpacePathStore, secretService secret.Service, reg types.UpstreamProxy,
ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, reg types.UpstreamProxy,
) string {
if api.AuthType(reg.RepoAuthType) == api.AuthTypeUserPassword {
secretSpaceID := reg.SecretSpaceID
secretIdentifier := reg.SecretIdentifier
spacePath, err := spacePathStore.FindPrimaryBySpaceID(ctx, secretSpaceID)
spacePath, err := spaceFinder.FindByID(ctx, secretSpaceID)
if err != nil {
log.Error().Msgf("failed to find space path: %v", err)
return ""
}
decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Value, secretIdentifier)
decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Path, secretIdentifier)
if err != nil {
log.Error().Msgf("failed to decrypt secret: %v", err)
return ""

View File

@ -25,11 +25,12 @@ import (
"time"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/common/lib/errors"
"github.com/harness/gitness/registry/app/manifest"
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/commons"
cfg "github.com/harness/gitness/registry/config"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/secret"
@ -91,20 +92,20 @@ type controller struct {
localRegistry registryInterface
localManifestRegistry registryManifestInterface
secretService secret.Service
spacePathStore store.SpacePathStore
spaceFinder refcache.SpaceFinder
manifestCacheHandlerMap map[string]ManifestCacheHandler
}
// NewProxyController -- get the proxy controller instance.
func NewProxyController(
l registryInterface, lm registryManifestInterface, secretService secret.Service,
spacePathStore store.SpacePathStore, manifestCacheHandlerMap map[string]ManifestCacheHandler,
spaceFinder refcache.SpaceFinder, manifestCacheHandlerMap map[string]ManifestCacheHandler,
) Controller {
return &controller{
localRegistry: l,
localManifestRegistry: lm,
secretService: secretService,
spacePathStore: spacePathStore,
spaceFinder: spaceFinder,
manifestCacheHandlerMap: manifestCacheHandlerMap,
}
}
@ -229,18 +230,19 @@ func (c *controller) ProxyManifest(
// This GoRoutine is to push the manifest from Remote to Local registry.
go func(_, ct string) {
session, _ := request.AuthSessionFrom(ctx)
ctx2 := request.WithAuthSession(context.Background(), session)
ctx2 := request.WithAuthSession(ctx, session)
ctx2 = context.WithoutCancel(ctx2)
ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "UpdateManifest")
var count = 0
for n := 0; n < maxManifestWait; n++ {
time.Sleep(sleepIntervalSec * time.Second)
count++
log.Ctx(ctx2).Info().Str("goRoutine", "UpdateManifest").Msgf("Current retry=%v artifact: %v:%v, digest: %s",
log.Ctx(ctx2).Info().Msgf("Current retry=%v artifact: %v:%v, digest: %s",
count, repoKey, imageName,
art.Digest)
_, des, _, e := c.localRegistry.PullManifest(ctx2, art, acceptHeader, ifNoneMatchHeader)
if len(e) > 0 {
log.Ctx(ctx2).Info().Str("goRoutine",
"UpdateManifest").Stack().Err(err).Msgf("Local manifest doesn't exist, error %v", e[0])
log.Ctx(ctx2).Info().Stack().Err(err).Msgf("Local manifest doesn't exist, error %v", e[0])
}
// Push manifest to localRegistry when pull with digest, or artifact not found, or digest mismatch.
errs := []error{}
@ -249,7 +251,6 @@ func (c *controller) ProxyManifest(
if len(artInfo.Digest) == 0 {
artInfo.Digest = dig
}
err = c.waitAndPushManifest(ctx2, art, ct, man)
if err != nil {
continue
@ -260,10 +261,9 @@ func (c *controller) ProxyManifest(
if e == nil || commons.IsEmpty(errs) {
_, _, _, err := c.localRegistry.PullManifest(ctx2, art, acceptHeader, ifNoneMatchHeader)
if err != nil {
log.Ctx(ctx2).Error().Str("goRoutine",
"UpdateManifest").Stack().Msgf("failed to get manifest, error %v", err)
log.Ctx(ctx2).Error().Stack().Msgf("failed to get manifest, error %v", err)
} else {
log.Ctx(ctx2).Info().Str("goRoutine", "UpdateManifest").Msgf(
log.Ctx(ctx2).Info().Msgf(
"Completed manifest push to localRegistry registry. Image: %s, Tag: %s, Digest: %s",
art.Image, art.Tag, art.Digest,
)
@ -296,7 +296,7 @@ func (c *controller) ProxyBlob(
remoteImage := getRemoteRepo(art)
log.Debug().Msgf("The blob doesn't exist, proxy the request to the target server, url:%v", remoteImage)
rHelper, err := NewRemoteHelper(ctx, c.spacePathStore, c.secretService, repoKey, proxy)
rHelper, err := NewRemoteHelper(ctx, c.spaceFinder, c.secretService, repoKey, proxy)
if err != nil {
return 0, nil, err
}
@ -316,18 +316,17 @@ func (c *controller) ProxyBlob(
log.Error().Stack().Err(err).Msg("failed to get auth session from context")
return
}
ctx2 := request.WithAuthSession(context.Background(), session)
ctx2 = log.Ctx(ctx2).With().
Str("goRoutine", "AddBlob").
Logger().WithContext(ctx2)
ctx2 := request.WithAuthSession(ctx, session)
ctx2 = context.WithoutCancel(ctx2)
ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "AddBlob")
ctx2 = log.Ctx(ctx2).With().Logger().WithContext(ctx2)
err := c.putBlobToLocal(ctx2, art, remoteImage, repoKey, desc, rHelper)
if err != nil {
log.Ctx(ctx2).Error().Str("goRoutine",
"AddBlob").Stack().Err(err).Msgf("error while putting blob to localRegistry registry, %v", err)
log.Ctx(ctx2).Error().Stack().Err(err).
Msgf("error while putting blob to localRegistry registry, %v", err)
return
}
log.Ctx(ctx2).Info().Str("goRoutine", "AddBlob").Msgf("Successfully updated the cache for digest %s",
art.Digest)
log.Ctx(ctx2).Info().Msgf("Successfully updated the cache for digest %s", art.Digest)
}(art)
return size, bReader, nil
}

View File

@ -20,11 +20,12 @@ import (
"strings"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/maven/utils"
"github.com/harness/gitness/registry/app/storage"
cfg "github.com/harness/gitness/registry/config"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/secret"
@ -32,9 +33,9 @@ import (
)
type controller struct {
localRegistry registryInterface
secretService secret.Service
spacePathStore store.SpacePathStore
localRegistry registryInterface
secretService secret.Service
spaceFinder refcache.SpaceFinder
}
type Controller interface {
@ -49,12 +50,12 @@ type Controller interface {
// NewProxyController -- get the proxy controller instance.
func NewProxyController(
l registryInterface, secretService secret.Service,
spacePathStore store.SpacePathStore,
spaceFinder refcache.SpaceFinder,
) Controller {
return &controller{
localRegistry: l,
secretService: secretService,
spacePathStore: spacePathStore,
localRegistry: l,
secretService: secretService,
spaceFinder: spaceFinder,
}
}
@ -70,7 +71,7 @@ func (c *controller) ProxyFile(
responseHeaders = &commons.ResponseHeaders{
Headers: make(map[string]string),
}
rHelper, err := NewRemoteHelper(ctx, c.spacePathStore, c.secretService, proxy)
rHelper, err := NewRemoteHelper(ctx, c.spaceFinder, c.secretService, proxy)
if err != nil {
return responseHeaders, nil, err
}
@ -100,15 +101,15 @@ func (c *controller) ProxyFile(
log.Error().Stack().Err(err).Msg("failed to get auth session from context")
return
}
ctx2 := request.WithAuthSession(context.Background(), session)
ctx2 := request.WithAuthSession(ctx, session)
ctx2 = context.WithoutCancel(ctx2)
ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "goRoutine")
err = c.putFileToLocal(ctx2, info, rHelper)
if err != nil {
log.Ctx(ctx2).Error().Str("goRoutine",
"AddMavenFile").Stack().Err(err).Msgf("error while putting file to localRegistry, %v", err)
log.Ctx(ctx2).Error().Stack().Err(err).Msgf("error while putting file to localRegistry, %v", err)
return
}
log.Ctx(ctx2).Info().Str("goRoutine", "AddMavenFile").Msgf("Successfully updated file "+
log.Ctx(ctx2).Info().Msgf("Successfully updated file "+
"to registry: %s with file path: %s",
info.RegIdentifier, filePath)
}(info)

View File

@ -18,7 +18,7 @@ import (
"context"
"io"
"github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/remote/adapter"
@ -50,7 +50,7 @@ type remoteHelper struct {
// NewRemoteHelper create a remote interface.
func NewRemoteHelper(
ctx context.Context, spacePathStore store.SpacePathStore, secretService secret.Service,
ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service,
proxy types.UpstreamProxy,
) (RemoteInterface, error) {
if proxy.Source == string(api.UpstreamConfigSourceMavenCentral) {
@ -60,13 +60,13 @@ func NewRemoteHelper(
upstreamProxy: proxy,
secretService: secretService,
}
if err := r.init(ctx, spacePathStore, string(api.UpstreamConfigSourceMavenCentral)); err != nil {
if err := r.init(ctx, spaceFinder, string(api.UpstreamConfigSourceMavenCentral)); err != nil {
return nil, err
}
return r, nil
}
func (r *remoteHelper) init(ctx context.Context, spacePathStore store.SpacePathStore, proxyType string) error {
func (r *remoteHelper) init(ctx context.Context, spaceFinder refcache.SpaceFinder, proxyType string) error {
if r.registry != nil {
return nil
}
@ -75,7 +75,7 @@ func (r *remoteHelper) init(ctx context.Context, spacePathStore store.SpacePathS
if err != nil {
return err
}
adp, err := factory.Create(ctx, spacePathStore, r.upstreamProxy, r.secretService)
adp, err := factory.Create(ctx, spaceFinder, r.upstreamProxy, r.secretService)
if err != nil {
return err
}

View File

@ -19,7 +19,7 @@ package proxy
import (
"io"
"github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/manifest"
"github.com/harness/gitness/registry/app/remote/adapter"
@ -58,7 +58,7 @@ type remoteHelper struct {
// NewRemoteHelper create a remote interface.
func NewRemoteHelper(
ctx context.Context, spacePathStore store.SpacePathStore, secretService secret.Service, repoKey string,
ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, repoKey string,
proxy types.UpstreamProxy,
) (RemoteInterface, error) {
if proxy.Source == string(api.UpstreamConfigSourceDockerhub) {
@ -73,13 +73,13 @@ func NewRemoteHelper(
if proxy.Source == string(api.UpstreamConfigSourceCustom) {
adapterType = string(api.UpstreamConfigSourceDockerhub)
}
if err := r.init(ctx, spacePathStore, adapterType); err != nil {
if err := r.init(ctx, spaceFinder, adapterType); err != nil {
return nil, err
}
return r, nil
}
func (r *remoteHelper) init(ctx context.Context, spacePathStore store.SpacePathStore, proxyType string) error {
func (r *remoteHelper) init(ctx context.Context, spaceFinder refcache.SpaceFinder, proxyType string) error {
if r.registry != nil {
return nil
}
@ -89,7 +89,7 @@ func (r *remoteHelper) init(ctx context.Context, spacePathStore store.SpacePathS
if err != nil {
return err
}
adp, err := factory.Create(ctx, spacePathStore, r.upstreamProxy, r.secretService)
adp, err := factory.Create(ctx, spaceFinder, r.upstreamProxy, r.secretService)
if err != nil {
return err
}

View File

@ -21,7 +21,7 @@ import (
"time"
"github.com/harness/gitness/app/api/request"
corestore "github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/registry/app/store/database/util"
@ -35,19 +35,24 @@ import (
"github.com/pkg/errors"
)
var (
ulimit = uint64(0)
uoffset = uint64(0)
)
type UpstreamproxyDao struct {
registryDao store.RegistryRepository
db *sqlx.DB
spacePathStore corestore.SpacePathStore
registryDao store.RegistryRepository
db *sqlx.DB
spaceFinder refcache.SpaceFinder
}
func NewUpstreamproxyDao(
db *sqlx.DB, registryDao store.RegistryRepository, spacePathStore corestore.SpacePathStore,
db *sqlx.DB, registryDao store.RegistryRepository, spaceFinder refcache.SpaceFinder,
) store.UpstreamProxyConfigRepository {
return &UpstreamproxyDao{
registryDao: registryDao,
db: db,
spacePathStore: spacePathStore,
registryDao: registryDao,
db: db,
spaceFinder: spaceFinder,
}
}
@ -307,8 +312,16 @@ func (r UpstreamproxyDao) GetAll(
q = q.Where(" AND r.registry_package_type in ? ", packageTypes)
}
q = q.OrderBy(" r.registry_" + sortByField + " " + sortByOrder).Limit(uint64(limit)).Offset(uint64(offset))
if limit > 0 {
ulimit = uint64(limit)
}
if offset > 0 {
uoffset = uint64(offset)
}
q = q.OrderBy(" r.registry_" + sortByField + " " + sortByOrder).
Limit(ulimit).
Offset(uoffset)
sql, args, err := q.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
@ -411,11 +424,11 @@ func (r UpstreamproxyDao) mapToUpstreamProxy(
secretSpacePath := ""
if dst.SecretSpaceID.Valid {
primary, err := r.spacePathStore.FindPrimaryBySpaceID(ctx, int64(dst.SecretSpaceID.Int32))
primary, err := r.spaceFinder.FindByID(ctx, int64(dst.SecretSpaceID.Int32))
if err != nil {
return nil, fmt.Errorf("failed to get secret space path: %w", err)
}
secretSpacePath = primary.Value
secretSpacePath = primary.Path
}
userNameSecretIdentifier := ""
@ -429,11 +442,11 @@ func (r UpstreamproxyDao) mapToUpstreamProxy(
userNameSecretSpacePath := ""
if dst.UserNameSecretSpaceID.Valid {
primary, err := r.spacePathStore.FindPrimaryBySpaceID(ctx, int64(dst.UserNameSecretSpaceID.Int32))
primary, err := r.spaceFinder.FindByID(ctx, int64(dst.UserNameSecretSpaceID.Int32))
if err != nil {
return nil, fmt.Errorf("failed to get secret space path: %w", err)
}
userNameSecretSpacePath = primary.Value
userNameSecretSpacePath = primary.Path
}
return &types.UpstreamProxy{

View File

@ -15,7 +15,7 @@
package database
import (
corestore "github.com/harness/gitness/app/store"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/store/database/dbtx"
@ -26,9 +26,9 @@ import (
func ProvideUpstreamDao(
db *sqlx.DB,
registryDao store.RegistryRepository,
spacePathStore corestore.SpacePathStore,
spaceFinder refcache.SpaceFinder,
) store.UpstreamProxyConfigRepository {
return NewUpstreamproxyDao(db, registryDao, spacePathStore)
return NewUpstreamproxyDao(db, registryDao, spaceFinder)
}
func ProvideRepoDao(db *sqlx.DB, mtRepository store.MediaTypesRepository) store.RegistryRepository {

View File

@ -14,8 +14,11 @@
package config
type contextKey string
// const variables.
const (
PostgresqlDatabase = "postgres"
Sqlite = "sqlite3"
PostgresqlDatabase = "postgres"
Sqlite = "sqlite3"
GoRoutineKey contextKey = "goRoutine"
)