USEFULL
/
drone
mirror of https://github.com/harness/drone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bandwidth and download metrics (#2568) 

* fix
* merge conflict
* change artifact version to text
* review comments
* review comments
* Merge branch 'main' of https://git0.harness.io/l7B_kbSEQD2wjrM7PShm5w/PROD/Harness_Commons/gitness into AH-283-main-fix
* fix
* merge conflict resolved
* fix
* fix
* fix
* Merge branch 'main' of https://git0.harness.io/l7B_kbSEQD2wjrM7PShm5w/PROD/Harness_Commons/gitness into AH-283-main-fix
* download and bandwidth stats
* download and bandwidth stats
* bandwidth and download metrics
CODE-2402
Pragyesh Mishra 2024-09-17 07:15:41 +00:00 committed by Harness
parent 1e644163d3
commit 34513680fb
36 changed files with 1160 additions and 1418 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.down.sql Normal file
View File

@ -0,0 +1,22 @@
CREATE INDEX index_artifact_on_registry_id ON artifacts (artifact_registry_id);
ALTER TABLE artifacts DROP CONSTRAINT fk_images_image_id;
ALTER TABLE artifacts DROP CONSTRAINT unique_artifact_image_id_and_version;
ALTER TABLE artifacts DROP COLUMN artifact_image_id;
ALTER TABLE artifacts DROP COLUMN artifact_version;
ALTER TABLE artifacts ADD COLUMN artifact_name TEXT NOT NULL;
ALTER TABLE artifacts ADD COLUMN artifact_registry_id INTEGER NOT NULL;
ALTER TABLE artifacts ADD COLUMN artifact_labels TEXT;
ALTER TABLE artifacts ADD COLUMN artifact_enabled BOOLEAN DEFAULT FALSE;
ALTER TABLE artifacts ADD CONSTRAINT check_artifact_name_length CHECK ((LENGTH(artifact_name) <= 255));
ALTER TABLE artifacts ADD CONSTRAINT unique_artifact_registry_id_and_name UNIQUE (artifact_registry_id, artifact_name);
ALTER TABLE artifacts ADD CONSTRAINT fk_registries_registry_id FOREIGN KEY (artifact_registry_id)
REFERENCES registries(registry_id) ON DELETE CASCADE;
DROP TABLE images;

34
app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.up.sql Normal file
View File

@ -0,0 +1,34 @@
CREATE TABLE images
(
image_id SERIAL PRIMARY KEY,
image_name TEXT NOT NULL,
image_registry_id INTEGER NOT NULL
CONSTRAINT fk_registries_registry_id
references registries(registry_id),
image_labels TEXT,
image_enabled BOOLEAN DEFAULT FALSE,
image_created_at BIGINT NOT NULL,
image_updated_at BIGINT NOT NULL,
image_created_by INTEGER NOT NULL,
image_updated_by INTEGER NOT NULL,
CONSTRAINT unique_image_registry_id_and_name UNIQUE (image_registry_id, image_name),
CONSTRAINT check_image_name_length CHECK ((LENGTH(image_name) <= 255))
);
DROP INDEX index_artifact_on_registry_id;
ALTER TABLE artifacts DROP CONSTRAINT check_artifact_name_length;
ALTER TABLE artifacts DROP CONSTRAINT unique_artifact_registry_id_and_name;
ALTER TABLE artifacts DROP CONSTRAINT fk_registries_registry_id;
ALTER TABLE artifacts DROP COLUMN artifact_name;
ALTER TABLE artifacts DROP COLUMN artifact_registry_id;
ALTER TABLE artifacts DROP COLUMN artifact_labels;..
ALTER TABLE artifacts DROP COLUMN artifact_enabled;
ALTER TABLE artifacts ADD COLUMN artifact_version TEXT NOT NULL;
ALTER TABLE artifacts ADD COLUMN artifact_image_id INTEGER NOT NULL;
ALTER TABLE artifacts ADD CONSTRAINT fk_images_image_id FOREIGN KEY (artifact_image_id) REFERENCES images(image_id);
ALTER TABLE artifacts ADD CONSTRAINT unique_artifact_image_id_and_version UNIQUE (artifact_image_id, artifact_version);

5
app/store/database/migrate/postgres/0073_create_ar_table_bandwidth_stats_and_download_stats.down.sql Normal file
View File

@ -0,0 +1,5 @@
DROP TABLE bandwidth_stats;
DROP TABLE download_stats;
ALTER TABLE artifact_stats_archive RENAME TO artifact_stats;

29
app/store/database/migrate/postgres/0073_create_ar_table_bandwidth_stats_and_download_stats.up.sql Normal file
View File

@ -0,0 +1,29 @@
CREATE TABLE download_stats
(
download_stat_id SERIAL PRIMARY KEY,
download_stat_artifact_id INTEGER NOT NULL
CONSTRAINT fk_artifacts_artifact_id
REFERENCES artifacts(artifact_id) ,
download_stat_timestamp BIGINT NOT NULL,
download_stat_created_at BIGINT NOT NULL,
download_stat_updated_at BIGINT NOT NULL,
download_stat_created_by INTEGER,
download_stat_updated_by INTEGER
);
CREATE TABLE bandwidth_stats
(
bandwidth_stat_id SERIAL PRIMARY KEY,
bandwidth_stat_image_id INTEGER NOT NULL
CONSTRAINT fk_images_image_id
REFERENCES images(image_id) ,
bandwidth_stat_timestamp BIGINT NOT NULL,
bandwidth_stat_bytes BIGINT NOT NULL,
bandwidth_stat_type TEXT NOT NULL,
bandwidth_stat_created_at BIGINT NOT NULL,
bandwidth_stat_updated_at BIGINT NOT NULL,
bandwidth_stat_created_by INTEGER,
bandwidth_stat_updated_by INTEGER
);
ALTER TABLE artifact_stats RENAME TO artifact_stats_archive;

33
app/store/database/migrate/sqlite/0072_create_ar_table_images_and_alter_table_artifacts.down.sql Normal file
View File

@ -0,0 +1,33 @@
CREATE TABLE artifacts_temp
(
artifact_id INTEGER PRIMARY KEY AUTOINCREMENT,
artifact_name TEXT NOT NULL,
artifact_registry_id INTEGER NOT NULL
CONSTRAINT fk_registries_registry_id
REFERENCES registries(registry_id)
ON DELETE CASCADE,
artifact_labels TEXT,
artifact_enabled BOOLEAN DEFAULT FALSE,
artifact_created_at INTEGER,
artifact_updated_at INTEGER,
artifact_created_by INTEGER,
artifact_updated_by INTEGER,
CONSTRAINT unique_artifact_registry_id_and_name UNIQUE (artifact_registry_id, artifact_name),
CONSTRAINT check_artifact_name_length CHECK ((LENGTH(artifact_name) <= 255))
);
INSERT INTO artifacts_temp (artifact_name, artifact_registry_id, artifact_labels)
SELECT i.image_name AS artifact_name,
i.image_registry_id AS artifact_registry_id,
i.images_labels AS artifact_labels
FROM artifacts a
JOIN images i ON a.artifact_image_id = i.image_id;
DROP TABLE artifacts;
ALTER TABLE artifacts_temp
RENAME TO artifacts;
CREATE INDEX index_artifact_on_registry_id ON artifacts (artifact_registry_id);
DROP TABLE images;

62
app/store/database/migrate/sqlite/0072_create_ar_table_images_and_alter_table_artifacts.up.sql Normal file
View File

@ -0,0 +1,62 @@
CREATE TABLE images
(
image_id INTEGER PRIMARY KEY AUTOINCREMENT,
image_name TEXT NOT NULL,
image_registry_id INTEGER NOT NULL
CONSTRAINT fk_registries_registry_id
REFERENCES registries(registry_id),
image_labels text,
image_enabled BOOLEAN DEFAULT FALSE,
image_created_at INTEGER NOT NULL,
image_updated_at INTEGER NOT NULL,
image_created_by INTEGER NOT NULL,
image_updated_by INTEGER NOT NULL,
CONSTRAINT unique_image_registry_id_and_name UNIQUE (image_registry_id, image_name),
CONSTRAINT check_image_name_length CHECK ((LENGTH(image_name) <= 255))
);
INSERT INTO images (image_name, image_registry_id, image_labels, image_enabled, image_created_at,
image_updated_at, image_created_by, image_updated_by)
SELECT artifact_name AS image_name,
artifact_registry_id AS image_registry_id,
artifact_labels AS image_labels,
artifact_enabled AS image_enabled,
artifact_created_at AS image_created_at,
artifact_updated_at AS image_updated_at,
artifact_created_by AS image_created_by,
artifact_updated_by AS image_updated_by
FROM artifacts;
CREATE TABLE artifacts_temp
(
artifact_id INTEGER PRIMARY KEY AUTOINCREMENT,
artifact_version TEXT NOT NULL,
artifact_image_id INTEGER NOT NULL
CONSTRAINT fk_images_image_id
REFERENCES images(image_id),
artifact_created_at INTEGER NOT NULL,
artifact_updated_at INTEGER NOT NULL,
artifact_created_by INTEGER NOT NULL,
artifact_updated_by INTEGER NOT NULL,
CONSTRAINT unique_artifact_image_id_and_version UNIQUE (artifact_image_id, artifact_version)
);
INSERT INTO artifacts_temp (artifact_image_id, artifact_version, artifact_created_at, artifact_updated_at,
artifact_created_by, artifact_updated_by)
SELECT i.image_id AS artifact_image_id,
m.manifest_digest AS artifact_version,
m.manifest_created_at AS artifact_created_at,
m.manifest_updated_at AS artifact_updated_at,
m.manifest_created_by AS artifact_created_by,
m.manifest_updated_by AS artifact_updated_by
FROM artifacts a
JOIN images i ON a.artifact_name = i.image_name AND a.artifact_registry_id = i.image_registry_id
JOIN manifests m ON a.artifact_name = m.manifest_image_name AND a.artifact_registry_id = m.manifest_registry_id;
DROP INDEX index_artifact_on_registry_id;
DROP TABLE artifacts;
ALTER TABLE artifacts_temp
RENAME TO artifacts;

5
app/store/database/migrate/sqlite/0073_create_ar_table_bandwidth_stats_and_download_stats.down.sql Normal file
View File

@ -0,0 +1,5 @@
DROP TABLE bandwidth_stats;
DROP TABLE download_stats;
ALTER TABLE artifact_stats_archive RENAME TO artifact_stats;

29
app/store/database/migrate/sqlite/0073_create_ar_table_bandwidth_stats_and_download_stats.up.sql Normal file
View File

@ -0,0 +1,29 @@
CREATE TABLE download_stats
(
download_stat_id INTEGER PRIMARY KEY AUTOINCREMENT,
download_stat_artifact_id INTEGER NOT NULL
CONSTRAINT fk_artifacts_artifact_id
REFERENCES artifacts(artifact_id) ,
download_stat_timestamp INTEGER NOT NULL,
download_stat_created_at INTEGER NOT NULL,
download_stat_updated_at INTEGER NOT NULL,
download_stat_created_by INTEGER NOT NULL,
download_stat_updated_by INTEGER NOT NULL
);
CREATE TABLE bandwidth_stats
(
bandwidth_stat_id INTEGER PRIMARY KEY AUTOINCREMENT,
bandwidth_stat_image_id INTEGER NOT NULL
CONSTRAINT fk_images_image_id
REFERENCES images(image_id) ,
bandwidth_stat_timestamp INTEGER NOT NULL,
bandwidth_stat_bytes INTEGER,
bandwidth_stat_type TEXT NOT NULL,
bandwidth_stat_created_at INTEGER NOT NULL,
bandwidth_stat_updated_at INTEGER NOT NULL,
bandwidth_stat_created_by INTEGER NOT NULL,
bandwidth_stat_updated_by INTEGER NOT NULL
);
ALTER TABLE artifact_stats RENAME TO artifact_stats_archive;

10
cmd/gitness/wire_gen.go
View File

@ -434,12 +434,14 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
registryRepository := database2.ProvideRepoDao(db, mediaTypesRepository)
manifestReferenceRepository := database2.ProvideManifestRefDao(db)
tagRepository := database2.ProvideTagDao(db)
imageRepository := database2.ProvideImageDao(db)
artifactRepository := database2.ProvideArtifactDao(db)
artifactStatRepository := database2.ProvideArtifactStatDao(db)
layerRepository := database2.ProvideLayerDao(db, mediaTypesRepository)
manifestService := docker.ManifestServiceProvider(registryRepository, manifestRepository, blobRepository, mediaTypesRepository, manifestReferenceRepository, tagRepository, artifactRepository, artifactStatRepository, layerRepository, gcService, transactor)
manifestService := docker.ManifestServiceProvider(registryRepository, manifestRepository, blobRepository, mediaTypesRepository, manifestReferenceRepository, tagRepository, imageRepository, artifactRepository, layerRepository, gcService, transactor)
registryBlobRepository := database2.ProvideRegistryBlobDao(db)
localRegistry := docker.LocalRegistryProvider(app, manifestService, blobRepository, registryRepository, manifestRepository, registryBlobRepository, mediaTypesRepository, tagRepository, artifactRepository, artifactStatRepository, gcService, transactor)
bandwidthStatRepository := database2.ProvideBandwidthStatDao(db)
downloadStatRepository := database2.ProvideDownloadStatDao(db)
localRegistry := docker.LocalRegistryProvider(app, manifestService, blobRepository, registryRepository, manifestRepository, registryBlobRepository, mediaTypesRepository, tagRepository, imageRepository, artifactRepository, bandwidthStatRepository, downloadStatRepository, gcService, transactor)
upstreamProxyConfigRepository := database2.ProvideUpstreamDao(db, registryRepository, spacePathStore)
secretService := secret3.ProvideSecretService(secretStore, encrypter, spacePathStore)
remoteRegistry := docker.RemoteRegistryProvider(localRegistry, app, upstreamProxyConfigRepository, spacePathStore, secretService)
@ -448,7 +450,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
handler := api2.NewHandlerProvider(dockerController, spaceStore, tokenStore, controller, authenticator, provider, authorizer)
registryOCIHandler := router.OCIHandlerProvider(handler)
cleanupPolicyRepository := database2.ProvideCleanupPolicyDao(db, transactor)
apiHandler := router.APIHandlerProvider(registryRepository, upstreamProxyConfigRepository, tagRepository, manifestRepository, cleanupPolicyRepository, artifactRepository, storageDriver, spaceStore, transactor, authenticator, provider, authorizer, auditService, spacePathStore)
apiHandler := router.APIHandlerProvider(registryRepository, upstreamProxyConfigRepository, tagRepository, manifestRepository, cleanupPolicyRepository, imageRepository, storageDriver, spaceStore, transactor, authenticator, provider, authorizer, auditService, spacePathStore)
appRouter := router.AppRouterProvider(registryOCIHandler, apiHandler)
routerRouter := router2.ProvideRouter(ctx, config, authenticator, repoController, reposettingsController, executionController, logsController, spaceController, pipelineController, secretController, triggerController, connectorController, templateController, pluginController, pullreqController, webhookController, githookController, gitInterface, serviceaccountController, controller, principalController, usergroupController, checkController, systemController, uploadController, keywordsearchController, infraproviderController, gitspaceController, migrateController, aiagentController, capabilitiesController, provider, openapiService, appRouter)
serverServer := server2.ProvideServer(config, routerRouter)

6
registry/app/api/controller/metadata/controller.go
View File

@ -26,7 +26,7 @@ import (
// APIController simple struct.
type APIController struct {
ArtifactStore store.ArtifactRepository
ImageStore store.ImageRepository
RegistryRepository store.RegistryRepository
UpstreamProxyStore store.UpstreamProxyConfigRepository
TagStore store.TagRepository
@ -47,7 +47,7 @@ func NewAPIController(
tagStore store.TagRepository,
manifestStore store.ManifestRepository,
cleanupPolicyStore store.CleanupPolicyRepository,
artifactStore store.ArtifactRepository,
imageStore store.ImageRepository,
driver storagedriver.StorageDriver,
spaceStore corestore.SpaceStore,
tx dbtx.Transactor,
@ -62,7 +62,7 @@ func NewAPIController(
TagStore: tagStore,
ManifestStore: manifestStore,
CleanupPolicyStore: cleanupPolicyStore,
ArtifactStore: artifactStore,
ImageStore: imageStore,
spaceStore: spaceStore,
StorageDriver: driver,
tx: tx,

4
registry/app/api/controller/metadata/get_artifacts_labels.go
View File

@ -58,11 +58,11 @@ func (c *APIController) ListArtifactLabels(
}, nil
}
labels, err := c.ArtifactStore.GetLabelsByParentIDAndRepo(
labels, err := c.ImageStore.GetLabelsByParentIDAndRepo(
ctx, regInfo.parentID,
regInfo.RegistryIdentifier, regInfo.limit, regInfo.offset, regInfo.searchTerm,
)
count, _ := c.ArtifactStore.CountLabelsByParentIDAndRepo(
count, _ := c.ImageStore.CountLabelsByParentIDAndRepo(
ctx, regInfo.parentID,
regInfo.RegistryIdentifier, regInfo.searchTerm,
)

2
registry/app/api/controller/metadata/get_client_setup_details.go
View File

@ -71,7 +71,7 @@ func (c *APIController) GetClientSetupDetails(
}
if imageParam != nil {
_, err := c.ArtifactStore.GetByName(ctx, reg.ID, string(*imageParam))
_, err := c.ImageStore.GetByName(ctx, reg.ID, string(*imageParam))
if err != nil {
return artifact.GetClientSetupDetails404JSONResponse{
NotFoundJSONResponse: artifact.NotFoundJSONResponse(

10
registry/app/api/controller/metadata/update_artifact.go
View File

@ -60,7 +60,7 @@ func (c *APIController) UpdateArtifactLabels(
a := string(r.Artifact)
artifactEntity, err := c.ArtifactStore.GetByRepoAndName(ctx, regInfo.parentID, regInfo.RegistryIdentifier, a)
artifactEntity, err := c.ImageStore.GetByRepoAndName(ctx, regInfo.parentID, regInfo.RegistryIdentifier, a)
if len(artifactEntity.Name) == 0 {
return artifact.UpdateArtifactLabels404JSONResponse{
@ -77,7 +77,7 @@ func (c *APIController) UpdateArtifactLabels(
return throwModifyArtifact400Error(err), nil
}
err = c.ArtifactStore.Update(ctx, existingArtifact)
err = c.ImageStore.Update(ctx, existingArtifact)
if err != nil {
return throwModifyArtifact400Error(err), nil
@ -107,9 +107,9 @@ func throwModifyArtifact400Error(err error) artifact.UpdateArtifactLabels400JSON
func AttachLabels(
dto artifact.ArtifactLabelRequest,
existingArtifact *types.Artifact,
) (*types.Artifact, error) {
return &types.Artifact{
existingArtifact *types.Image,
) (*types.Image, error) {
return &types.Image{
ID: existingArtifact.ID,
RegistryID: existingArtifact.RegistryID,
Name: existingArtifact.Name,

4
registry/app/api/router/harness/route.go
View File

@ -56,7 +56,7 @@ func NewAPIHandler(
tagDao store.TagRepository,
manifestDao store.ManifestRepository,
cleanupPolicyDao store.CleanupPolicyRepository,
artifactDao store.ArtifactRepository,
imageDao store.ImageRepository,
driver storagedriver.StorageDriver,
baseURL string,
spaceStore corestore.SpaceStore,
@ -77,7 +77,7 @@ func NewAPIHandler(
tagDao,
manifestDao,
cleanupPolicyDao,
artifactDao,
imageDao,
driver,
spaceStore,
tx,

4
registry/app/api/router/wire.go
View File

@ -44,7 +44,7 @@ func APIHandlerProvider(
tagDao store.TagRepository,
manifestDao store.ManifestRepository,
cleanupPolicyDao store.CleanupPolicyRepository,
artifactDao store.ArtifactRepository,
imageDao store.ImageRepository,
driver storagedriver.StorageDriver,
spaceStore corestore.SpaceStore,
tx dbtx.Transactor,
@ -60,7 +60,7 @@ func APIHandlerProvider(
tagDao,
manifestDao,
cleanupPolicyDao,
artifactDao,
imageDao,
driver,
config.APIURL,
spaceStore,

185
registry/app/pkg/docker/local.go
View File

@ -34,6 +34,7 @@ import (
"strings"
"time"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/registry/app/dist_temp/dcontext"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
"github.com/harness/gitness/registry/app/manifest"
@ -108,38 +109,43 @@ func NewLocalRegistry(
app *App, ms ManifestService, manifestDao store.ManifestRepository,
registryDao store.RegistryRepository, registryBlobDao store.RegistryBlobRepository,
blobRepo store.BlobRepository, mtRepository store.MediaTypesRepository,
tagDao store.TagRepository, artifactDao store.ArtifactRepository, artifactStatDao store.ArtifactStatRepository,
tagDao store.TagRepository, imageDao store.ImageRepository, artifactDao store.ArtifactRepository,
bandwidthStatDao store.BandwidthStatRepository, downloadStatDao store.DownloadStatRepository,
gcService gc.Service, tx dbtx.Transactor,
) Registry {
return &LocalRegistry{
App: app,
ms: ms,
registryDao: registryDao,
manifestDao: manifestDao,
registryBlobDao: registryBlobDao,
blobRepo: blobRepo,
mtRepository: mtRepository,
tagDao: tagDao,
artifactDao: artifactDao,
artifactStatDao: artifactStatDao,
gcService: gcService,
tx: tx,
App: app,
ms: ms,
registryDao: registryDao,
manifestDao: manifestDao,
registryBlobDao: registryBlobDao,
blobRepo: blobRepo,
mtRepository: mtRepository,
tagDao: tagDao,
imageDao: imageDao,
artifactDao: artifactDao,
bandwidthStatDao: bandwidthStatDao,
downloadStatDao: downloadStatDao,
gcService: gcService,
tx: tx,
}
}
type LocalRegistry struct {
App *App
ms ManifestService
registryDao store.RegistryRepository
manifestDao store.ManifestRepository
registryBlobDao store.RegistryBlobRepository
blobRepo store.BlobRepository
mtRepository store.MediaTypesRepository
tagDao store.TagRepository
artifactDao store.ArtifactRepository
artifactStatDao store.ArtifactStatRepository
gcService gc.Service
tx dbtx.Transactor
App *App
ms ManifestService
registryDao store.RegistryRepository
manifestDao store.ManifestRepository
registryBlobDao store.RegistryBlobRepository
blobRepo store.BlobRepository
mtRepository store.MediaTypesRepository
tagDao store.TagRepository
imageDao store.ImageRepository
artifactDao store.ArtifactRepository
bandwidthStatDao store.BandwidthStatRepository
downloadStatDao store.DownloadStatRepository
gcService gc.Service
tx dbtx.Transactor
}
func (r *LocalRegistry) Base() error {
@ -397,6 +403,20 @@ func (r *LocalRegistry) fetchBlobInternal(
return responseHeaders, nil, -1, nil, "", errs
}
if http.MethodGet == method {
go func(art pkg.RegistryInfo, dgst digest.Digest) {
// Cloning Context.
session, _ := request.AuthSessionFrom(ctx)
ctx3 := request.WithAuthSession(context.Background(), session)
err := r.dbBlobDownloadComplete(ctx3, dgst, info)
if err != nil {
log.Error().Stack().Err(err).Msgf("error while putting bandwidth stat of artifact, %v", err)
return
}
log.Info().Msgf("Successfully updated the bandwidth stat metrics %s", art.Digest)
}(info, dgst)
}
if redirectURL != "" {
return responseHeaders, nil, -1, nil, redirectURL, errs
}
@ -414,7 +434,19 @@ func (r *LocalRegistry) PullManifest(
acceptHeaders []string,
ifNoneMatchHeader []string,
) (responseHeaders *commons.ResponseHeaders, descriptor manifest.Descriptor, manifest manifest.Manifest, errs []error) {
return r.ManifestExist(ctx, artInfo, acceptHeaders, ifNoneMatchHeader)
responseHeaders, descriptor, manifest, errs = r.ManifestExist(ctx, artInfo, acceptHeaders, ifNoneMatchHeader)
go func(art pkg.RegistryInfo) {
// Cloning Context.
session, _ := request.AuthSessionFrom(ctx)
ctx2 := request.WithAuthSession(context.Background(), session)
err := r.dbGetManifestComplete(ctx2, artInfo)
if err != nil {
log.Error().Stack().Err(err).Msgf("error while putting download stat of artifact, %v", err)
return
}
log.Info().Msgf("Successfully updated the download stat metrics %s", art.Digest)
}(artInfo)
return responseHeaders, descriptor, manifest, errs
}
func (r *LocalRegistry) getDigestByTag(ctx context.Context, artInfo pkg.RegistryInfo) (digest.Digest, error) {
@ -1572,6 +1604,89 @@ func (r *LocalRegistry) dbBlobLinkExists(
return nil
}
func (r *LocalRegistry) dbBlobDownloadComplete(
ctx context.Context,
dgst digest.Digest,
info pkg.RegistryInfo,
) error {
err := r.tx.WithTx(
ctx, func(ctx context.Context) error {
registry, err := r.registryDao.GetByParentIDAndName(ctx, info.ParentID, info.RegIdentifier)
if err != nil {
return err
}
blob, err := r.blobRepo.FindByDigestAndRootParentID(ctx, dgst, info.RootParentID)
if err != nil {
return err
}
image, err := r.imageDao.GetByName(ctx, registry.ID, info.Image)
if err != nil {
return err
}
bandwidthStat := &types.BandwidthStat{
ImageID: image.ID,
Type: types.BandwidthTypeDOWNLOAD,
Bytes: blob.Size,
}
if err := r.bandwidthStatDao.Create(ctx, bandwidthStat); err != nil {
return err
}
return nil
}, dbtx.TxDefault,
)
if err != nil {
log.Error().Msgf("failed to put download bandwidth stat in database: %v", err)
return fmt.Errorf("committing database transaction: %w", err)
}
return nil
}
func (r *LocalRegistry) dbGetManifestComplete(
ctx context.Context,
info pkg.RegistryInfo,
) error {
err := r.tx.WithTx(
ctx, func(ctx context.Context) error {
registry, err := r.registryDao.GetByParentIDAndName(ctx, info.ParentID, info.RegIdentifier)
if err != nil {
return err
}
image, err := r.imageDao.GetByName(ctx, registry.ID, info.Image)
if err != nil {
return err
}
artifact, err := r.artifactDao.GetByName(ctx, image.ID, info.Digest)
if err != nil {
return err
}
downloadStat := &types.DownloadStat{
ArtifactID: artifact.ID,
}
if err := r.downloadStatDao.Create(ctx, downloadStat); err != nil {
return err
}
return nil
}, dbtx.TxDefault,
)
if err != nil {
log.Error().Msgf("failed to put download stat in database: %v", err)
return fmt.Errorf("committing database transaction: %w", err)
}
return nil
}
func (r *LocalRegistry) dbPutBlobUploadComplete(
ctx context.Context,
repoName string,
@ -1604,27 +1719,23 @@ func (r *LocalRegistry) dbPutBlobUploadComplete(
return err
}
artifact := &types.Artifact{
image := &types.Image{
Name: info.Image,
RegistryID: registry.ID,
Enabled: false,
}
if err := r.artifactDao.CreateOrUpdate(ctx, artifact); err != nil {
if err := r.imageDao.CreateOrUpdate(ctx, image); err != nil {
return err
}
now := time.Now().UTC()
midnight := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 1, 0, time.UTC)
artifactStat := &types.ArtifactStat{
ArtifactID: artifact.ID,
Date: midnight.UnixMilli(),
UploadBytes: int64(size),
bandwidthStat := &types.BandwidthStat{
ImageID: image.ID,
Type: types.BandwidthTypeUPLOAD,
Bytes: int64(size),
}
if err := r.artifactStatDao.CreateOrUpdate(ctx, artifactStat); err != nil {
if err := r.bandwidthStatDao.Create(ctx, bandwidthStat); err != nil {
return err
}

71
registry/app/pkg/docker/manifest_service.go
View File

@ -44,38 +44,38 @@ import (
)
type manifestService struct {
registryDao store.RegistryRepository
manifestDao store.ManifestRepository
layerDao store.LayerRepository
blobRepo store.BlobRepository
mtRepository store.MediaTypesRepository
tagDao store.TagRepository
artifactDao store.ArtifactRepository
artifactStatDao store.ArtifactStatRepository
manifestRefDao store.ManifestReferenceRepository
gcService gc.Service
tx dbtx.Transactor
registryDao store.RegistryRepository
manifestDao store.ManifestRepository
layerDao store.LayerRepository
blobRepo store.BlobRepository
mtRepository store.MediaTypesRepository
tagDao store.TagRepository
imageDao store.ImageRepository
artifactDao store.ArtifactRepository
manifestRefDao store.ManifestReferenceRepository
gcService gc.Service
tx dbtx.Transactor
}
func NewManifestService(
registryDao store.RegistryRepository, manifestDao store.ManifestRepository,
blobRepo store.BlobRepository, mtRepository store.MediaTypesRepository, tagDao store.TagRepository,
artifactDao store.ArtifactRepository, artifactStatDao store.ArtifactStatRepository,
imageDao store.ImageRepository, artifactDao store.ArtifactRepository,
layerDao store.LayerRepository, manifestRefDao store.ManifestReferenceRepository,
tx dbtx.Transactor, gcService gc.Service,
) ManifestService {
return &manifestService{
registryDao: registryDao,
manifestDao: manifestDao,
layerDao: layerDao,
blobRepo: blobRepo,
mtRepository: mtRepository,
tagDao: tagDao,
artifactDao: artifactDao,
artifactStatDao: artifactStatDao,
manifestRefDao: manifestRefDao,
gcService: gcService,
tx: tx,
registryDao: registryDao,
manifestDao: manifestDao,
layerDao: layerDao,
blobRepo: blobRepo,
mtRepository: mtRepository,
tagDao: tagDao,
artifactDao: artifactDao,
imageDao: imageDao,
manifestRefDao: manifestRefDao,
gcService: gcService,
tx: tx,
}
}
@ -201,12 +201,21 @@ func (l *manifestService) dbTagManifest(
return err
}
artifact := &types.Artifact{
image := &types.Image{
Name: imageName,
RegistryID: dbRepo.ID,
Enabled: true,
}
if err := l.imageDao.CreateOrUpdate(ctx, image); err != nil {
return err
}
artifact := &types.Artifact{
ImageID: image.ID,
Version: string(dgst),
}
if err := l.artifactDao.CreateOrUpdate(ctx, artifact); err != nil {
return err
}
@ -222,23 +231,9 @@ func (l *manifestService) dbTagManifest(
return err
}
a, err := l.artifactDao.GetByName(ctx, dbRepo.ID, imageName)
if err != nil {
return err
}
now := time.Now().UTC()
midnight := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 1, 0, time.UTC)
artifactStat := &types.ArtifactStat{
ArtifactID: a.ID,
Date: midnight.UnixMilli(),
DownloadCount: 1,
}
if err := l.artifactStatDao.CreateOrUpdate(ctx, artifactStat); err != nil {
return err
}
return nil
},

13
registry/app/pkg/docker/wire.go
View File

@ -34,25 +34,26 @@ func LocalRegistryProvider(
registryDao store.RegistryRepository, manifestDao store.ManifestRepository,
registryBlobDao store.RegistryBlobRepository,
mtRepository store.MediaTypesRepository,
tagDao store.TagRepository, artifactDao store.ArtifactRepository, artifactStatDao store.ArtifactStatRepository,
tagDao store.TagRepository, imageDao store.ImageRepository, artifactDao store.ArtifactRepository,
bandwidthStatDao store.BandwidthStatRepository, downloadStatDao store.DownloadStatRepository,
gcService gc.Service, tx dbtx.Transactor,
) *LocalRegistry {
return NewLocalRegistry(
app, ms, manifestDao, registryDao, registryBlobDao, blobRepo,
mtRepository, tagDao, artifactDao, artifactStatDao, gcService, tx,
mtRepository, tagDao, imageDao, artifactDao, bandwidthStatDao, downloadStatDao, gcService, tx,
).(*LocalRegistry)
}
func ManifestServiceProvider(
registryDao store.RegistryRepository,
manifestDao store.ManifestRepository, blobRepo store.BlobRepository, mtRepository store.MediaTypesRepository,
manifestRefDao store.ManifestReferenceRepository, tagDao store.TagRepository, artifactDao store.ArtifactRepository,
artifactStatDao store.ArtifactStatRepository, layerDao store.LayerRepository,
manifestRefDao store.ManifestReferenceRepository, tagDao store.TagRepository, imageDao store.ImageRepository,
artifactDao store.ArtifactRepository, layerDao store.LayerRepository,
gcService gc.Service, tx dbtx.Transactor,
) ManifestService {
return NewManifestService(
registryDao, manifestDao, blobRepo, mtRepository, tagDao,
artifactDao, artifactStatDao, layerDao, manifestRefDao, tx, gcService,
registryDao, manifestDao, blobRepo, mtRepository, tagDao, imageDao,
artifactDao, layerDao, manifestRefDao, tx, gcService,
)
}

31
registry/app/store/database.go
View File

@ -382,14 +382,14 @@ type RegistryBlobRepository interface {
) (bool, error)
}
type ArtifactRepository interface {
type ImageRepository interface {
// Get an Artifact specified by ID
Get(ctx context.Context, id int64) (*types.Artifact, error)
Get(ctx context.Context, id int64) (*types.Image, error)
// Get an Artifact specified by Artifact Name
GetByName(
ctx context.Context, repoID int64,
ctx context.Context, registryID int64,
name string,
) (*types.Artifact, error)
) (*types.Image, error)
// Get the Labels specified by Parent ID and Repo
GetLabelsByParentIDAndRepo(
ctx context.Context, parentID int64,
@ -405,17 +405,26 @@ type ArtifactRepository interface {
GetByRepoAndName(
ctx context.Context, parentID int64,
repo string, name string,
) (*types.Artifact, error)
// Get the Labels specified by Parent ID
GetLabelsByParentID(ctx context.Context, parentID int64) (labels []string, err error)
) (*types.Image, error)
// Create an Artifact
CreateOrUpdate(ctx context.Context, artifact *types.Artifact) error
CreateOrUpdate(ctx context.Context, image *types.Image) error
// Update an Artifact
Update(ctx context.Context, artifact *types.Artifact) (err error)
Update(ctx context.Context, artifact *types.Image) (err error)
}
type ArtifactStatRepository interface {
CreateOrUpdate(ctx context.Context, artifactStat *types.ArtifactStat) error
type ArtifactRepository interface {
// Get an Artifact specified by ID
GetByName(ctx context.Context, imageID int64, version string) (*types.Artifact, error)
// Create an Artifact
CreateOrUpdate(ctx context.Context, artifact *types.Artifact) error
}
type DownloadStatRepository interface {
Create(ctx context.Context, downloadStat *types.DownloadStat) error
}
type BandwidthStatRepository interface {
Create(ctx context.Context, bandwidthStat *types.BandwidthStat) error
}
type GCBlobTaskRepository interface {

254
registry/app/store/database/artifact.go
View File

@ -17,14 +17,12 @@ package database
import (
"context"
"database/sql"
"sort"
"time"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/registry/app/store/database/util"
"github.com/harness/gitness/registry/types"
gitness_store "github.com/harness/gitness/store"
databaseg "github.com/harness/gitness/store/database"
"github.com/harness/gitness/store/database/dbtx"
@ -43,25 +41,20 @@ func NewArtifactDao(db *sqlx.DB) store.ArtifactRepository {
}
type artifactDB struct {
ID int64 `db:"artifact_id"`
Name string `db:"artifact_name"`
RegistryID int64 `db:"artifact_registry_id"`
Labels sql.NullString `db:"artifact_labels"`
Enabled bool `db:"artifact_enabled"`
CreatedAt int64 `db:"artifact_created_at"`
UpdatedAt int64 `db:"artifact_updated_at"`
CreatedBy int64 `db:"artifact_created_by"`
UpdatedBy int64 `db:"artifact_updated_by"`
ID int64 `db:"artifact_id"`
Version string `db:"artifact_version"`
ImageID int64 `db:"artifact_image_id"`
CreatedAt int64 `db:"artifact_created_at"`
UpdatedAt int64 `db:"artifact_updated_at"`
CreatedBy int64 `db:"artifact_created_by"`
UpdatedBy int64 `db:"artifact_updated_by"`
}
type artifactLabelDB struct {
Labels sql.NullString `db:"labels"`
}
func (a ArtifactDao) Get(ctx context.Context, id int64) (*types.Artifact, error) {
func (a ArtifactDao) GetByName(ctx context.Context, imageID int64,
version string) (*types.Artifact, error) {
q := databaseg.Builder.Select(util.ArrToStringByDelimiter(util.GetDBTagsFromStruct(artifactDB{}), ",")).
From("artifacts").
Where("artifact_id = ?", id)
Where("artifact_image_id = ? AND artifact_version = ?", imageID, version)
sql, args, err := q.ToSql()
if err != nil {
@ -77,149 +70,25 @@ func (a ArtifactDao) Get(ctx context.Context, id int64) (*types.Artifact, error)
return a.mapToArtifact(ctx, dst)
}
func (a ArtifactDao) GetByRepoAndName(ctx context.Context, parentID int64,
repo string, name string) (*types.Artifact, error) {
q := databaseg.Builder.Select("a.artifact_id, a.artifact_name, "+
" a.artifact_registry_id, a.artifact_labels, a.artifact_created_at, "+
" a.artifact_updated_at, a.artifact_created_by, a.artifact_updated_by").
From("artifacts a").
Join(" registries r ON r.registry_id = a.artifact_registry_id").
Where("r.registry_parent_id = ? AND r.registry_name = ? AND a.artifact_name = ?",
parentID, repo, name)
sql, args, err := q.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
}
db := dbtx.GetAccessor(ctx, a.db)
dst := new(artifactDB)
if err = db.GetContext(ctx, dst, sql, args...); err != nil {
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed to get artifact")
}
return a.mapToArtifact(ctx, dst)
}
func (a ArtifactDao) GetByName(ctx context.Context, repoID int64, name string) (*types.Artifact, error) {
q := databaseg.Builder.Select(util.ArrToStringByDelimiter(util.GetDBTagsFromStruct(artifactDB{}), ",")).
From("artifacts").
Where("artifact_registry_id = ? AND artifact_name = ?", repoID, name)
sql, args, err := q.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
}
db := dbtx.GetAccessor(ctx, a.db)
dst := new(artifactDB)
if err = db.GetContext(ctx, dst, sql, args...); err != nil {
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed to get artifact")
}
return a.mapToArtifact(ctx, dst)
}
func (a ArtifactDao) GetLabelsByParentIDAndRepo(ctx context.Context, parentID int64, repo string,
limit int, offset int, search string) (labels []string, err error) {
q := databaseg.Builder.Select("a.artifact_labels as labels").
From("artifacts a").
Join("registries r ON r.registry_id = a.artifact_registry_id").
Where("r.registry_parent_id = ? AND r.registry_name = ?", parentID, repo)
if search != "" {
q = q.Where("a.artifact_labels LIKE ?", "%"+search+"%")
}
q = q.OrderBy("a.artifact_labels ASC").Limit(uint64(limit)).Offset(uint64(offset))
sql, args, err := q.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
}
dst := []*artifactLabelDB{}
db := dbtx.GetAccessor(ctx, a.db)
if err = db.SelectContext(ctx, &dst, sql, args...); err != nil {
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed to get artifact labels")
}
return a.mapToArtifactLabels(dst), nil
}
func (a ArtifactDao) CountLabelsByParentIDAndRepo(ctx context.Context, parentID int64, repo,
search string) (count int64, err error) {
q := databaseg.Builder.Select("a.artifact_labels as labels").
From("artifacts a").
Join("registries r ON r.registry_id = a.artifact_registry_id").
Where("r.registry_parent_id = ? AND r.registry_name = ?", parentID, repo)
if search != "" {
q = q.Where("a.artifact_labels LIKE ?", "%"+search+"%")
}
sql, args, err := q.ToSql()
if err != nil {
return -1, errors.Wrap(err, "Failed to convert query to sql")
}
db := dbtx.GetAccessor(ctx, a.db)
dst := []*artifactLabelDB{}
if err = db.SelectContext(ctx, &dst, sql, args...); err != nil {
return -1, databaseg.ProcessSQLErrorf(ctx, err, "Failed to get artifact labels")
}
return int64(len(dst)), nil
}
func (a ArtifactDao) GetLabelsByParentID(ctx context.Context, parentID int64) (labels []string, err error) {
q := databaseg.Builder.Select("a.artifact_labels as labels").
From("artifacts a").
Join("registries r ON r.registry_id = a.artifact_registry_id").
Where("r.registry_parent_id = ?", parentID)
sql, args, err := q.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
}
db := dbtx.GetAccessor(ctx, a.db)
dst := []*artifactLabelDB{}
if err = db.SelectContext(ctx, &dst, sql, args...); err != nil {
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed to get artifact labels")
}
return a.mapToArtifactLabels(dst), nil
}
func (a ArtifactDao) CreateOrUpdate(ctx context.Context, artifact *types.Artifact) error {
const sqlQuery = `
INSERT INTO artifacts (
artifact_registry_id
,artifact_name
,artifact_enabled
artifact_image_id
,artifact_version
,artifact_created_at
,artifact_updated_at
,artifact_created_by
,artifact_updated_by
) VALUES (
:artifact_registry_id
,:artifact_name
,:artifact_enabled
:artifact_image_id
,:artifact_version
,:artifact_created_at
,:artifact_updated_at
,:artifact_created_by
,:artifact_updated_by
)
ON CONFLICT (artifact_registry_id, artifact_name)
DO UPDATE SET
artifact_enabled = :artifact_enabled
ON CONFLICT (artifact_image_id, artifact_version)
DO NOTHING
RETURNING artifact_id`
db := dbtx.GetAccessor(ctx, a.db)
@ -234,39 +103,6 @@ func (a ArtifactDao) CreateOrUpdate(ctx context.Context, artifact *types.Artifac
return nil
}
func (a ArtifactDao) Update(ctx context.Context, artifact *types.Artifact) (err error) {
var sqlQuery = " UPDATE artifacts SET " + util.GetSetDBKeys(artifactDB{}, "artifact_id") +
" WHERE artifact_id = :artifact_id "
dbArtifact := a.mapToInternalArtifact(ctx, artifact)
// update Version (used for optimistic locking) and Updated time
dbArtifact.UpdatedAt = time.Now().UnixMilli()
db := dbtx.GetAccessor(ctx, a.db)
query, arg, err := db.BindNamed(sqlQuery, dbArtifact)
if err != nil {
return databaseg.ProcessSQLErrorf(ctx, err, "Failed to bind artifact object")
}
result, err := db.ExecContext(ctx, query, arg...)
if err != nil {
return databaseg.ProcessSQLErrorf(ctx, err, "Failed to update artifact")
}
count, err := result.RowsAffected()
if err != nil {
return databaseg.ProcessSQLErrorf(ctx, err, "Failed to get number of updated rows")
}
if count == 0 {
return gitness_store.ErrVersionConflict
}
return nil
}
func (a ArtifactDao) mapToInternalArtifact(ctx context.Context, in *types.Artifact) *artifactDB {
session, _ := request.AuthSessionFrom(ctx)
@ -280,18 +116,14 @@ func (a ArtifactDao) mapToInternalArtifact(ctx context.Context, in *types.Artifa
in.UpdatedAt = time.Now()
in.UpdatedBy = session.Principal.ID
sort.Strings(in.Labels)
return &artifactDB{
ID: in.ID,
Name: in.Name,
RegistryID: in.RegistryID,
Labels: util.GetEmptySQLString(util.ArrToString(in.Labels)),
Enabled: in.Enabled,
CreatedAt: in.CreatedAt.UnixMilli(),
UpdatedAt: in.UpdatedAt.UnixMilli(),
CreatedBy: in.CreatedBy,
UpdatedBy: in.UpdatedBy,
ID: in.ID,
Version: in.Version,
ImageID: in.ImageID,
CreatedAt: in.CreatedAt.UnixMilli(),
UpdatedAt: in.UpdatedAt.UnixMilli(),
CreatedBy: in.CreatedBy,
UpdatedBy: in.UpdatedBy,
}
}
@ -299,38 +131,12 @@ func (a ArtifactDao) mapToArtifact(_ context.Context, dst *artifactDB) (*types.A
createdBy := dst.CreatedBy
updatedBy := dst.UpdatedBy
return &types.Artifact{
ID: dst.ID,
Name: dst.Name,
RegistryID: dst.RegistryID,
Labels: util.StringToArr(dst.Labels.String),
Enabled: dst.Enabled,
CreatedAt: time.UnixMilli(dst.CreatedAt),
UpdatedAt: time.UnixMilli(dst.UpdatedAt),
CreatedBy: createdBy,
UpdatedBy: updatedBy,
ID: dst.ID,
Version: dst.Version,
ImageID: dst.ImageID,
CreatedAt: time.UnixMilli(dst.CreatedAt),
UpdatedAt: time.UnixMilli(dst.UpdatedAt),
CreatedBy: createdBy,
UpdatedBy: updatedBy,
}, nil
}
func (a ArtifactDao) mapToArtifactLabels(dst []*artifactLabelDB) []string {
elements := make(map[string]bool)
res := []string{}
for _, labels := range dst {
elements, res = a.mapToArtifactLabel(elements, res, labels)
}
return res
}
func (a ArtifactDao) mapToArtifactLabel(elements map[string]bool, res []string,
dst *artifactLabelDB) (map[string]bool, []string) {
if dst == nil {
return elements, res
}
labels := util.StringToArr(dst.Labels.String)
for _, label := range labels {
if !elements[label] {
elements[label] = true
res = append(res, label)
}
}
return elements, res
}

126
registry/app/store/database/artifact_stat.go
View File

@ -1,126 +0,0 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package database
import (
"context"
"database/sql"
"errors"
"time"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/registry/types"
databaseg "github.com/harness/gitness/store/database"
"github.com/harness/gitness/store/database/dbtx"
"github.com/jmoiron/sqlx"
)
type ArtifactStatDao struct {
db *sqlx.DB
}
func NewArtifactStatDao(db *sqlx.DB) store.ArtifactStatRepository {
return &ArtifactStatDao{
db: db,
}
}
type artifactStatDB struct {
ID int64 `db:"artifact_stat_id"`
ArtifactID int64 `db:"artifact_stat_artifact_id"`
Date int64 `db:"artifact_stat_date"`
DownloadCount int64 `db:"artifact_stat_download_count"`
UploadBytes int64 `db:"artifact_stat_upload_bytes"`
DownloadBytes int64 `db:"artifact_stat_download_bytes"`
CreatedAt int64 `db:"artifact_stat_created_at"`
UpdatedAt int64 `db:"artifact_stat_updated_at"`
CreatedBy int64 `db:"artifact_stat_created_by"`
UpdatedBy int64 `db:"artifact_stat_updated_by"`
}
func (a ArtifactStatDao) CreateOrUpdate(ctx context.Context, artifactStat *types.ArtifactStat) error {
const sqlQuery = `
INSERT INTO artifact_stats (
artifact_stat_artifact_id
,artifact_stat_date
,artifact_stat_download_count
,artifact_stat_upload_bytes
,artifact_stat_download_bytes
,artifact_stat_created_at
,artifact_stat_updated_at
,artifact_stat_created_by
,artifact_stat_updated_by
) VALUES (
:artifact_stat_artifact_id
,:artifact_stat_date
,:artifact_stat_download_count
,:artifact_stat_upload_bytes
,:artifact_stat_download_bytes
,:artifact_stat_created_at
,:artifact_stat_updated_at
,:artifact_stat_created_by
,:artifact_stat_updated_by
)
ON CONFLICT (artifact_stat_artifact_id, artifact_stat_date)
DO UPDATE SET
artifact_stat_download_count =
artifact_stats.artifact_stat_download_count + EXCLUDED.artifact_stat_download_count,
artifact_stat_upload_bytes =
artifact_stats.artifact_stat_upload_bytes + EXCLUDED.artifact_stat_upload_bytes,
artifact_stat_download_bytes =
artifact_stats.artifact_stat_download_bytes + EXCLUDED.artifact_stat_download_bytes
RETURNING artifact_stat_id`
db := dbtx.GetAccessor(ctx, a.db)
query, arg, err := db.BindNamed(sqlQuery, a.mapToInternalArtifactStat(ctx, artifactStat))
if err != nil {
return databaseg.ProcessSQLErrorf(ctx, err, "Failed to bind artifact object")
}
if err = db.QueryRowContext(ctx, query,
arg...).Scan(&artifactStat.ID); err != nil && !errors.Is(err, sql.ErrNoRows) {
return databaseg.ProcessSQLErrorf(ctx, err, "Insert query failed")
}
return nil
}
func (a ArtifactStatDao) mapToInternalArtifactStat(ctx context.Context, in *types.ArtifactStat) *artifactStatDB {
session, _ := request.AuthSessionFrom(ctx)
if in.CreatedAt.IsZero() {
in.CreatedAt = time.Now()
}
if in.CreatedBy == 0 {
in.CreatedBy = session.Principal.ID
}
in.UpdatedAt = time.Now()
return &artifactStatDB{
ID: in.ID,
ArtifactID: in.ArtifactID,
Date: in.Date,
DownloadCount: in.DownloadCount,
UploadBytes: in.UploadBytes,
DownloadBytes: in.DownloadBytes,
CreatedAt: in.CreatedAt.UnixMilli(),
UpdatedAt: in.UpdatedAt.UnixMilli(),
CreatedBy: in.CreatedBy,
UpdatedBy: session.Principal.ID,
}
}

114
registry/app/store/database/bandwidth_stat.go Normal file
View File

@ -0,0 +1,114 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package database
import (
"context"
"database/sql"
"errors"
"time"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/registry/types"
databaseg "github.com/harness/gitness/store/database"
"github.com/harness/gitness/store/database/dbtx"
"github.com/jmoiron/sqlx"
)
type BandwidthStatDao struct {
db *sqlx.DB
}
func NewBandwidthStatDao(db *sqlx.DB) store.BandwidthStatRepository {
return &BandwidthStatDao{
db: db,
}
}
type bandwidthStatDB struct {
ID int64 `db:"bandwidth_stat_id"`
ImageID int64 `db:"bandwidth_stat_image_id"`
Timestamp int64 `db:"bandwidth_stat_timestamp"`
Type types.BandwidthType `db:"bandwidth_stat_type"`
Bytes int64 `db:"bandwidth_stat_bytes"`
CreatedAt int64 `db:"bandwidth_stat_created_at"`
UpdatedAt int64 `db:"bandwidth_stat_updated_at"`
CreatedBy int64 `db:"bandwidth_stat_created_by"`
UpdatedBy int64 `db:"bandwidth_stat_updated_by"`
}
func (b BandwidthStatDao) Create(ctx context.Context, bandwidthStat *types.BandwidthStat) error {
const sqlQuery = `
INSERT INTO bandwidth_stats (
bandwidth_stat_image_id
,bandwidth_stat_timestamp
,bandwidth_stat_type
,bandwidth_stat_bytes
,bandwidth_stat_created_at
,bandwidth_stat_updated_at
,bandwidth_stat_created_by
,bandwidth_stat_updated_by
) VALUES (
:bandwidth_stat_image_id
,:bandwidth_stat_timestamp
,:bandwidth_stat_type
,:bandwidth_stat_bytes
,:bandwidth_stat_created_at
,:bandwidth_stat_updated_at
,:bandwidth_stat_created_by
,:bandwidth_stat_updated_by
)
RETURNING bandwidth_stat_id`
db := dbtx.GetAccessor(ctx, b.db)
query, arg, err := db.BindNamed(sqlQuery, b.mapToInternalBandwidthStat(ctx, bandwidthStat))
if err != nil {
return databaseg.ProcessSQLErrorf(ctx, err, "Failed to bind bandwidth stat object")
}
if err = db.QueryRowContext(ctx, query,
arg...).Scan(&bandwidthStat.ID); err != nil && !errors.Is(err, sql.ErrNoRows) {
return databaseg.ProcessSQLErrorf(ctx, err, "Insert query failed")
}
return nil
}
func (b BandwidthStatDao) mapToInternalBandwidthStat(ctx context.Context,
in *types.BandwidthStat) *bandwidthStatDB {
session, _ := request.AuthSessionFrom(ctx)
if in.CreatedAt.IsZero() {
in.CreatedAt = time.Now()
}
if in.CreatedBy == 0 {
in.CreatedBy = session.Principal.ID
}
in.UpdatedAt = time.Now()
return &bandwidthStatDB{
ID: in.ID,
ImageID: in.ImageID,
Timestamp: time.Now().UnixMilli(),
Type: in.Type,
Bytes: in.Bytes,
CreatedAt: in.CreatedAt.UnixMilli(),
UpdatedAt: in.UpdatedAt.UnixMilli(),
CreatedBy: in.CreatedBy,
UpdatedBy: session.Principal.ID,
}
}

106
registry/app/store/database/download_stat.go Normal file
View File

@ -0,0 +1,106 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package database
import (
"context"
"database/sql"
"errors"
"time"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/registry/types"
databaseg "github.com/harness/gitness/store/database"
"github.com/harness/gitness/store/database/dbtx"
"github.com/jmoiron/sqlx"
)
type DownloadStatDao struct {
db *sqlx.DB
}
func NewDownloadStatDao(db *sqlx.DB) store.DownloadStatRepository {
return &DownloadStatDao{
db: db,
}
}
type downloadStatDB struct {
ID int64 `db:"download_stat_id"`
ArtifactID int64 `db:"download_stat_artifact_id"`
Timestamp int64 `db:"download_stat_timestamp"`
CreatedAt int64 `db:"download_stat_created_at"`
UpdatedAt int64 `db:"download_stat_updated_at"`
CreatedBy int64 `db:"download_stat_created_by"`
UpdatedBy int64 `db:"download_stat_updated_by"`
}
func (d DownloadStatDao) Create(ctx context.Context, downloadStat *types.DownloadStat) error {
const sqlQuery = `
INSERT INTO download_stats (
download_stat_artifact_id
,download_stat_timestamp
,download_stat_created_at
,download_stat_updated_at
,download_stat_created_by
,download_stat_updated_by
) VALUES (
:download_stat_artifact_id
,:download_stat_timestamp
,:download_stat_created_at
,:download_stat_updated_at
,:download_stat_created_by
,:download_stat_updated_by
)
RETURNING download_stat_id`
db := dbtx.GetAccessor(ctx, d.db)
query, arg, err := db.BindNamed(sqlQuery, d.mapToInternalDownloadStat(ctx, downloadStat))
if err != nil {
return databaseg.ProcessSQLErrorf(ctx, err, "Failed to bind download stat object")
}
if err = db.QueryRowContext(ctx, query,
arg...).Scan(&downloadStat.ID); err != nil && !errors.Is(err, sql.ErrNoRows) {
return databaseg.ProcessSQLErrorf(ctx, err, "Insert query failed")
}
return nil
}
func (d DownloadStatDao) mapToInternalDownloadStat(ctx context.Context,
in *types.DownloadStat) *downloadStatDB {
session, _ := request.AuthSessionFrom(ctx)
if in.CreatedAt.IsZero() {
in.CreatedAt = time.Now()
}
if in.CreatedBy == 0 {
in.CreatedBy = session.Principal.ID
}
in.UpdatedAt = time.Now()
return &downloadStatDB{
ID: in.ID,
ArtifactID: in.ArtifactID,
Timestamp: time.Now().UnixMilli(),
CreatedAt: in.CreatedAt.UnixMilli(),
UpdatedAt: in.UpdatedAt.UnixMilli(),
CreatedBy: in.CreatedBy,
UpdatedBy: session.Principal.ID,
}
}

314
registry/app/store/database/image.go Normal file
View File

@ -0,0 +1,314 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package database
import (
"context"
"database/sql"
"sort"
"time"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/registry/app/store/database/util"
"github.com/harness/gitness/registry/types"
gitness_store "github.com/harness/gitness/store"
databaseg "github.com/harness/gitness/store/database"
"github.com/harness/gitness/store/database/dbtx"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
)
type ImageDao struct {
db *sqlx.DB
}
func NewImageDao(db *sqlx.DB) store.ImageRepository {
return &ImageDao{
db: db,
}
}
type imageDB struct {
ID int64 `db:"image_id"`
Name string `db:"image_name"`
RegistryID int64 `db:"image_registry_id"`
Labels sql.NullString `db:"image_labels"`
Enabled bool `db:"image_enabled"`
CreatedAt int64 `db:"image_created_at"`
UpdatedAt int64 `db:"image_updated_at"`
CreatedBy int64 `db:"image_created_by"`
UpdatedBy int64 `db:"image_updated_by"`
}
type imageLabelDB struct {
Labels sql.NullString `db:"labels"`
}
func (i ImageDao) Get(ctx context.Context, id int64) (*types.Image, error) {
q := databaseg.Builder.Select(util.ArrToStringByDelimiter(util.GetDBTagsFromStruct(imageDB{}), ",")).
From("images").
Where("image_id = ?", id)
sql, args, err := q.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
}
db := dbtx.GetAccessor(ctx, i.db)
dst := new(imageDB)
if err = db.GetContext(ctx, dst, sql, args...); err != nil {
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed to get image")
}
return i.mapToImage(ctx, dst)
}
func (i ImageDao) GetByName(ctx context.Context, registryID int64, name string) (*types.Image, error) {
q := databaseg.Builder.Select(util.ArrToStringByDelimiter(util.GetDBTagsFromStruct(imageDB{}), ",")).
From("images").
Where("image_registry_id = ? AND image_name = ?", registryID, name)
sql, args, err := q.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
}
db := dbtx.GetAccessor(ctx, i.db)
dst := new(imageDB)
if err = db.GetContext(ctx, dst, sql, args...); err != nil {
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed to get image")
}
return i.mapToImage(ctx, dst)
}
func (i ImageDao) CreateOrUpdate(ctx context.Context, image *types.Image) error {
const sqlQuery = `
INSERT INTO images (
image_registry_id
,image_name
,image_enabled
,image_created_at
,image_updated_at
,image_created_by
,image_updated_by
) VALUES (
:image_registry_id
,:image_name
,:image_enabled
,:image_created_at
,:image_updated_at
,:image_created_by
,:image_updated_by
)
ON CONFLICT (image_registry_id, image_name)
DO UPDATE SET
image_enabled = :image_enabled
RETURNING image_id`
db := dbtx.GetAccessor(ctx, i.db)
query, arg, err := db.BindNamed(sqlQuery, i.mapToInternalImage(ctx, image))
if err != nil {
return databaseg.ProcessSQLErrorf(ctx, err, "Failed to bind image object")
}
if err = db.QueryRowContext(ctx, query, arg...).Scan(&image.ID); err != nil && !errors.Is(err, sql.ErrNoRows) {
return databaseg.ProcessSQLErrorf(ctx, err, "Insert query failed")
}
return nil
}
func (i ImageDao) GetLabelsByParentIDAndRepo(ctx context.Context, parentID int64, repo string,
limit int, offset int, search string) (labels []string, err error) {
q := databaseg.Builder.Select("a.image_labels as labels").
From("images a").
Join("registries r ON r.registry_id = a.image_registry_id").
Where("r.registry_parent_id = ? AND r.registry_name = ?", parentID, repo)
if search != "" {
q = q.Where("a.image_labels LIKE ?", "%"+search+"%")
}
q = q.OrderBy("a.image_labels ASC").Limit(uint64(limit)).Offset(uint64(offset))
sql, args, err := q.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
}
dst := []*imageLabelDB{}
db := dbtx.GetAccessor(ctx, i.db)
if err = db.SelectContext(ctx, &dst, sql, args...); err != nil {
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed to get artifact labels")
}
return i.mapToImageLabels(dst), nil
}
func (i ImageDao) CountLabelsByParentIDAndRepo(ctx context.Context, parentID int64, repo,
search string) (count int64, err error) {
q := databaseg.Builder.Select("a.image_labels as labels").
From("images a").
Join("registries r ON r.registry_id = a.image_registry_id").
Where("r.registry_parent_id = ? AND r.registry_name = ?", parentID, repo)
if search != "" {
q = q.Where("a.image_labels LIKE ?", "%"+search+"%")
}
sql, args, err := q.ToSql()
if err != nil {
return -1, errors.Wrap(err, "Failed to convert query to sql")
}
db := dbtx.GetAccessor(ctx, i.db)
dst := []*imageLabelDB{}
if err = db.SelectContext(ctx, &dst, sql, args...); err != nil {
return -1, databaseg.ProcessSQLErrorf(ctx, err, "Failed to get artifact labels")
}
return int64(len(dst)), nil
}
func (i ImageDao) GetByRepoAndName(ctx context.Context, parentID int64,
repo string, name string) (*types.Image, error) {
q := databaseg.Builder.Select("a.image_id, a.image_name, "+
" a.image_registry_id, a.image_labels, a.image_created_at, "+
" a.image_updated_at, a.image_created_by, a.image_updated_by").
From("images a").
Join(" registries r ON r.registry_id = a.image_registry_id").
Where("r.registry_parent_id = ? AND r.registry_name = ? AND a.image_name = ?",
parentID, repo, name)
sql, args, err := q.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
}
db := dbtx.GetAccessor(ctx, i.db)
dst := new(imageDB)
if err = db.GetContext(ctx, dst, sql, args...); err != nil {
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed to get artifact")
}
return i.mapToImage(ctx, dst)
}
func (i ImageDao) Update(ctx context.Context, image *types.Image) (err error) {
var sqlQuery = " UPDATE images SET " + util.GetSetDBKeys(imageDB{}, "image_id") +
" WHERE image_id = :image_id "
dbImage := i.mapToInternalImage(ctx, image)
// update Version (used for optimistic locking) and Updated time
dbImage.UpdatedAt = time.Now().UnixMilli()
db := dbtx.GetAccessor(ctx, i.db)
query, arg, err := db.BindNamed(sqlQuery, dbImage)
if err != nil {
return databaseg.ProcessSQLErrorf(ctx, err, "Failed to bind images object")
}
result, err := db.ExecContext(ctx, query, arg...)
if err != nil {
return databaseg.ProcessSQLErrorf(ctx, err, "Failed to update images")
}
count, err := result.RowsAffected()
if err != nil {
return databaseg.ProcessSQLErrorf(ctx, err, "Failed to get number of updated rows")
}
if count == 0 {
return gitness_store.ErrVersionConflict
}
return nil
}
func (i ImageDao) mapToInternalImage(ctx context.Context, in *types.Image) *imageDB {
session, _ := request.AuthSessionFrom(ctx)
if in.CreatedAt.IsZero() {
in.CreatedAt = time.Now()
}
if in.CreatedBy == 0 {
in.CreatedBy = session.Principal.ID
}
in.UpdatedAt = time.Now()
in.UpdatedBy = session.Principal.ID
sort.Strings(in.Labels)
return &imageDB{
ID: in.ID,
Name: in.Name,
RegistryID: in.RegistryID,
Labels: util.GetEmptySQLString(util.ArrToString(in.Labels)),
Enabled: in.Enabled,
CreatedAt: in.CreatedAt.UnixMilli(),
UpdatedAt: in.UpdatedAt.UnixMilli(),
CreatedBy: in.CreatedBy,
UpdatedBy: in.UpdatedBy,
}
}
func (i ImageDao) mapToImage(_ context.Context, dst *imageDB) (*types.Image, error) {
createdBy := dst.CreatedBy
updatedBy := dst.UpdatedBy
return &types.Image{
ID: dst.ID,
Name: dst.Name,
RegistryID: dst.RegistryID,
Labels: util.StringToArr(dst.Labels.String),
Enabled: dst.Enabled,
CreatedAt: time.UnixMilli(dst.CreatedAt),
UpdatedAt: time.UnixMilli(dst.UpdatedAt),
CreatedBy: createdBy,
UpdatedBy: updatedBy,
}, nil
}
func (i ImageDao) mapToImageLabels(dst []*imageLabelDB) []string {
elements := make(map[string]bool)
res := []string{}
for _, labels := range dst {
elements, res = i.mapToImageLabel(elements, res, labels)
}
return res
}
func (i ImageDao) mapToImageLabel(elements map[string]bool, res []string,
dst *imageLabelDB) (map[string]bool, []string) {
if dst == nil {
return elements, res
}
labels := util.StringToArr(dst.Labels.String)
for _, label := range labels {
if !elements[label] {
elements[label] = true
res = append(res, label)
}
}
return elements, res
}

21
registry/app/store/database/registry.go
View File

@ -223,9 +223,9 @@ func (r registryDao) GetAll(
From("registries r").
LeftJoin("upstream_proxy_configs u ON r.registry_id = u.upstream_proxy_config_registry_id").
LeftJoin(
"(SELECT r.registry_id, count(a.artifact_id) as artifact_count FROM"+
" registries r LEFT JOIN artifacts a ON r.registry_id = a.artifact_registry_id"+
" WHERE r.registry_parent_id = ? AND a.artifact_enabled = true GROUP BY r.registry_id ) as t2"+
"(SELECT r.registry_id, count(a.image_id) as artifact_count FROM"+
" registries r LEFT JOIN images a ON r.registry_id = a.image_registry_id"+
" WHERE r.registry_parent_id = ? AND a.image_enabled = true GROUP BY r.registry_id ) as t2"+
" ON r.registry_id = t2.registry_id ", parentID,
).
LeftJoin(
@ -235,15 +235,12 @@ func (r registryDao) GetAll(
"GROUP BY r.registry_id) as t3 ON r.registry_id = t3.registry_id ", parentID,
).
LeftJoin(
"(SELECT b.artifact_registry_id as registry_id,"+
" sum(COALESCE(a.artifact_stat_download_count,0)) as"+
" download_count FROM artifact_stats a "+
" LEFT JOIN artifacts b"+
" ON a.artifact_stat_artifact_id = b.artifact_id LEFT JOIN registries"+
" c ON b.artifact_registry_id = c.registry_id"+
" WHERE c.registry_parent_id = ? AND b.artifact_enabled = true GROUP BY b.artifact_registry_id)"+
" as t4 ON r.registry_id = t4.registry_id", parentID,
).
"(SELECT i.image_registry_id, COUNT(d.download_stat_id) as download_count "+
"FROM artifacts a "+
" JOIN images i on i.image_id = a.artifact_image_id"+
" JOIN download_stats d ON d.download_stat_artifact_id = a.artifact_id"+
" WHERE i.image_enabled = true GROUP BY i.image_registry_id ) as t4 "+
" ON r.registry_id = t4.image_registry_id").
Where("r.registry_parent_id = ?", parentID)
if search != "" {

67
registry/app/store/database/tag.go
View File

@ -348,7 +348,7 @@ func (t tagDao) GetAllArtifactsByParentID(
q := databaseg.Builder.Select(
"r.registry_name as repo_name, t.tag_image_name as name,"+
" r.registry_package_type as package_type, t.tag_name as latest_version,"+
" t.tag_updated_at as modified_at, ar.artifact_labels as labels, t2.download_count as download_count ",
" t.tag_updated_at as modified_at, ar.image_labels as labels, t2.download_count as download_count ",
).
From("tags t").
Join(
@ -359,16 +359,19 @@ func (t tagDao) GetAllArtifactsByParentID(
).
Join("registries r ON t.tag_registry_id = r.registry_id").
Join(
"artifacts ar ON ar.artifact_registry_id = t.tag_registry_id AND"+
" ar.artifact_name = t.tag_image_name",
"images ar ON ar.image_registry_id = t.tag_registry_id AND"+
" ar.image_name = t.tag_image_name",
).
LeftJoin(
"(SELECT b.artifact_name as artifact_name, COALESCE(sum(a.artifact_stat_download_count),0) as"+
" download_count FROM artifact_stats a LEFT JOIN artifacts b"+
" ON a.artifact_stat_artifact_id = b.artifact_id LEFT JOIN registries c"+
" ON b.artifact_registry_id = c.registry_id"+
" WHERE c.registry_parent_id = ? AND b.artifact_enabled = true GROUP BY b.artifact_name) as t2"+
" ON t.tag_image_name = t2.artifact_name", parentID,
"(SELECT i.image_name, t1.download_count FROM"+
" ( SELECT a.artifact_image_id, COUNT(d.download_stat_id) as download_count"+
" FROM artifacts a "+
" LEFT JOIN download_stats d ON d.download_stat_artifact_id = a.artifact_id GROUP BY"+
" a.artifact_image_id ) as t1 "+
" JOIN images ON i.image_id = t1.artifact_image_id "+
" JOIN registries r ON r.registry_id = i.image_registry_id "+
" WHERE r.registry_parent_id = ? GROUP BY i.image_name) as t2"+
" ON t.tag_image_name = t2.image_name", parentID,
).
Where("a.rank = 1")
@ -381,7 +384,7 @@ func (t tagDao) GetAllArtifactsByParentID(
labelsVal := util.GetEmptySQLString(util.ArrToString(labels))
labelsVal.String = labelSeparatorStart + labelsVal.String + labelSeparatorEnd
q = q.Where("'^_' || ar.artifact_labels || '^_' LIKE ?", labelsVal)
q = q.Where("'^_' || ar.image_labels || '^_' LIKE ?", labelsVal)
}
if search != "" {
@ -419,8 +422,8 @@ func (t tagDao) CountAllArtifactsByParentID(
).
Join("registries r ON t.tag_registry_id = r.registry_id").
Join(
"artifacts ar ON ar.artifact_registry_id = t.tag_registry_id" +
" AND ar.artifact_name = t.tag_image_name",
"images ar ON ar.image_registry_id = t.tag_registry_id" +
" AND ar.image_name = t.tag_image_name",
).
Where("a.rank = 1 ")
@ -436,7 +439,7 @@ func (t tagDao) CountAllArtifactsByParentID(
sort.Strings(labels)
labelsVal := util.GetEmptySQLString(util.ArrToString(labels))
labelsVal.String = labelSeparatorStart + labelsVal.String + labelSeparatorEnd
q = q.Where("'^_' || ar.artifact_labels || '^_' LIKE ?", labelsVal)
q = q.Where("'^_' || ar.image_labels || '^_' LIKE ?", labelsVal)
}
sql, args, err := q.ToSql()
@ -494,14 +497,15 @@ func (t tagDao) GetLatestTagMetadata(
"r.registry_name as repo_name,"+
" r.registry_package_type as package_type, t.tag_image_name as name, "+
"t.tag_name as latest_version, t.tag_created_at as created_at,"+
" t.tag_updated_at as modified_at, ar.artifact_labels as labels",
" t.tag_updated_at as modified_at, ar.image_labels as labels",
).
From("tags t").
Join("registries r ON t.tag_registry_id = r.registry_id").
Join(
"artifacts ar ON ar.artifact_registry_id = t.tag_registry_id "+
"AND ar.artifact_name = t.tag_image_name",
"images ar ON ar.image_registry_id = t.tag_registry_id "+
"AND ar.image_name = t.tag_image_name",
).
LeftJoin("(SELECT downlad)").
Where(
"r.registry_parent_id = ? AND r.registry_name = ?"+
" AND t.tag_image_name = ?", parentID, repoKey, imageName,
@ -617,7 +621,8 @@ func (t tagDao) GetAllArtifactsByRepo(
q := databaseg.Builder.Select(
"r.registry_name as repo_name, t.tag_image_name as name,"+
" r.registry_package_type as package_type, t.tag_name as latest_version,"+
" t.tag_updated_at as modified_at, ar.artifact_labels as labels, t2.download_count ",
" t.tag_updated_at as modified_at, ar.image_labels as labels, "+
" COALESCE(t2.download_count, 0) as download_count ",
).
From("tags t").
Join(
@ -629,17 +634,19 @@ func (t tagDao) GetAllArtifactsByRepo(
).
Join("registries r ON t.tag_registry_id = r.registry_id").
Join(
"artifacts ar ON ar.artifact_registry_id = t.tag_registry_id"+
" AND ar.artifact_name = t.tag_image_name",
"images ar ON ar.image_registry_id = t.tag_registry_id"+
" AND ar.image_name = t.tag_image_name",
).
LeftJoin(
"(SELECT b.artifact_name as artifact_name, COALESCE(sum(a.artifact_stat_download_count),0) as"+
" download_count FROM artifact_stats a LEFT JOIN artifacts b"+
" ON a.artifact_stat_artifact_id = b.artifact_id LEFT JOIN registries c"+
" ON b.artifact_registry_id = c.registry_id"+
" WHERE c.registry_parent_id = ? AND c.registry_name = ? AND b.artifact_enabled = true"+
" GROUP BY b.artifact_name) as t2"+
" ON t.tag_image_name = t2.artifact_name", parentID, repoKey,
"( SELECT i.image_name, COALESCE(t1.download_count, 0) as download_count FROM"+
" ( SELECT a.artifact_image_id, COUNT(d.download_stat_id) as download_count"+
" FROM artifacts a "+
" JOIN download_stats d ON d.download_stat_artifact_id = a.artifact_id GROUP BY"+
" a.artifact_image_id ) as t1 "+
" JOIN images i ON i.image_id = t1.artifact_image_id "+
" JOIN registries r ON r.registry_id = i.image_registry_id "+
" WHERE r.registry_parent_id = ? AND r.registry_name = ? GROUP BY i.image_name) as t2"+
" ON t.tag_image_name = t2.image_name", parentID, repoKey,
).
Where("a.rank = 1 ")
@ -651,7 +658,7 @@ func (t tagDao) GetAllArtifactsByRepo(
sort.Strings(labels)
labelsVal := util.GetEmptySQLString(util.ArrToString(labels))
labelsVal.String = labelSeparatorStart + labelsVal.String + labelSeparatorEnd
q = q.Where("'^_' || ar.artifact_labels || '^_' LIKE ?", labelsVal)
q = q.Where("'^_' || ar.image_labels || '^_' LIKE ?", labelsVal)
}
q = q.OrderBy("t.tag_" + sortByField + " " + sortByOrder).Limit(uint64(limit)).Offset(uint64(offset))
@ -684,8 +691,8 @@ func (t tagDao) CountAllArtifactsByRepo(
).
Join("registries r ON t.tag_registry_id = r.registry_id").
Join(
"artifacts ar ON ar.artifact_registry_id = t.tag_registry_id AND" +
" ar.artifact_name = t.tag_image_name",
"images ar ON ar.image_registry_id = t.tag_registry_id AND" +
" ar.image_name = t.tag_image_name",
).
Where("a.rank = 1 ")
@ -697,7 +704,7 @@ func (t tagDao) CountAllArtifactsByRepo(
sort.Strings(labels)
labelsVal := util.GetEmptySQLString(util.ArrToString(labels))
labelsVal.String = labelSeparatorStart + labelsVal.String + labelSeparatorEnd
q = q.Where("'^_' || ar.artifact_labels || '^_' LIKE ?", labelsVal)
q = q.Where("'^_' || ar.image_labels || '^_' LIKE ?", labelsVal)
}
sql, args, err := q.ToSql()

16
registry/app/store/database/wire.go
View File

@ -47,12 +47,20 @@ func ProvideRegistryBlobDao(db *sqlx.DB) store.RegistryBlobRepository {
return NewRegistryBlobDao(db)
}
func ProvideImageDao(db *sqlx.DB) store.ImageRepository {
return NewImageDao(db)
}
func ProvideArtifactDao(db *sqlx.DB) store.ArtifactRepository {
return NewArtifactDao(db)
}
func ProvideArtifactStatDao(db *sqlx.DB) store.ArtifactStatRepository {
return NewArtifactStatDao(db)
func ProvideDownloadStatDao(db *sqlx.DB) store.DownloadStatRepository {
return NewDownloadStatDao(db)
}
func ProvideBandwidthStatDao(db *sqlx.DB) store.BandwidthStatRepository {
return NewBandwidthStatDao(db)
}
func ProvideTagDao(db *sqlx.DB) store.TagRepository {
@ -86,6 +94,8 @@ var WireSet = wire.NewSet(
ProvideCleanupPolicyDao,
ProvideManifestRefDao,
ProvideLayerDao,
ProvideImageDao,
ProvideArtifactDao,
ProvideArtifactStatDao,
ProvideDownloadStatDao,
ProvideBandwidthStatDao,
)

0
registry/app/store/migrations/files/postgres/000001_init.down.sql
View File

561
registry/app/store/migrations/files/postgres/000001_init.up.sql
View File

@ -1,561 +0,0 @@
create table registries
(
registry_id SERIAL primary key,
registry_name text not null
constraint registry_name_len_check
check (length(registry_name) <= 255),
registry_root_parent_id INTEGER not null,
registry_parent_id INTEGER not null,
registry_description text,
registry_type text not null,
registry_package_type text not null,
registry_upstream_proxies text,
registry_allowed_pattern text,
registry_blocked_pattern text,
registry_created_at BIGINT not null,
registry_updated_at BIGINT not null,
registry_created_by INTEGER not null,
registry_updated_by INTEGER not null,
registry_labels text,
constraint unique_registries
unique (registry_root_parent_id, registry_name)
);
create table media_types
(
mt_id SERIAL primary key,
mt_media_type text not null
constraint unique_media_types_type
unique,
mt_created_at BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM now()) * 1000)::BIGINT
);
create table blobs
(
blob_id SERIAL primary key,
blob_root_parent_id INTEGER not null,
blob_digest bytea not null,
blob_media_type_id INTEGER not null
constraint fk_blobs_media_type_id_media_types
references media_types(mt_id),
blob_size BIGINT not null,
blob_created_at BIGINT not null,
blob_created_by INTEGER not null,
constraint unique_digest_root_parent_id unique (blob_digest, blob_root_parent_id)
);
create index index_blobs_on_media_type_id
on blobs (blob_media_type_id);
create table registry_blobs
(
rblob_id SERIAL primary key,
rblob_registry_id INTEGER not null
constraint fk_registry_blobs_rpstry_id_registries
references registries
on delete cascade,
rblob_blob_id INTEGER not null
constraint fk_registry_blobs_blob_id_blobs
references blobs
on delete cascade,
rblob_image_name text
constraint registry_blobs_image_len_check
check (length(rblob_image_name) <= 255),
rblob_created_at BIGINT not null,
rblob_updated_at BIGINT not null,
rblob_created_by INTEGER not null,
rblob_updated_by INTEGER not null,
constraint unique_registry_blobs_registry_id_blob_id_image
unique (rblob_registry_id, rblob_blob_id, rblob_image_name)
);
create index index_registry_blobs_on_reg_id
on registry_blobs (rblob_registry_id);
create index index_registry_blobs_on_reg_blob_id
on registry_blobs (rblob_registry_id, rblob_blob_id);
create table manifests
(
manifest_id SERIAL primary key,
manifest_registry_id INTEGER not null
constraint fk_manifests_registry_id_registries
references registries(registry_id)
on delete cascade,
manifest_schema_version smallint not null,
manifest_media_type_id INTEGER not null
constraint fk_manifests_media_type_id_media_types
references media_types(mt_id),
manifest_artifact_media_type text,
manifest_total_size BIGINT not null,
manifest_configuration_media_type text,
manifest_configuration_payload bytea,
manifest_configuration_blob_id INTEGER
constraint fk_manifests_configuration_blob_id_blobs
references blobs(blob_id),
manifest_configuration_digest bytea,
manifest_digest bytea not null,
manifest_payload bytea not null,
manifest_non_conformant boolean default false,
manifest_non_distributable_layers boolean default false,
manifest_subject_id INTEGER,
manifest_subject_digest bytea,
manifest_annotations bytea,
manifest_image_name text not null
constraint manifests_img_name_len_check
check (length(manifest_image_name) <= 255),
manifest_created_at BIGINT not null,
manifest_created_by INTEGER not null,
manifest_updated_at BIGINT not null,
manifest_updated_by INTEGER not null,
constraint unique_manifests_registry_id_image_name_and_digest
unique (manifest_registry_id, manifest_image_name, manifest_digest),
constraint unique_manifests_registry_id_id_cfg_blob_id
unique (manifest_registry_id, manifest_id, manifest_configuration_blob_id),
constraint fk_manifests_subject_id_manifests
foreign key (manifest_subject_id) references manifests
on delete cascade
);
create index index_manifests_on_media_type_id
on manifests (manifest_media_type_id);
create index index_manifests_on_configuration_blob_id
on manifests (manifest_configuration_blob_id);
create table manifest_references
(
manifest_ref_id SERIAL primary key,
manifest_ref_registry_id INTEGER not null,
manifest_ref_parent_id INTEGER not null,
manifest_ref_child_id INTEGER not null,
manifest_ref_created_at BIGINT not null,
manifest_ref_updated_at BIGINT not null,
manifest_ref_created_by INTEGER not null,
manifest_ref_updated_by INTEGER not null,
constraint unique_manifest_references_prt_id_chd_id
unique (manifest_ref_registry_id, manifest_ref_parent_id, manifest_ref_child_id),
constraint fk_manifest_references_parent_id_mnfsts
foreign key (manifest_ref_parent_id) references manifests
on delete cascade,
constraint fk_manifest_references_child_id_mnfsts
foreign key (manifest_ref_child_id) references manifests,
constraint check_manifest_references_parent_id_and_child_id_differ
check (manifest_ref_parent_id <> manifest_ref_child_id)
);
create index index_manifest_references_on_rpstry_id_child_id
on manifest_references (manifest_ref_registry_id, manifest_ref_child_id);
create table layers
(
layer_id SERIAL primary key,
layer_registry_id INTEGER not null,
layer_manifest_id INTEGER not null,
layer_media_type_id INTEGER not null
constraint fk_layer_media_type_id_media_types
references media_types,
layer_blob_id INTEGER not null
constraint fk_layer_blob_id_blobs
references blobs,
layer_size BIGINT not null,
layer_created_at BIGINT not null,
layer_updated_at BIGINT not null,
layer_created_by INTEGER not null,
layer_updated_by INTEGER not null,
constraint unique_layer_rpstry_id_and_mnfst_id_and_blob_id
unique (layer_registry_id, layer_manifest_id, layer_blob_id),
constraint unique_layer_rpstry_id_and_id_and_blob_id
unique (layer_registry_id, layer_id, layer_blob_id),
constraint fk_manifst_id_manifests
foreign key (layer_manifest_id) references manifests(manifest_id)
on delete cascade
);
create index index_layer_on_media_type_id
on layers (layer_media_type_id);
create index index_layer_on_blob_id
on layers (layer_blob_id);
create table artifacts
(
artifact_id SERIAL primary key,
artifact_name text not null,
artifact_registry_id INTEGER not null
constraint fk_registries_registry_id
references registries(registry_id)
on delete cascade,
artifact_labels text,
artifact_enabled boolean default false,
artifact_created_at BIGINT,
artifact_updated_at BIGINT,
artifact_created_by INTEGER,
artifact_updated_by INTEGER,
constraint unique_artifact_registry_id_and_name unique (artifact_registry_id, artifact_name),
constraint check_artifact_name_length check ((char_length(artifact_name) <= 255))
);
create index index_artifact_on_registry_id ON artifacts USING btree (artifact_registry_id);
create table artifact_stats
(
artifact_stat_id SERIAL primary key,
artifact_stat_artifact_id INTEGER not null
constraint fk_artifacts_artifact_id
references artifacts(artifact_id) on delete cascade,
artifact_stat_date BIGINT,
artifact_stat_download_count BIGINT,
artifact_stat_upload_bytes BIGINT,
artifact_stat_download_bytes BIGINT,
artifact_stat_created_at BIGINT not null,
artifact_stat_updated_at BIGINT not null,
artifact_stat_created_by INTEGER not null,
artifact_stat_updated_by INTEGER not null,
constraint unique_artifact_stats_artifact_id_and_date unique (artifact_stat_artifact_id, artifact_stat_date)
);
create table tags
(
tag_id SERIAL primary key,
tag_name text not null
constraint tag_name_len_check
check (char_length(tag_name) <= 128),
tag_image_name text not null
constraint tag_img_name_len_check
check (length(tag_image_name) <= 255),
tag_registry_id INTEGER not null,
tag_manifest_id INTEGER not null,
tag_created_at BIGINT,
tag_updated_at BIGINT,
tag_created_by INTEGER,
tag_updated_by INTEGER,
constraint fk_tag_manifest_id_manifests FOREIGN KEY
(tag_manifest_id) REFERENCES manifests (manifest_id) ON DELETE CASCADE,
constraint unique_tag_registry_id_and_name_and_image_name
unique (tag_registry_id, tag_name, tag_image_name)
);
create index index_tag_on_rpository_id_and_manifest_id
on tags (tag_registry_id, tag_manifest_id);
create table upstream_proxy_configs
(
upstream_proxy_config_id SERIAL primary key,
upstream_proxy_config_registry_id INTEGER not null
constraint fk_upstream_proxy_config_registry_id
references registries
on delete cascade,
upstream_proxy_config_source text,
upstream_proxy_config_url text,
upstream_proxy_config_auth_type text not null,
upstream_proxy_config_user_name text,
upstream_proxy_config_secret_identifier text,
upstream_proxy_config_secret_space_id INTEGER,
constraint fk_layers_secret_identifier_and_secret_space_id
foreign key (upstream_proxy_config_secret_identifier, upstream_proxy_config_secret_space_id)
references secrets(secret_uid, secret_space_id)
on delete cascade,
upstream_proxy_config_token text,
upstream_proxy_config_created_at BIGINT,
upstream_proxy_config_updated_at BIGINT,
upstream_proxy_config_created_by INTEGER,
upstream_proxy_config_updated_by INTEGER
);
create index index_upstream_proxy_config_on_registry_id
on upstream_proxy_configs (upstream_proxy_config_registry_id);
create table cleanup_policies
(
cp_id SERIAL primary key,
cp_registry_id INTEGER not null
constraint fk_cleanup_policies_registry_id
references registries ON DELETE CASCADE,
cp_name text,
cp_expiry_time_ms BIGINT,
cp_created_at BIGINT not null,
cp_updated_at BIGINT not null,
cp_created_by INTEGER not null,
cp_updated_by INTEGER not null
);
create index index_cleanup_policies_on_registry_id
on cleanup_policies (cp_registry_id);
create table cleanup_policy_prefix_mappings
(
cpp_id SERIAL primary key,
cpp_cleanup_policy_id INTEGER not null
constraint fk_cleanup_policies_id
references cleanup_policies(cp_id) ON DELETE CASCADE,
cpp_prefix text not null,
cpp_prefix_type text not null
);
create index index_cleanup_policy_map_on_policy_id
on cleanup_policy_prefix_mappings (cpp_cleanup_policy_id);
insert into media_types (mt_media_type)
values ('application/vnd.docker.distribution.manifest.v1+json'),
('application/vnd.docker.distribution.manifest.v1+prettyjws'),
('application/vnd.docker.distribution.manifest.v2+json'),
('application/vnd.docker.distribution.manifest.list.v2+json'),
('application/vnd.docker.image.rootfs.diff.tar'),
('application/vnd.docker.image.rootfs.diff.tar.gzip'),
('application/vnd.docker.image.rootfs.foreign.diff.tar.gzip'),
('application/vnd.docker.container.image.v1+json'),
('application/vnd.docker.container.image.rootfs.diff+x-gtar'),
('application/vnd.docker.plugin.v1+json'),
('application/vnd.oci.image.layer.v1.tar'),
('application/vnd.oci.image.layer.v1.tar+gzip'),
('application/vnd.oci.image.layer.v1.tar+zstd'),
('application/vnd.oci.image.layer.nondistributable.v1.tar'),
('application/vnd.oci.image.layer.nondistributable.v1.tar+gzip'),
('application/vnd.oci.image.config.v1+json'),
('application/vnd.oci.image.manifest.v1+json'),
('application/vnd.oci.image.index.v1+json'),
('application/vnd.cncf.helm.config.v1+json'),
('application/tar+gzip'),
('application/octet-stream'),
('application/vnd.buildkit.cacheconfig.v0'),
('application/vnd.cncf.helm.chart.content.v1.tar+gzip'),
('application/vnd.cncf.helm.chart.provenance.v1.prov');
CREATE TABLE gc_blob_review_queue
(
blob_id INTEGER NOT NULL,
review_after BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM (NOW() + INTERVAL '1 day'))),
review_count INTEGER NOT NULL DEFAULT 0,
created_at BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
event text NOT NULL,
CONSTRAINT pk_gc_blob_review_queue primary key (blob_id)
);
CREATE INDEX index_gc_blob_review_queue_on_review_after ON gc_blob_review_queue USING btree (review_after);
CREATE TABLE gc_review_after_defaults
(
event text NOT NULL,
value interval NOT NULL,
CONSTRAINT pk_gc_review_after_defaults PRIMARY KEY (event),
CONSTRAINT check_gc_review_after_defaults_event_length CHECK ((char_length(event) <= 255))
);
INSERT INTO gc_review_after_defaults (event, value)
VALUES ('blob_upload', interval '1 day'),
('manifest_upload', interval '1 day'),
('manifest_delete', interval '1 day'),
('layer_delete', interval '1 day'),
('manifest_list_delete', interval '1 day'),
('tag_delete', interval '1 day'),
('tag_switch', interval '1 day')
ON CONFLICT (event)
DO NOTHING;
CREATE TABLE gc_manifest_review_queue
(
registry_id INTEGER NOT NULL,
manifest_id INTEGER NOT NULL,
review_after BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM (NOW() + INTERVAL '1 day'))),
review_count INTEGER NOT NULL DEFAULT 0,
created_at BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
event text NOT NULL,
CONSTRAINT pk_gc_manifest_review_queue PRIMARY KEY (registry_id, manifest_id),
CONSTRAINT fk_gc_manifest_review_queue_rp_id_mfst_id_mnfsts FOREIGN KEY (manifest_id) REFERENCES manifests (manifest_id) ON DELETE CASCADE
);
CREATE INDEX index_gc_manifest_review_queue_on_review_after ON gc_manifest_review_queue USING btree (review_after);
CREATE OR REPLACE FUNCTION gc_review_after(e text)
RETURNS BIGINT
VOLATILE
AS
$$
DECLARE
result timestamp WITH time zone;
BEGIN
SELECT (now() + value)
INTO result
FROM gc_review_after_defaults
WHERE event = e;
IF result IS NULL THEN
RETURN EXTRACT(EPOCH FROM (now() + interval '1 day'));
ELSE
RETURN EXTRACT(EPOCH FROM result);
END IF;
END;
$$
LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION gc_track_blob_uploads()
RETURNS TRIGGER
AS
$$
BEGIN
INSERT INTO gc_blob_review_queue (blob_id, review_after, event)
VALUES (NEW.blob_id, gc_review_after('blob_upload'), 'blob_upload')
ON CONFLICT (blob_id)
DO UPDATE SET review_after = gc_review_after('blob_upload'),
event = 'blob_upload';
RETURN NULL;
END;
$$
LANGUAGE plpgsql;
CREATE TRIGGER gc_track_blob_uploads_trigger
AFTER INSERT
ON blobs
FOR EACH ROW
EXECUTE PROCEDURE public.gc_track_blob_uploads();
CREATE OR REPLACE FUNCTION gc_track_manifest_uploads()
RETURNS TRIGGER
AS
$$
BEGIN
INSERT INTO gc_manifest_review_queue (registry_id, manifest_id, review_after, event)
VALUES (NEW.manifest_registry_id, NEW.manifest_id, gc_review_after('manifest_upload'), 'manifest_upload');
RETURN NULL;
END;
$$
LANGUAGE plpgsql;
CREATE TRIGGER gc_track_manifest_uploads_trigger
AFTER INSERT
ON manifests
FOR EACH ROW
EXECUTE PROCEDURE gc_track_manifest_uploads();
CREATE OR REPLACE FUNCTION gc_track_deleted_manifests()
RETURNS TRIGGER
AS
$$
BEGIN
IF OLD.manifest_configuration_blob_id IS NOT NULL THEN -- not all manifests have a configuration
INSERT INTO gc_blob_review_queue (blob_id, review_after, event)
VALUES (OLD.manifest_configuration_blob_id, gc_review_after('manifest_delete'), 'manifest_delete')
ON CONFLICT (blob_id)
DO UPDATE SET
review_after = gc_review_after('manifest_delete'),
event = 'manifest_delete';
END IF;
RETURN NULL;
END;
$$
LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION gc_track_deleted_layers()
RETURNS TRIGGER
AS
$$
BEGIN
IF (TG_LEVEL = 'STATEMENT') THEN
INSERT INTO gc_blob_review_queue (blob_id, review_after, event)
SELECT deleted_rows.layer_blob_id,
gc_review_after('layer_delete'),
'layer_delete'
FROM old_table deleted_rows
JOIN
blobs b ON deleted_rows.layer_blob_id = b.blob_id
ORDER BY deleted_rows.layer_blob_id ASC
ON CONFLICT (blob_id)
DO UPDATE SET review_after = gc_review_after('layer_delete'),
event = 'layer_delete';
ELSIF (TG_LEVEL = 'ROW') THEN
INSERT INTO gc_blob_review_queue (blob_id, review_after, event)
VALUES (OLD.blob_id, gc_review_after('layer_delete'), 'layer_delete')
ON CONFLICT (blob_id)
DO UPDATE SET review_after = gc_review_after('layer_delete'),
event = 'layer_delete';
END IF;
RETURN NULL;
END;
$$
LANGUAGE plpgsql;
CREATE TRIGGER gc_track_deleted_manifests_trigger
AFTER DELETE
ON manifests
FOR EACH ROW
EXECUTE PROCEDURE gc_track_deleted_manifests();
CREATE TRIGGER gc_track_deleted_layers_trigger
AFTER DELETE
ON layers
REFERENCING OLD TABLE AS old_table
FOR EACH STATEMENT
EXECUTE FUNCTION gc_track_deleted_layers();
CREATE OR REPLACE FUNCTION gc_track_deleted_manifest_lists()
RETURNS TRIGGER
AS
$$
BEGIN
INSERT INTO gc_manifest_review_queue (registry_id, manifest_id, review_after, event)
VALUES (OLD.manifest_ref_registry_id, OLD.manifest_ref_child_id, gc_review_after('manifest_list_delete'), 'manifest_list_delete')
ON CONFLICT (registry_id, manifest_id)
DO UPDATE SET review_after = gc_review_after('manifest_list_delete'),
event = 'manifest_list_delete';
RETURN NULL;
END;
$$
LANGUAGE plpgsql;
CREATE TRIGGER gc_track_deleted_manifest_lists_trigger
AFTER DELETE
ON manifest_references
FOR EACH ROW
EXECUTE PROCEDURE gc_track_deleted_manifest_lists();
CREATE OR REPLACE FUNCTION gc_track_deleted_tags()
RETURNS TRIGGER
AS
$$
BEGIN
IF EXISTS (SELECT 1
FROM manifests
WHERE manifest_registry_id = OLD.tag_registry_id
AND manifest_id = OLD.tag_registry_id) THEN
INSERT INTO gc_manifest_review_queue (registry_id, manifest_id, review_after, event)
VALUES (OLD.tag_registry_id, OLD.tag_manifest_id, gc_review_after('tag_delete'), 'tag_delete')
ON CONFLICT (registry_id, manifest_id)
DO UPDATE SET review_after = gc_review_after('tag_delete'),
event = 'tag_delete';
END IF;
RETURN NULL;
END;
$$
LANGUAGE plpgsql;
CREATE TRIGGER gc_track_deleted_tag_trigger
AFTER DELETE
ON tags
FOR EACH ROW
EXECUTE PROCEDURE gc_track_deleted_tags();
CREATE OR REPLACE FUNCTION gc_track_switched_tags()
RETURNS TRIGGER
AS
$$
BEGIN
INSERT INTO gc_manifest_review_queue (registry_id, manifest_id, review_after, event)
VALUES (OLD.tag_registry_id, OLD.tag_manifest_id, gc_review_after('tag_switch'), 'tag_switch')
ON CONFLICT (registry_id, manifest_id)
DO UPDATE SET review_after = gc_review_after('tag_switch'),
event = 'tag_switch';
RETURN NULL;
END;
$$
LANGUAGE plpgsql;
CREATE TRIGGER gc_track_switched_tag_trigger
AFTER UPDATE OF tag_manifest_id
ON tags
FOR EACH ROW
EXECUTE PROCEDURE gc_track_switched_tags();

0
registry/app/store/migrations/files/sqlite/000001_init.down.sql
View File

330
registry/app/store/migrations/files/sqlite/000001_init.up.sql
View File

@ -1,330 +0,0 @@
create table registries
(
registry_id INTEGER PRIMARY KEY AUTOINCREMENT,
registry_name text not null
constraint registry_name_len_check
check (length(registry_name) <= 255),
registry_root_parent_id INTEGER not null,
registry_parent_id INTEGER not null,
registry_description text,
registry_type text not null,
registry_package_type text not null,
registry_upstream_proxies text,
registry_allowed_pattern text,
registry_blocked_pattern text,
registry_labels text,
registry_created_at INTEGER not null,
registry_updated_at INTEGER not null,
registry_created_by INTEGER not null,
registry_updated_by INTEGER not null,
constraint unique_registries
unique (registry_root_parent_id, registry_name)
);
create table media_types
(
mt_id INTEGER PRIMARY KEY AUTOINCREMENT,
mt_media_type text not null
constraint unique_media_types_type
unique,
mt_created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
);
create table blobs
(
blob_id INTEGER PRIMARY KEY AUTOINCREMENT,
blob_root_parent_id INTEGER not null,
blob_digest bytea not null,
blob_media_type_id INTEGER not null
constraint fk_blobs_media_type_id_media_types
references media_types(mt_id),
blob_size INTEGER not null,
blob_created_at INTEGER not null,
blob_created_by INTEGER not null,
constraint unique_digest_root_parent_id unique (blob_digest, blob_root_parent_id)
);
create index index_blobs_on_media_type_id
on blobs (blob_media_type_id);
create table registry_blobs
(
rblob_id INTEGER PRIMARY KEY AUTOINCREMENT,
rblob_registry_id INTEGER not null
constraint fk_registry_blobs_rpstry_id_registries
references registries(registry_id)
on delete cascade,
rblob_blob_id INTEGER not null
constraint fk_registry_blobs_blob_id_blobs
references blobs(blob_id)
on delete cascade,
rblob_image_name text
constraint registry_blobs_image_len_check
check (length(rblob_image_name) <= 255),
rblob_created_at INTEGER not null,
rblob_updated_at INTEGER not null,
rblob_created_by INTEGER not null,
rblob_updated_by INTEGER not null,
constraint unique_registry_blobs_registry_id_blob_id_image
unique (rblob_registry_id, rblob_blob_id, rblob_image_name)
);
create index index_registry_blobs_on_reg_id
on registry_blobs (rblob_registry_id);
create index index_registry_blobs_on_reg_blob_id
on registry_blobs (rblob_registry_id, rblob_blob_id);
create table manifests
(
manifest_id INTEGER PRIMARY KEY AUTOINCREMENT,
manifest_registry_id INTEGER not null
constraint fk_manifests_registry_id_registries
references registries(registry_id)
on delete cascade,
manifest_schema_version smallint not null,
manifest_media_type_id INTEGER not null
constraint fk_manifests_media_type_id_media_types
references media_types(mt_id),
manifest_artifact_media_type text,
manifest_total_size INTEGER not null,
manifest_configuration_media_type text,
manifest_configuration_payload bytea,
manifest_configuration_blob_id INTEGER
constraint fk_manifests_configuration_blob_id_blobs
references blobs(blob_id),
manifest_configuration_digest bytea,
manifest_digest bytea not null,
manifest_payload bytea not null,
manifest_non_conformant boolean default false,
manifest_non_distributable_layers boolean default false,
manifest_subject_id INTEGER,
manifest_subject_digest bytea,
manifest_annotations bytea,
manifest_image_name text not null
constraint manifests_img_name_len_check
check (length(manifest_image_name) <= 255),
manifest_created_at INTEGER not null,
manifest_created_by INTEGER not null,
manifest_updated_at INTEGER not null,
manifest_updated_by INTEGER not null,
constraint unique_manifests_registry_id_image_name_and_digest
unique (manifest_registry_id, manifest_image_name, manifest_digest),
constraint unique_manifests_registry_id_id_cfg_blob_id
unique (manifest_registry_id, manifest_id, manifest_configuration_blob_id),
constraint fk_manifests_subject_id_manifests
foreign key (manifest_subject_id) references manifests(manifest_id)
on delete cascade
);
create index index_manifests_on_media_type_id
on manifests (manifest_media_type_id);
create index index_manifests_on_configuration_blob_id
on manifests (manifest_configuration_blob_id);
create table manifest_references
(
manifest_ref_id INTEGER PRIMARY KEY AUTOINCREMENT,
manifest_ref_registry_id INTEGER not null,
manifest_ref_parent_id INTEGER not null,
manifest_ref_child_id INTEGER not null,
manifest_ref_created_at INTEGER not null,
manifest_ref_updated_at INTEGER not null,
manifest_ref_created_by INTEGER not null,
manifest_ref_updated_by INTEGER not null,
constraint unique_manifest_references_prt_id_chd_id
unique (manifest_ref_registry_id, manifest_ref_parent_id, manifest_ref_child_id),
constraint fk_manifest_ref_parent_id_manifests_manifest_id
foreign key (manifest_ref_parent_id) references manifests(manifest_id)
on delete cascade,
constraint fk_manifest_ref_child_id_manifests_manifest_id
foreign key (manifest_ref_child_id) references manifests(manifest_id),
constraint check_manifest_references_parent_id_and_child_id_differ
check (manifest_ref_parent_id <> manifest_ref_child_id)
);
create index index_manifest_references_on_rpstry_id_child_id
on manifest_references (manifest_ref_registry_id, manifest_ref_child_id);
create table layers
(
layer_id INTEGER PRIMARY KEY AUTOINCREMENT,
layer_registry_id INTEGER not null,
layer_manifest_id INTEGER not null,
layer_media_type_id INTEGER not null
constraint fk_layer_media_type_id_media_types
references media_types(mt_id),
layer_blob_id INTEGER not null
constraint fk_layer_blob_id_blobs
references blobs(blob_id),
layer_size INTEGER not null,
layer_created_at INTEGER not null,
layer_updated_at INTEGER not null,
layer_created_by INTEGER not null,
layer_updated_by INTEGER not null,
constraint unique_layer_rpstry_id_and_mnfst_id_and_blob_id
unique (layer_registry_id, layer_manifest_id, layer_blob_id),
constraint unique_layer_rpstry_id_and_id_and_blob_id
unique (layer_registry_id, layer_id, layer_blob_id),
constraint fk_layer_manifest_id_and_manifests_manifest_id
foreign key (layer_manifest_id) references manifests(manifest_id)
on delete cascade
);
create index index_layer_on_media_type_id
on layers (layer_media_type_id);
create index index_layer_on_blob_id
on layers (layer_blob_id);
create table artifacts
(
artifact_id INTEGER PRIMARY KEY AUTOINCREMENT,
artifact_name text not null,
artifact_registry_id INTEGER not null
constraint fk_registries_registry_id
references registries(registry_id)
on delete cascade,
artifact_labels text,
artifact_enabled boolean default false,
artifact_created_at INTEGER,
artifact_updated_at INTEGER,
artifact_created_by INTEGER,
artifact_updated_by INTEGER,
constraint unique_artifact_registry_id_and_name unique (artifact_registry_id, artifact_name),
constraint check_artifact_name_length check ((length(artifact_name) <= 255))
);
create index index_artifact_on_registry_id ON artifacts (artifact_registry_id);
create table artifact_stats
(
artifact_stat_id INTEGER PRIMARY KEY AUTOINCREMENT,
artifact_stat_artifact_id INTEGER not null
constraint fk_artifacts_artifact_id
references artifacts(artifact_id) on delete cascade,
artifact_stat_date INTEGER,
artifact_stat_download_count INTEGER,
artifact_stat_upload_bytes INTEGER,
artifact_stat_download_bytes INTEGER,
artifact_stat_created_at INTEGER not null,
artifact_stat_updated_at INTEGER not null,
artifact_stat_created_by INTEGER not null,
artifact_stat_updated_by INTEGER not null,
constraint unique_artifact_stats_artifact_id_and_date unique (artifact_stat_artifact_id, artifact_stat_date)
);
create table tags
(
tag_id INTEGER PRIMARY KEY AUTOINCREMENT,
tag_name text not null
constraint tag_name_len_check
check (length(tag_name) <= 128),
tag_image_name text not null
constraint tag_img_name_len_check
check (length(tag_image_name) <= 255),
tag_registry_id INTEGER not null,
tag_manifest_id INTEGER not null,
tag_created_at INTEGER,
tag_updated_at INTEGER,
tag_created_by INTEGER,
tag_updated_by INTEGER,
constraint fk_tag_manifest_id_and_manifests_manifest_id FOREIGN KEY
(tag_manifest_id) REFERENCES manifests (manifest_id) ON DELETE CASCADE,
constraint unique_tag_registry_id_and_name_and_image_name
unique (tag_registry_id, tag_name, tag_image_name)
);
create index index_tag_on_rpository_id_and_manifest_id
on tags (tag_registry_id, tag_manifest_id);
create table upstream_proxy_configs
(
upstream_proxy_config_id INTEGER PRIMARY KEY AUTOINCREMENT,
upstream_proxy_config_registry_id INTEGER not null
constraint fk_upstream_proxy_config_registry_id
references registries(registry_id)
on delete cascade,
upstream_proxy_config_source text,
upstream_proxy_config_url text,
upstream_proxy_config_auth_type text not null,
upstream_proxy_config_user_name text,
upstream_proxy_config_secret_identifier text,
upstream_proxy_config_secret_space_id int,
upstream_proxy_config_token text,
upstream_proxy_config_created_at INTEGER,
upstream_proxy_config_updated_at INTEGER,
upstream_proxy_config_created_by INTEGER,
upstream_proxy_config_updated_by INTEGER,
constraint fk_layers_secret_identifier_and_secret_space_id FOREIGN KEY
(upstream_proxy_config_secret_identifier, upstream_proxy_config_secret_space_id) REFERENCES secrets(secret_uid, secret_space_id)
ON DELETE CASCADE
);
create index index_upstream_proxy_config_on_registry_id
on upstream_proxy_configs (upstream_proxy_config_registry_id);
create table cleanup_policies
(
cp_id INTEGER PRIMARY KEY AUTOINCREMENT,
cp_registry_id INTEGER not null
constraint fk_cleanup_policies_registry_id
references registries(registry_id) ON DELETE CASCADE,
cp_name text,
cp_expiry_time_ms INTEGER,
cp_created_at INTEGER not null,
cp_updated_at INTEGER not null,
cp_created_by INTEGER not null,
cp_updated_by INTEGER not null
);
create index index_cleanup_policies_on_registry_id
on cleanup_policies (cp_registry_id);
create table cleanup_policy_prefix_mappings
(
cpp_id INTEGER PRIMARY KEY AUTOINCREMENT,
cpp_cleanup_policy_id INTEGER not null
constraint fk_cleanup_policy_prefix_registry_id
references cleanup_policies(cp_id) ON DELETE CASCADE,
cpp_prefix text not null,
cpp_prefix_type text not null
);
create index index_cleanup_policy_map_on_policy_id
on cleanup_policy_prefix_mappings (cpp_cleanup_policy_id);
insert into media_types (mt_media_type)
values ('application/vnd.docker.distribution.manifest.v1+json'),
('application/vnd.docker.distribution.manifest.v1+prettyjws'),
('application/vnd.docker.distribution.manifest.v2+json'),
('application/vnd.docker.distribution.manifest.list.v2+json'),
('application/vnd.docker.image.rootfs.diff.tar'),
('application/vnd.docker.image.rootfs.diff.tar.gzip'),
('application/vnd.docker.image.rootfs.foreign.diff.tar.gzip'),
('application/vnd.docker.container.image.v1+json'),
('application/vnd.docker.container.image.rootfs.diff+x-gtar'),
('application/vnd.docker.plugin.v1+json'),
('application/vnd.oci.image.layer.v1.tar'),
('application/vnd.oci.image.layer.v1.tar+gzip'),
('application/vnd.oci.image.layer.v1.tar+zstd'),
('application/vnd.oci.image.layer.nondistributable.v1.tar'),
('application/vnd.oci.image.layer.nondistributable.v1.tar+gzip'),
('application/vnd.oci.image.config.v1+json'),
('application/vnd.oci.image.manifest.v1+json'),
('application/vnd.oci.image.index.v1+json'),
('application/vnd.cncf.helm.config.v1+json'),
('application/tar+gzip'),
('application/octet-stream'),
('application/vnd.buildkit.cacheconfig.v0'),
('application/vnd.cncf.helm.chart.content.v1.tar+gzip'),
('application/vnd.cncf.helm.chart.provenance.v1.prov');

1
registry/distribution-spec Submodule

@ -0,0 +1 @@
Subproject commit 11b8e3fba7d2d7329513d0cff53058243c334858

16
registry/types/artifact.go
View File

@ -20,13 +20,11 @@ import (
// Artifact DTO object.
type Artifact struct {
ID int64
Name string
RegistryID int64
Labels []string
Enabled bool
CreatedAt time.Time
UpdatedAt time.Time
CreatedBy int64
UpdatedBy int64
ID int64
Version string
ImageID int64
CreatedAt time.Time
UpdatedAt time.Time
CreatedBy int64
UpdatedBy int64
}

40
registry/types/bandwidth_stat.go Normal file
View File

@ -0,0 +1,40 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"time"
)
// Defines values for BandwidthType.
const (
BandwidthTypeUPLOAD BandwidthType = "UPLOAD"
BandwidthTypeDOWNLOAD BandwidthType = "DOWNLOAD"
)
// BandwidthStat DTO object.
type BandwidthStat struct {
ID int64
ImageID int64
Timestamp time.Time
Type BandwidthType
Bytes int64
CreatedAt time.Time
UpdatedAt time.Time
CreatedBy int64
UpdatedBy int64
}
type BandwidthType string

21
registry/types/artifact_stat.go → registry/types/download_stat.go
View File

@ -18,16 +18,13 @@ import (
"time"
)
// ArtifactStat DTO object.
type ArtifactStat struct {
ID int64
ArtifactID int64
Date int64
DownloadCount int64
UploadBytes int64
DownloadBytes int64
CreatedAt time.Time
UpdatedAt time.Time
CreatedBy int64
UpdatedBy int64
// DownloadStat DTO object.
type DownloadStat struct {
ID int64
ArtifactID int64
Timestamp time.Time
CreatedAt time.Time
UpdatedAt time.Time
CreatedBy int64
UpdatedBy int64
}

32
registry/types/image.go Normal file
View File

@ -0,0 +1,32 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"time"
)
// Image DTO object.
type Image struct {
ID int64
Name string
RegistryID int64
Enabled bool
Labels []string
CreatedAt time.Time
UpdatedAt time.Time
CreatedBy int64
UpdatedBy int64
}