USEFULL
/
drone
mirror of https://github.com/harness/drone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat:[AH-938]: bug fixes for generic artifacts (#3351) 

* feat:[AH-938]: fix checks
* feat:[AH-938]: bug fixes for generic artifacts
pull/3616/head
Sourabh Awashti 2025-01-30 09:20:58 +00:00 committed by Harness
parent d131a4f2aa
commit 1478d44c2e
7 changed files with 185 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
registry/app/api/handler/generic/pull_artifact.go
View File

@ -33,6 +33,7 @@ func (h *Handler) PullArtifact(w http.ResponseWriter, r *http.Request) {
headers, fileReader, redirectURL, err := h.Controller.PullArtifact(ctx, info)
if commons.IsEmptyError(err) {
w.Header().Set("Content-Disposition", "attachment; filename="+info.FileName)
if redirectURL != "" {
http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect)
return
@ -48,7 +49,6 @@ func (h *Handler) serveContent(
w http.ResponseWriter, r *http.Request, fileReader *storage.FileReader, info pkg.GenericArtifactInfo,
) {
if fileReader != nil {
w.Header().Set("Content-Disposition", "attachment; filename="+info.FileName)
http.ServeContent(w, r, info.FileName, time.Time{}, fileReader)
}
}

7
registry/app/api/handler/generic/push_artifact.go
View File

@ -38,9 +38,14 @@ func (h *Handler) PushArtifact(w http.ResponseWriter, r *http.Request) {
}
ctx := r.Context()
defer file.Close()
headers, err := h.Controller.UploadArtifact(ctx, info, file)
headers, sha256, err := h.Controller.UploadArtifact(ctx, info, file)
if commons.IsEmptyError(err) {
headers.WriteToResponse(w)
_, err := w.Write([]byte(fmt.Sprintf("Pushed.\nSha256: %s", sha256)))
if err != nil {
handleErrors(r.Context(), errcode.ErrCodeUnknown.WithDetail(err), w)
return
}
}
handleErrors(r.Context(), err, w)
}

4
registry/app/pkg/filemanager/file_manager.go
View File

@ -201,8 +201,8 @@ func (f *FileManager) DownloadFile(
) (fileReader *storage.FileReader, size int64, redirectURL string, err error) {
node, err := f.nodesDao.GetByPathAndRegistryID(ctx, regInfo.ID, filePath)
if err != nil {
return nil, 0, "", fmt.Errorf("failed to get the node for path: %s, "+
"with registry: %s, with error %s", filePath, regInfo.Name, err)
return nil, 0, "", fmt.Errorf("failed to get the file for path: %s, "+
"with registry: %s", filePath, regInfo.Name)
}
blob, err := f.genericBlobDao.FindByID(ctx, node.BlobID)

56
registry/app/pkg/generic/controller.go
View File

@ -89,7 +89,7 @@ func NewDBStore(
const regNameFormat = "registry : [%s]"
func (c Controller) UploadArtifact(ctx context.Context, info pkg.GenericArtifactInfo,
file multipart.File) (*commons.ResponseHeaders, errcode.Error) {
file multipart.File) (*commons.ResponseHeaders, string, errcode.Error) {
responseHeaders := &commons.ResponseHeaders{
Headers: make(map[string]string),
Code: 0,
@ -99,14 +99,20 @@ func (c Controller) UploadArtifact(ctx context.Context, info pkg.GenericArtifact
enum.PermissionArtifactsUpload,
)
if err != nil {
return nil, errcode.ErrCodeDenied.WithDetail(err)
return nil, "", errcode.ErrCodeDenied.WithDetail(err)
}
err = c.CheckIfFileAlreadyExist(ctx, info)
if err != nil {
return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(err)
}
path := info.Image + "/" + info.Version + "/" + info.FileName
fileInfo, err := c.fileManager.UploadFile(ctx, path, info.RegIdentifier, info.RegistryID,
info.RootParentID, info.RootIdentifier, file, nil, info.FileName)
if err != nil {
return responseHeaders, errcode.ErrCodeUnknown.WithDetail(err)
return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err)
}
err = c.tx.WithTx(
ctx, func(ctx context.Context) error {
@ -157,10 +163,10 @@ func (c Controller) UploadArtifact(ctx context.Context, info pkg.GenericArtifact
})
if err != nil {
return responseHeaders, errcode.ErrCodeUnknown.WithDetail(err)
return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err)
}
responseHeaders.Code = http.StatusCreated
return responseHeaders, errcode.Error{}
return responseHeaders, fileInfo.Sha256, errcode.Error{}
}
func (c Controller) updateMetadata(dbArtifact *types.Artifact, metadata *database.GenericMetadata,
@ -210,11 +216,47 @@ func (c Controller) PullArtifact(ctx context.Context, info pkg.GenericArtifactIn
path := "/" + info.Image + "/" + info.Version + "/" + info.FileName
fileReader, _, redirectURL, err := c.fileManager.DownloadFile(ctx, path, types.Registry{
ID: info.RegistryID,
Name: info.RootIdentifier,
Name: info.RegIdentifier,
}, info.RootIdentifier)
if err != nil {
return responseHeaders, nil, "", errcode.ErrCodeUnknown.WithDetail(err)
return responseHeaders, nil, "", errcode.ErrCodeRootNotFound.WithDetail(err)
}
responseHeaders.Code = http.StatusOK
return responseHeaders, fileReader, redirectURL, errcode.Error{}
}
func (c Controller) CheckIfFileAlreadyExist(ctx context.Context, info pkg.GenericArtifactInfo) error {
image, err := c.DBStore.ImageDao.GetByName(ctx, info.RegistryID, info.Image)
if err != nil && !strings.Contains(err.Error(), "resource not found") {
return fmt.Errorf("failed to fetch the image for artifact : [%s] with "+
regNameFormat, info.Image, info.RegIdentifier)
}
if image == nil {
return nil
}
dbArtifact, err := c.DBStore.ArtifactDao.GetByName(ctx, image.ID, info.Version)
if err != nil && !strings.Contains(err.Error(), "resource not found") {
return fmt.Errorf("failed to fetch artifact : [%s] with "+
regNameFormat, info.Image, info.RegIdentifier)
}
if dbArtifact == nil {
return nil
}
metadata := &database.GenericMetadata{}
err = json.Unmarshal(dbArtifact.Metadata, metadata)
if err == nil {
for _, file := range metadata.Files {
if file.Filename == info.FileName {
return fmt.Errorf("file: [%s] with Artifact: [%s], Version: [%s] and registry: [%s] already exist",
info.FileName, info.Image, info.Version, info.RegIdentifier)
}
}
}
return nil
}

2
registry/app/store/database/artifact.go
View File

@ -341,7 +341,7 @@ func (a ArtifactDao) GetAllArtifactsByRepo(
Where("a1.rank = 1 ")
if search != "" {
q = q.Where("image_name LIKE ?", sqlPartialMatch(search))
q = q.Where("i.image_name LIKE ?", sqlPartialMatch(search))
}
if len(labels) > 0 {

35
registry/app/store/database/registry.go
View File

@ -211,18 +211,17 @@ func (r registryDao) GetByIDIn(ctx context.Context, ids []int64) (*[]types.Regis
}
type RegistryMetadataDB struct {
RegID string `db:"registry_id"`
RegIdentifier string `db:"reg_identifier"`
Description sql.NullString `db:"description"`
PackageType artifact.PackageType `db:"package_type"`
Type artifact.RegistryType `db:"type"`
LastModified int64 `db:"last_modified"`
URL sql.NullString `db:"url"`
ArtifactCount int64 `db:"artifact_count"`
DownloadCount int64 `db:"download_count"`
Size int64 `db:"size"`
GenericBlobSize int64 `db:"generic_blob_size"`
Labels sql.NullString `db:"registry_labels"`
RegID string `db:"registry_id"`
RegIdentifier string `db:"reg_identifier"`
Description sql.NullString `db:"description"`
PackageType artifact.PackageType `db:"package_type"`
Type artifact.RegistryType `db:"type"`
LastModified int64 `db:"last_modified"`
URL sql.NullString `db:"url"`
ArtifactCount int64 `db:"artifact_count"`
DownloadCount int64 `db:"download_count"`
Size int64 `db:"size"`
Labels sql.NullString `db:"registry_labels"`
}
func (r registryDao) GetAll(
@ -246,8 +245,10 @@ func (r registryDao) GetAll(
r.registry_updated_at AS last_modified,
COALESCE(u.upstream_proxy_config_url, '') AS url,
COALESCE(artifact_count.count, 0) AS artifact_count,
COALESCE(blob_sizes.total_size, 0) AS size,
COALESCE(generic_blob_sizes.total_size, 0) AS generic_blob_size,
CASE
WHEN COALESCE(blob_sizes.total_size, 0) = 0 THEN COALESCE(generic_blob_sizes.total_size, 0)
ELSE COALESCE(blob_sizes.total_size, 0)
END AS size,
r.registry_labels,
COALESCE(download_stats.download_count, 0) AS download_count
`
@ -730,10 +731,6 @@ func (r registryDao) mapToRegistryMetadataList(
}
func (r registryDao) mapToRegistryMetadata(_ context.Context, dst *RegistryMetadataDB) *store.RegistryMetadata {
size := dst.Size
if size == 0 {
size = dst.GenericBlobSize
}
return &store.RegistryMetadata{
RegID: dst.RegID,
RegIdentifier: dst.RegIdentifier,
@ -744,7 +741,7 @@ func (r registryDao) mapToRegistryMetadata(_ context.Context, dst *RegistryMetad
URL: dst.URL.String,
ArtifactCount: dst.ArtifactCount,
DownloadCount: dst.DownloadCount,
Size: size,
Size: dst.Size,
Labels: util.StringToArr(dst.Labels.String),
}
}

164
registry/app/store/database/tag.go
View File

@ -354,11 +354,111 @@ func (t tagDao) GetAllArtifactsByParentID(
latestVersion bool,
packageTypes []string,
) (*[]types.ArtifactMetadata, error) {
q := databaseg.Builder.Select(
q1 := t.GetAllArtifactOnParentIDQueryForNonOCI(parentID, latestVersion, registryIDs, packageTypes, search)
q2 := t.GetAllArtifactsQueryByParentIDForOCI(parentID, latestVersion, registryIDs, packageTypes, search)
q1SQL, q1Args, err := q1.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
}
q2SQL, _, err := q2.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
}
// Combine q1 and q2 with UNION ALL
finalQuery := fmt.Sprintf(`
SELECT repo_name, name, package_type, version, modified_at, labels, download_count
FROM (%s UNION ALL %s) AS combined
`, q1SQL, q2SQL)
// Combine query arguments
finalArgs := q1Args
// Apply sorting based on provided field
sortField := "modified_at"
if sortByField == downloadCount {
sortField = "download_count"
} else if sortByField == imageName {
sortField = "name"
}
finalQuery = fmt.Sprintf("%s ORDER BY %s %s", finalQuery, sortField, sortByOrder)
// Add pagination (LIMIT and OFFSET) **after** the WHERE and ORDER BY clauses
finalQuery = fmt.Sprintf("%s LIMIT %d OFFSET %d", finalQuery, limit, offset)
db := dbtx.GetAccessor(ctx, t.db)
dst := []*artifactMetadataDB{}
if err = db.SelectContext(ctx, &dst, finalQuery, finalArgs...); err != nil {
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed executing custom list query")
}
return t.mapToArtifactMetadataList(ctx, dst)
}
func (t tagDao) GetAllArtifactsQueryByParentIDForOCI(parentID int64, latestVersion bool, registryIDs *[]string,
packageTypes []string, search string) sq.SelectBuilder {
q2 := databaseg.Builder.Select(
`r.registry_name as repo_name,
t.tag_image_name as name,
r.registry_package_type as package_type,
t.tag_name as version,
t.tag_updated_at as modified_at,
i.image_labels as labels,
COALESCE(t2.download_count,0) as download_count `,
).
From("tags t").
Join("registries r ON t.tag_registry_id = r.registry_id").
Where("r.registry_parent_id = ?", parentID).
Join(
"images i ON i.image_registry_id = t.tag_registry_id AND"+
" i.image_name = t.tag_image_name",
).
LeftJoin(
`( SELECT i.image_name, SUM(COALESCE(t1.download_count, 0)) as download_count FROM
( SELECT a.artifact_image_id, COUNT(d.download_stat_id) as download_count
FROM artifacts a JOIN download_stats d ON d.download_stat_artifact_id = a.artifact_id
GROUP BY a.artifact_image_id ) as t1
JOIN images i ON i.image_id = t1.artifact_image_id
JOIN registries r ON r.registry_id = i.image_registry_id
WHERE r.registry_parent_id = ? GROUP BY i.image_name) as t2
ON t.tag_image_name = t2.image_name`, parentID,
)
if latestVersion {
q2 = q2.Join(
`(SELECT t.tag_id as id, ROW_NUMBER() OVER (PARTITION BY t.tag_registry_id, t.tag_image_name
ORDER BY t.tag_updated_at DESC) AS rank FROM tags t
JOIN registries r ON t.tag_registry_id = r.registry_id
WHERE r.registry_parent_id = ? ) AS a
ON t.tag_id = a.id`, parentID, // nolint:goconst
).
Where("a.rank = 1")
}
if len(*registryIDs) > 0 {
q2 = q2.Where(sq.Eq{"r.registry_name": registryIDs})
}
if len(packageTypes) > 0 {
q2 = q2.Where(sq.Eq{"r.registry_package_type": packageTypes})
}
if search != "" {
q2 = q2.Where("t.tag_image_name LIKE ?", sqlPartialMatch(search))
}
return q2
}
func (t tagDao) GetAllArtifactOnParentIDQueryForNonOCI(parentID int64, latestVersion bool, registryIDs *[]string,
packageTypes []string, search string) sq.SelectBuilder {
q1 := databaseg.Builder.Select(
`r.registry_name as repo_name,
i.image_name as name,
r.registry_package_type as package_type,
tag_q.tag_name AS tag,
r.registry_package_type as package_type,
ar.artifact_version as version,
ar.artifact_updated_at as modified_at,
i.image_labels as labels,
@ -367,7 +467,7 @@ func (t tagDao) GetAllArtifactsByParentID(
From("artifacts ar").
Join("images i ON i.image_id = ar.artifact_image_id").
Join("registries r ON i.image_registry_id = r.registry_id").
Where("r.registry_parent_id = ?", parentID).
Where("r.registry_parent_id = ? AND r.registry_package_type NOT IN ('DOCKER', 'HELM')", parentID).
LeftJoin(
`( SELECT a.artifact_version, SUM(COALESCE(t1.download_count, 0)) as download_count FROM
( SELECT a.artifact_id, COUNT(d.download_stat_id) as download_count
@ -381,7 +481,7 @@ func (t tagDao) GetAllArtifactsByParentID(
)
if latestVersion {
q = q.Join(
q1 = q1.Join(
`(SELECT ar.artifact_id as id, ROW_NUMBER() OVER (PARTITION BY ar.artifact_image_id
ORDER BY ar.artifact_updated_at DESC) AS rank FROM artifacts ar
JOIN images i ON i.image_id = ar.artifact_image_id
@ -389,65 +489,21 @@ func (t tagDao) GetAllArtifactsByParentID(
WHERE r.registry_parent_id = ? ) AS a
ON ar.artifact_id = a.id`, parentID, // nolint:goconst
).
Where("a.rank = 1").
LeftJoin(
`(SELECT t.tag_name, t.tag_image_name, t.tag_registry_id, m.manifest_digest,
ROW_NUMBER() OVER (PARTITION BY t.tag_image_name,
t.tag_registry_id ORDER BY t.tag_updated_at DESC) AS tag_rank
FROM tags t
JOIN manifests m ON t.tag_manifest_id = m.manifest_id) AS tag_q
ON CASE
WHEN ar.artifact_version ~ '^[0-9A-Fa-f]+$' THEN decode(ar.artifact_version, 'hex')
ELSE NULL
END = tag_q.manifest_digest
AND i.image_name = tag_q.tag_image_name
AND i.image_registry_id = tag_q.tag_registry_id AND tag_q.tag_rank = 1`,
)
} else {
q = q.LeftJoin(
`(SELECT t.tag_name, t.tag_image_name, t.tag_registry_id, m.manifest_digest FROM tags t
JOIN manifests m ON t.tag_manifest_id = m.manifest_id) AS tag_q
ON CASE
WHEN ar.artifact_version ~ '^[0-9A-Fa-f]+$' THEN decode(ar.artifact_version, 'hex')
ELSE NULL
END = tag_q.manifest_digest AND
i.image_name = tag_q.tag_image_name AND i.image_registry_id = tag_q.tag_registry_id`,
)
Where("a.rank = 1")
}
if len(*registryIDs) > 0 {
q = q.Where(sq.Eq{"r.registry_name": registryIDs})
q1 = q1.Where(sq.Eq{"r.registry_name": registryIDs})
}
if len(packageTypes) > 0 {
q = q.Where(sq.Eq{"r.registry_package_type": packageTypes})
q1 = q1.Where(sq.Eq{"r.registry_package_type": packageTypes})
}
if search != "" {
q = q.Where("i.image_name LIKE ?", sqlPartialMatch(search))
q1 = q1.Where("i.image_name LIKE ?", sqlPartialMatch(search))
}
sortField := "image_" + sortByField
if sortByField == downloadCount {
sortField = downloadCount
} else if sortByField == imageName {
sortField = name
}
q = q.OrderBy(sortField + " " + sortByOrder).Limit(uint64(limit)).Offset(uint64(offset))
sql, args, err := q.ToSql()
if err != nil {
return nil, errors.Wrap(err, "Failed to convert query to sql")
}
db := dbtx.GetAccessor(ctx, t.db)
dst := []*artifactMetadataDB{}
if err = db.SelectContext(ctx, &dst, sql, args...); err != nil {
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed executing custom list query")
}
return t.mapToArtifactMetadataList(ctx, dst)
return q1
}
func (t tagDao) CountAllArtifactsByParentID(