From 7a53717c1184fe75fd293c8b667eb1b4c6bcefc5 Mon Sep 17 00:00:00 2001 From: Tudor Macari Date: Wed, 18 Sep 2024 18:29:26 +0000 Subject: [PATCH] feat: [AH-357]: fix migrations (#2704) * feat: [AH-357]: fix lint * fix migrations --- ..._images_and_alter_table_artifacts.down.sql | 62 ++++++++--- ...le_images_and_alter_table_artifacts.up.sql | 105 +++++++++++++++--- ..._images_and_alter_table_artifacts.down.sql | 61 +++++++++- ...le_images_and_alter_table_artifacts.up.sql | 59 +++++++++- registry/app/pkg/docker/manifest_service.go | 10 +- registry/app/store/database/tag.go | 3 +- 6 files changed, 256 insertions(+), 44 deletions(-) diff --git a/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.down.sql b/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.down.sql index 4b5ad2d36..5d8f31efd 100644 --- a/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.down.sql +++ b/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.down.sql @@ -1,22 +1,48 @@ +CREATE TABLE artifacts_temp +( + artifact_id SERIAL PRIMARY KEY, + artifact_name TEXT NOT NULL, + artifact_registry_id INTEGER NOT NULL + CONSTRAINT fk_registries_registry_id + REFERENCES registries(registry_id) + ON DELETE CASCADE, + artifact_labels TEXT, + artifact_enabled BOOLEAN DEFAULT FALSE, + artifact_created_at BIGINT, + artifact_updated_at BIGINT, + artifact_created_by INTEGER, + artifact_updated_by INTEGER, + CONSTRAINT unique_artifact_registry_id_and_name UNIQUE (artifact_registry_id, artifact_name), + CONSTRAINT check_artifact_name_length CHECK ((LENGTH(artifact_name) <= 255)) +); + +INSERT INTO artifacts_temp (artifact_name, artifact_registry_id, artifact_labels, artifact_enabled, + artifact_created_at, artifact_updated_at, artifact_created_by, artifact_updated_by) +SELECT i.image_name AS artifact_name, + i.image_registry_id AS artifact_registry_id, + i.image_labels AS artifact_labels, + i.image_enabled AS artifact_enabled, + i.image_created_at AS artifact_created_at, + i.image_updated_at AS artifact_updated_at, + i.image_created_by AS artifact_created_by, + i.image_updated_by AS artifact_updated_by + +FROM artifacts a + JOIN images i ON a.artifact_image_id = i.image_id; + +ALTER TABLE artifact_stats + DROP CONSTRAINT fk_artifacts_artifact_id; + +DROP TABLE artifacts; + + +ALTER TABLE artifacts_temp + RENAME TO artifacts; + +ALTER TABLE artifact_stats + ADD CONSTRAINT fk_artifacts_artifact_id FOREIGN KEY (artifact_stat_artifact_id) + REFERENCES artifacts(artifact_id); CREATE INDEX index_artifact_on_registry_id ON artifacts (artifact_registry_id); - -ALTER TABLE artifacts DROP CONSTRAINT fk_images_image_id; -ALTER TABLE artifacts DROP CONSTRAINT unique_artifact_image_id_and_version; - -ALTER TABLE artifacts DROP COLUMN artifact_image_id; -ALTER TABLE artifacts DROP COLUMN artifact_version; - -ALTER TABLE artifacts ADD COLUMN artifact_name TEXT NOT NULL; -ALTER TABLE artifacts ADD COLUMN artifact_registry_id INTEGER NOT NULL; -ALTER TABLE artifacts ADD COLUMN artifact_labels TEXT; -ALTER TABLE artifacts ADD COLUMN artifact_enabled BOOLEAN DEFAULT FALSE; - - -ALTER TABLE artifacts ADD CONSTRAINT check_artifact_name_length CHECK ((LENGTH(artifact_name) <= 255)); -ALTER TABLE artifacts ADD CONSTRAINT unique_artifact_registry_id_and_name UNIQUE (artifact_registry_id, artifact_name); -ALTER TABLE artifacts ADD CONSTRAINT fk_registries_registry_id FOREIGN KEY (artifact_registry_id) - REFERENCES registries(registry_id) ON DELETE CASCADE; - DROP TABLE images; diff --git a/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.up.sql b/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.up.sql index 8e3ca1429..1d004d010 100644 --- a/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.up.sql +++ b/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.up.sql @@ -4,8 +4,8 @@ CREATE TABLE images image_name TEXT NOT NULL, image_registry_id INTEGER NOT NULL CONSTRAINT fk_registries_registry_id - references registries(registry_id), - image_labels TEXT, + REFERENCES registries(registry_id), + image_labels text, image_enabled BOOLEAN DEFAULT FALSE, image_created_at BIGINT NOT NULL, image_updated_at BIGINT NOT NULL, @@ -15,20 +15,99 @@ CREATE TABLE images CONSTRAINT check_image_name_length CHECK ((LENGTH(image_name) <= 255)) ); +INSERT INTO images (image_name, image_registry_id, image_labels, image_enabled, image_created_at, + image_updated_at, image_created_by, image_updated_by) +SELECT artifact_name AS image_name, + artifact_registry_id AS image_registry_id, + artifact_labels AS image_labels, + artifact_enabled AS image_enabled, + artifact_created_at AS image_created_at, + artifact_updated_at AS image_updated_at, + artifact_created_by AS image_created_by, + artifact_updated_by AS image_updated_by +FROM artifacts; + + +CREATE TABLE artifacts_temp +( + artifact_id SERIAL PRIMARY KEY, + artifact_version TEXT NOT NULL, + artifact_image_id INTEGER NOT NULL + CONSTRAINT fk_images_image_id + REFERENCES images(image_id), + artifact_created_at BIGINT NOT NULL, + artifact_updated_at BIGINT NOT NULL, + artifact_created_by INTEGER NOT NULL, + artifact_updated_by INTEGER NOT NULL, + CONSTRAINT unique_artifact_image_id_and_version UNIQUE (artifact_image_id, artifact_version) +); + +INSERT INTO artifacts_temp (artifact_version, artifact_image_id, artifact_created_at, artifact_updated_at, + artifact_created_by, artifact_updated_by) +SELECT encode(m.manifest_digest, 'hex') AS artifact_version, + i.image_id AS artifact_image_id, + m.manifest_created_at AS artifact_created_at, + m.manifest_updated_at AS artifact_updated_at, + m.manifest_created_by AS artifact_created_by, + m.manifest_updated_by AS artifact_updated_by +FROM artifacts a + JOIN images i ON a.artifact_name = i.image_name AND a.artifact_registry_id = i.image_registry_id + JOIN manifests m ON a.artifact_name = m.manifest_image_name AND a.artifact_registry_id = m.manifest_registry_id; + + DROP INDEX index_artifact_on_registry_id; -ALTER TABLE artifacts DROP CONSTRAINT check_artifact_name_length; -ALTER TABLE artifacts DROP CONSTRAINT unique_artifact_registry_id_and_name; -ALTER TABLE artifacts DROP CONSTRAINT fk_registries_registry_id; +CREATE TABLE temp_artifact_stats AS +SELECT * +FROM artifact_stats; -ALTER TABLE artifacts DROP COLUMN artifact_name; -ALTER TABLE artifacts DROP COLUMN artifact_registry_id; -ALTER TABLE artifacts DROP COLUMN artifact_labels; -ALTER TABLE artifacts DROP COLUMN artifact_enabled; +DROP TABLE artifact_stats; -ALTER TABLE artifacts ADD COLUMN artifact_version TEXT NOT NULL; -ALTER TABLE artifacts ADD COLUMN artifact_image_id INTEGER NOT NULL; +DROP TABLE artifacts; +ALTER TABLE artifacts_temp + RENAME TO artifacts; -ALTER TABLE artifacts ADD CONSTRAINT fk_images_image_id FOREIGN KEY (artifact_image_id) REFERENCES images(image_id); -ALTER TABLE artifacts ADD CONSTRAINT unique_artifact_image_id_and_version UNIQUE (artifact_image_id, artifact_version); \ No newline at end of file +create table if not exists artifact_stats +( + artifact_stat_id SERIAL primary key, + artifact_stat_artifact_id INTEGER not null + constraint fk_artifacts_artifact_id + references artifacts(artifact_id), + artifact_stat_date BIGINT, + artifact_stat_download_count BIGINT, + artifact_stat_upload_bytes BIGINT, + artifact_stat_download_bytes BIGINT, + artifact_stat_created_at BIGINT not null, + artifact_stat_updated_at BIGINT not null, + artifact_stat_created_by INTEGER not null, + artifact_stat_updated_by INTEGER not null, + constraint unique_artifact_stats_artifact_id_and_date unique (artifact_stat_artifact_id, artifact_stat_date) +); + +INSERT INTO artifact_stats ( + artifact_stat_id, + artifact_stat_artifact_id, + artifact_stat_date, + artifact_stat_download_count, + artifact_stat_upload_bytes, + artifact_stat_download_bytes, + artifact_stat_created_at, + artifact_stat_updated_at, + artifact_stat_created_by, + artifact_stat_updated_by +) +SELECT + artifact_stat_id, + artifact_stat_artifact_id, + artifact_stat_date, + artifact_stat_download_count, + artifact_stat_upload_bytes, + artifact_stat_download_bytes, + artifact_stat_created_at, + artifact_stat_updated_at, + artifact_stat_created_by, + artifact_stat_updated_by +FROM temp_artifact_stats; + +DROP TABLE temp_artifact_stats; \ No newline at end of file diff --git a/app/store/database/migrate/sqlite/0072_create_ar_table_images_and_alter_table_artifacts.down.sql b/app/store/database/migrate/sqlite/0072_create_ar_table_images_and_alter_table_artifacts.down.sql index a2212d0d6..ac1a517eb 100644 --- a/app/store/database/migrate/sqlite/0072_create_ar_table_images_and_alter_table_artifacts.down.sql +++ b/app/store/database/migrate/sqlite/0072_create_ar_table_images_and_alter_table_artifacts.down.sql @@ -16,18 +16,75 @@ CREATE TABLE artifacts_temp CONSTRAINT check_artifact_name_length CHECK ((LENGTH(artifact_name) <= 255)) ); -INSERT INTO artifacts_temp (artifact_name, artifact_registry_id, artifact_labels) +INSERT INTO artifacts_temp (artifact_name, artifact_registry_id, artifact_labels, artifact_enabled, + artifact_created_at, artifact_updated_at, artifact_created_by, artifact_updated_by) SELECT i.image_name AS artifact_name, i.image_registry_id AS artifact_registry_id, - i.images_labels AS artifact_labels + i.image_labels AS artifact_labels, + i.image_enabled AS artifact_enabled, + i.image_created_at AS artifact_created_at, + i.image_updated_at AS artifact_updated_at, + i.image_created_by AS artifact_created_by, + i.image_updated_by AS artifact_updated_by + FROM artifacts a JOIN images i ON a.artifact_image_id = i.image_id; +CREATE TABLE temp_artifact_stats AS +SELECT * +FROM artifact_stats; + +DROP TABLE artifact_stats; + DROP TABLE artifacts; ALTER TABLE artifacts_temp RENAME TO artifacts; +create table if not exists artifact_stats +( + artifact_stat_id INTEGER PRIMARY KEY AUTOINCREMENT, + artifact_stat_artifact_id INTEGER not null + constraint fk_artifacts_artifact_id + references artifacts(artifact_id), + artifact_stat_date INTEGER, + artifact_stat_download_count INTEGER, + artifact_stat_upload_bytes INTEGER, + artifact_stat_download_bytes INTEGER, + artifact_stat_created_at INTEGER not null, + artifact_stat_updated_at INTEGER not null, + artifact_stat_created_by INTEGER not null, + artifact_stat_updated_by INTEGER not null, + constraint unique_artifact_stats_artifact_id_and_date unique (artifact_stat_artifact_id, artifact_stat_date) + ); + +INSERT INTO artifact_stats ( + artifact_stat_id, + artifact_stat_artifact_id, + artifact_stat_date, + artifact_stat_download_count, + artifact_stat_upload_bytes, + artifact_stat_download_bytes, + artifact_stat_created_at, + artifact_stat_updated_at, + artifact_stat_created_by, + artifact_stat_updated_by +) +SELECT + artifact_stat_id, + artifact_stat_artifact_id, + artifact_stat_date, + artifact_stat_download_count, + artifact_stat_upload_bytes, + artifact_stat_download_bytes, + artifact_stat_created_at, + artifact_stat_updated_at, + artifact_stat_created_by, + artifact_stat_updated_by +FROM temp_artifact_stats; + +DROP TABLE temp_artifact_stats; + CREATE INDEX index_artifact_on_registry_id ON artifacts (artifact_registry_id); DROP TABLE images; diff --git a/app/store/database/migrate/sqlite/0072_create_ar_table_images_and_alter_table_artifacts.up.sql b/app/store/database/migrate/sqlite/0072_create_ar_table_images_and_alter_table_artifacts.up.sql index 4ea33ca4a..f8a389676 100644 --- a/app/store/database/migrate/sqlite/0072_create_ar_table_images_and_alter_table_artifacts.up.sql +++ b/app/store/database/migrate/sqlite/0072_create_ar_table_images_and_alter_table_artifacts.up.sql @@ -42,10 +42,10 @@ CREATE TABLE artifacts_temp CONSTRAINT unique_artifact_image_id_and_version UNIQUE (artifact_image_id, artifact_version) ); -INSERT INTO artifacts_temp (artifact_image_id, artifact_version, artifact_created_at, artifact_updated_at, +INSERT INTO artifacts_temp (artifact_version, artifact_image_id, artifact_created_at, artifact_updated_at, artifact_created_by, artifact_updated_by) -SELECT i.image_id AS artifact_image_id, - m.manifest_digest AS artifact_version, +SELECT LOWER(hex(m.manifest_digest)) AS artifact_version, + i.image_id AS artifact_image_id, m.manifest_created_at AS artifact_created_at, m.manifest_updated_at AS artifact_updated_at, m.manifest_created_by AS artifact_created_by, @@ -56,7 +56,58 @@ FROM artifacts a DROP INDEX index_artifact_on_registry_id; + +CREATE TABLE temp_artifact_stats AS +SELECT * +FROM artifact_stats; + +DROP TABLE artifact_stats; + DROP TABLE artifacts; ALTER TABLE artifacts_temp - RENAME TO artifacts; \ No newline at end of file + RENAME TO artifacts; + +create table if not exists artifact_stats +( + artifact_stat_id INTEGER PRIMARY KEY AUTOINCREMENT, + artifact_stat_artifact_id INTEGER not null + constraint fk_artifacts_artifact_id + references artifacts(artifact_id), + artifact_stat_date INTEGER, + artifact_stat_download_count INTEGER, + artifact_stat_upload_bytes INTEGER, + artifact_stat_download_bytes INTEGER, + artifact_stat_created_at INTEGER not null, + artifact_stat_updated_at INTEGER not null, + artifact_stat_created_by INTEGER not null, + artifact_stat_updated_by INTEGER not null, + constraint unique_artifact_stats_artifact_id_and_date unique (artifact_stat_artifact_id, artifact_stat_date) +); + +INSERT INTO artifact_stats ( + artifact_stat_id, + artifact_stat_artifact_id, + artifact_stat_date, + artifact_stat_download_count, + artifact_stat_upload_bytes, + artifact_stat_download_bytes, + artifact_stat_created_at, + artifact_stat_updated_at, + artifact_stat_created_by, + artifact_stat_updated_by +) +SELECT + artifact_stat_id, + artifact_stat_artifact_id, + artifact_stat_date, + artifact_stat_download_count, + artifact_stat_upload_bytes, + artifact_stat_download_bytes, + artifact_stat_created_at, + artifact_stat_updated_at, + artifact_stat_created_by, + artifact_stat_updated_by +FROM temp_artifact_stats; + +DROP TABLE temp_artifact_stats; \ No newline at end of file diff --git a/registry/app/pkg/docker/manifest_service.go b/registry/app/pkg/docker/manifest_service.go index 58bc3ea36..0e76ef741 100644 --- a/registry/app/pkg/docker/manifest_service.go +++ b/registry/app/pkg/docker/manifest_service.go @@ -211,9 +211,13 @@ func (l *manifestService) dbTagManifest( return err } + digest, err := types.NewDigest(dgst) + if err != nil { + return err + } artifact := &types.Artifact{ ImageID: image.ID, - Version: string(dgst), + Version: digest.String(), } if err := l.artifactDao.CreateOrUpdate(ctx, artifact); err != nil { @@ -231,10 +235,6 @@ func (l *manifestService) dbTagManifest( return err } - if err != nil { - return err - } - return nil }, ) diff --git a/registry/app/store/database/tag.go b/registry/app/store/database/tag.go index 407318344..f8936f44f 100644 --- a/registry/app/store/database/tag.go +++ b/registry/app/store/database/tag.go @@ -505,7 +505,6 @@ func (t tagDao) GetLatestTagMetadata( "images ar ON ar.image_registry_id = t.tag_registry_id "+ "AND ar.image_name = t.tag_image_name", ). - LeftJoin("(SELECT downlad)"). Where( "r.registry_parent_id = ? AND r.registry_name = ?"+ " AND t.tag_image_name = ?", parentID, repoKey, imageName, @@ -638,7 +637,7 @@ func (t tagDao) GetAllArtifactsByRepo( " AND ar.image_name = t.tag_image_name", ). LeftJoin( - "( SELECT i.image_name, COALESCE(t1.download_count, 0) as download_count FROM"+ + "( SELECT i.image_name, SUM(COALESCE(t1.download_count, 0)) as download_count FROM"+ " ( SELECT a.artifact_image_id, COUNT(d.download_stat_id) as download_count"+ " FROM artifacts a "+ " JOIN download_stats d ON d.download_stat_artifact_id = a.artifact_id GROUP BY"+