From 8e4fd70ef0dea491ef2fb2a147848ed277211ba3 Mon Sep 17 00:00:00 2001 From: Arvind Choudhary Date: Wed, 2 Oct 2024 21:49:26 +0000 Subject: [PATCH] feat: [AH-406]: Added Registry Metrics for OSS (#2760) * [AH-406]: Fixes for review comments * [AH-406]: Added Registry Metrics for OSS --- app/services/metric/metrics.go | 69 +++++++++++++++---------- app/services/metric/wire.go | 5 ++ cmd/gitness/wire_gen.go | 2 +- registry/app/store/database.go | 2 + registry/app/store/database/artifact.go | 22 +++++++- registry/app/store/database/registry.go | 19 +++++++ 6 files changed, 90 insertions(+), 29 deletions(-) diff --git a/app/services/metric/metrics.go b/app/services/metric/metrics.go index 8a3a0c13a..d3e620a20 100644 --- a/app/services/metric/metrics.go +++ b/app/services/metric/metrics.go @@ -25,6 +25,7 @@ import ( "github.com/harness/gitness/app/services/system" "github.com/harness/gitness/app/store" "github.com/harness/gitness/job" + store2 "github.com/harness/gitness/registry/app/store" "github.com/harness/gitness/types" "github.com/harness/gitness/version" @@ -34,17 +35,19 @@ import ( const jobType = "metric-collector" type metricData struct { - IP string `json:"ip"` - Hostname string `json:"hostname"` - InstallID string `json:"install_id"` - Installer string `json:"installed_by"` - Installed string `json:"installed_at"` - Version string `json:"version"` - Users int64 `json:"user_count"` - Repos int64 `json:"repo_count"` - Pipelines int64 `json:"pipeline_count"` - Executions int64 `json:"execution_count"` - Gitspaces int64 `json:"gitspace_count"` + IP string `json:"ip"` + Hostname string `json:"hostname"` + InstallID string `json:"install_id"` + Installer string `json:"installed_by"` + Installed string `json:"installed_at"` + Version string `json:"version"` + Users int64 `json:"user_count"` + RepoCount int64 `json:"repo_count"` + PipelineCount int64 `json:"pipeline_count"` + ExecutionCount int64 `json:"execution_count"` + GitspaceCount int64 `json:"gitspace_count"` + RegistryCount int64 `json:"registry_count"` + ArtifactCount int64 `json:"artifact_count"` } type Collector struct { @@ -58,6 +61,8 @@ type Collector struct { executionStore store.ExecutionStore scheduler *job.Scheduler gitspaceConfigStore store.GitspaceConfigStore + registryStore store2.RegistryRepository + artifactStore store2.ArtifactRepository system *system.Service systemSettings *system.Settings } @@ -112,44 +117,56 @@ func (c *Collector) Handle(ctx context.Context, _ string, _ job.ProgressReporter // total users in the system totalUsers, err := c.userStore.CountUsers(ctx, &types.UserFilter{}) if err != nil { - return "", fmt.Errorf("failed to get users total count: %w", err) + return "", fmt.Errorf("failed to get users count: %w", err) } // total repos in the system totalRepos, err := c.repoStore.Count(ctx, 0, &types.RepoFilter{}) if err != nil { - return "", fmt.Errorf("failed to get repositories total count: %w", err) + return "", fmt.Errorf("failed to get repositories count: %w", err) } // total pipelines in the system totalPipelines, err := c.pipelineStore.Count(ctx, 0, types.ListQueryFilter{}) if err != nil { - return "", fmt.Errorf("failed to get pipelines total count: %w", err) + return "", fmt.Errorf("failed to get pipelines count: %w", err) } // total executions in the system totalExecutions, err := c.executionStore.Count(ctx, 0) if err != nil { - return "", fmt.Errorf("failed to get executions total count: %w", err) + return "", fmt.Errorf("failed to get executions count: %w", err) } // total gitspaces (configs) in the system totalGitspaces, err := c.gitspaceConfigStore.Count(ctx, &types.GitspaceFilter{IncludeDeleted: true}) if err != nil { - return "", fmt.Errorf("failed to get gitspace total count: %w", err) + return "", fmt.Errorf("failed to get gitspace count: %w", err) + } + + totalRegistries, err := c.registryStore.Count(ctx) + if err != nil { + return "", fmt.Errorf("failed to get registries count: %w", err) + } + + totalArtifacts, err := c.artifactStore.Count(ctx) + if err != nil { + return "", fmt.Errorf("failed to get artifacts count: %w", err) } data := metricData{ - Hostname: c.hostname, - InstallID: *c.systemSettings.InstallID, - Installer: users[0].Email, - Installed: time.UnixMilli(users[0].Created).Format("2006-01-02 15:04:05"), - Version: version.Version.String(), - Users: totalUsers, - Repos: totalRepos, - Pipelines: totalPipelines, - Executions: totalExecutions, - Gitspaces: totalGitspaces, + Hostname: c.hostname, + InstallID: *c.systemSettings.InstallID, + Installer: users[0].Email, + Installed: time.UnixMilli(users[0].Created).Format("2006-01-02 15:04:05"), + Version: version.Version.String(), + Users: totalUsers, + RepoCount: totalRepos, + PipelineCount: totalPipelines, + ExecutionCount: totalExecutions, + GitspaceCount: totalGitspaces, + RegistryCount: totalRegistries, + ArtifactCount: totalArtifacts, } buf := new(bytes.Buffer) diff --git a/app/services/metric/wire.go b/app/services/metric/wire.go index dae9cda62..4db2868cd 100644 --- a/app/services/metric/wire.go +++ b/app/services/metric/wire.go @@ -18,6 +18,7 @@ import ( "github.com/harness/gitness/app/services/system" "github.com/harness/gitness/app/store" "github.com/harness/gitness/job" + registrystore "github.com/harness/gitness/registry/app/store" "github.com/harness/gitness/types" "github.com/google/wire" @@ -37,6 +38,8 @@ func ProvideCollector( executor *job.Executor, gitspaceConfigStore store.GitspaceConfigStore, system *system.Service, + registryStore registrystore.RegistryRepository, + artifactStore registrystore.ArtifactRepository, ) (*Collector, error) { job := &Collector{ hostname: config.InstanceID, @@ -49,6 +52,8 @@ func ProvideCollector( executionStore: executionStore, scheduler: scheduler, gitspaceConfigStore: gitspaceConfigStore, + registryStore: registryStore, + artifactStore: artifactStore, system: system, } diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index ee1c4ae8c..60a836fb6 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -478,7 +478,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro return nil, err } systemService := system2.ProvideService(settingsService) - collector, err := metric.ProvideCollector(config, principalStore, repoStore, pipelineStore, executionStore, jobScheduler, executor, gitspaceConfigStore, systemService) + collector, err := metric.ProvideCollector(config, principalStore, repoStore, pipelineStore, executionStore, jobScheduler, executor, gitspaceConfigStore, systemService, registryRepository, artifactRepository) if err != nil { return nil, err } diff --git a/registry/app/store/database.go b/registry/app/store/database.go index a991feb54..709959f0a 100644 --- a/registry/app/store/database.go +++ b/registry/app/store/database.go @@ -376,6 +376,7 @@ type RegistryRepository interface { ) (ids []int64, err error) FetchUpstreamProxyKeys(ctx context.Context, ids []int64) (repokeys []string, err error) + Count(ctx context.Context) (int64, error) } type RegistryBlobRepository interface { @@ -424,6 +425,7 @@ type ArtifactRepository interface { GetByName(ctx context.Context, imageID int64, version string) (*types.Artifact, error) // Create an Artifact CreateOrUpdate(ctx context.Context, artifact *types.Artifact) error + Count(ctx context.Context) (int64, error) } type DownloadStatRepository interface { diff --git a/registry/app/store/database/artifact.go b/registry/app/store/database/artifact.go index 9ee8462da..96334d752 100644 --- a/registry/app/store/database/artifact.go +++ b/registry/app/store/database/artifact.go @@ -50,8 +50,7 @@ type artifactDB struct { UpdatedBy int64 `db:"artifact_updated_by"` } -func (a ArtifactDao) GetByName(ctx context.Context, imageID int64, - version string) (*types.Artifact, error) { +func (a ArtifactDao) GetByName(ctx context.Context, imageID int64, version string) (*types.Artifact, error) { q := databaseg.Builder.Select(util.ArrToStringByDelimiter(util.GetDBTagsFromStruct(artifactDB{}), ",")). From("artifacts"). Where("artifact_image_id = ? AND artifact_version = ?", imageID, version) @@ -103,6 +102,25 @@ func (a ArtifactDao) CreateOrUpdate(ctx context.Context, artifact *types.Artifac return nil } +func (a ArtifactDao) Count(ctx context.Context) (int64, error) { + stmt := databaseg.Builder.Select("COUNT(*)"). + From("artifacts") + + sql, args, err := stmt.ToSql() + if err != nil { + return 0, errors.Wrap(err, "Failed to convert query to sql") + } + + db := dbtx.GetAccessor(ctx, a.db) + + var count int64 + err = db.QueryRowContext(ctx, sql, args...).Scan(&count) + if err != nil { + return 0, databaseg.ProcessSQLErrorf(ctx, err, "Failed executing count query") + } + return count, nil +} + func (a ArtifactDao) mapToInternalArtifact(ctx context.Context, in *types.Artifact) *artifactDB { session, _ := request.AuthSessionFrom(ctx) diff --git a/registry/app/store/database/registry.go b/registry/app/store/database/registry.go index 9e1eac197..b4af83fea 100644 --- a/registry/app/store/database/registry.go +++ b/registry/app/store/database/registry.go @@ -139,6 +139,25 @@ func (r registryDao) GetByRootParentIDAndName( return r.mapToRegistry(ctx, dst) } +func (r registryDao) Count(ctx context.Context) (int64, error) { + stmt := databaseg.Builder.Select("COUNT(*)"). + From("registries") + + sql, args, err := stmt.ToSql() + if err != nil { + return 0, errors.Wrap(err, "Failed to convert query to sql") + } + + db := dbtx.GetAccessor(ctx, r.db) + + var count int64 + err = db.QueryRowContext(ctx, sql, args...).Scan(&count) + if err != nil { + return 0, databaseg.ProcessSQLErrorf(ctx, err, "Failed executing count query") + } + return count, nil +} + func (r registryDao) FetchUpstreamProxyKeys( ctx context.Context, ids []int64,