diff --git a/cli/server/server.go b/cli/server/server.go index b25b4133d..72e1d72e1 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -89,6 +89,14 @@ func (c *command) run(*kingpin.ParseContext) error { gHTTP, shutdownHTTP := system.server.ListenAndServe() g.Go(gHTTP.Wait) if c.enableCI { + // start populating plugins + g.Go(func() error { + err := system.pluginManager.Populate(ctx) + if err != nil { + log.Error().Err(err).Msg("could not populate plugins") + } + return nil + }) // start poller for CI build executions. g.Go(func() error { log := logrus.New() diff --git a/cli/server/system.go b/cli/server/system.go index f338c03b9..87dabb0f3 100644 --- a/cli/server/system.go +++ b/cli/server/system.go @@ -8,6 +8,7 @@ import ( gitrpcserver "github.com/harness/gitness/gitrpc/server" gitrpccron "github.com/harness/gitness/gitrpc/server/cron" "github.com/harness/gitness/internal/bootstrap" + "github.com/harness/gitness/internal/pipeline/plugin" "github.com/harness/gitness/internal/server" "github.com/harness/gitness/internal/services" @@ -19,19 +20,22 @@ type System struct { bootstrap bootstrap.Bootstrap server *server.Server gitRPCServer *gitrpcserver.GRPCServer + pluginManager *plugin.PluginManager poller *poller.Poller services services.Services gitRPCCronMngr *gitrpccron.Manager } // NewSystem returns a new system structure. -func NewSystem(bootstrap bootstrap.Bootstrap, server *server.Server, poller *poller.Poller, gitRPCServer *gitrpcserver.GRPCServer, +func NewSystem(bootstrap bootstrap.Bootstrap, server *server.Server, poller *poller.Poller, + gitRPCServer *gitrpcserver.GRPCServer, pluginManager *plugin.PluginManager, gitrpccron *gitrpccron.Manager, services services.Services) *System { return &System{ bootstrap: bootstrap, server: server, poller: poller, gitRPCServer: gitRPCServer, + pluginManager: pluginManager, services: services, gitRPCCronMngr: gitrpccron, } diff --git a/cmd/gitness/wire.go b/cmd/gitness/wire.go index d3e7fad0f..05ce27a52 100644 --- a/cmd/gitness/wire.go +++ b/cmd/gitness/wire.go @@ -10,8 +10,6 @@ package main import ( "context" - "github.com/harness/gitness/internal/services/exporter" - cliserver "github.com/harness/gitness/cli/server" "github.com/harness/gitness/encrypt" "github.com/harness/gitness/events" @@ -46,6 +44,7 @@ import ( "github.com/harness/gitness/internal/pipeline/commit" "github.com/harness/gitness/internal/pipeline/file" "github.com/harness/gitness/internal/pipeline/manager" + pluginmanager "github.com/harness/gitness/internal/pipeline/plugin" "github.com/harness/gitness/internal/pipeline/runner" "github.com/harness/gitness/internal/pipeline/scheduler" "github.com/harness/gitness/internal/pipeline/triggerer" @@ -53,6 +52,7 @@ import ( "github.com/harness/gitness/internal/server" "github.com/harness/gitness/internal/services" "github.com/harness/gitness/internal/services/codecomments" + "github.com/harness/gitness/internal/services/exporter" "github.com/harness/gitness/internal/services/importer" "github.com/harness/gitness/internal/services/job" "github.com/harness/gitness/internal/services/metric" @@ -138,6 +138,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e commit.WireSet, controllertrigger.WireSet, plugin.WireSet, + pluginmanager.WireSet, importer.WireSet, canceler.WireSet, exporter.WireSet, diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index a7c2d2696..2b27b8396 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -42,6 +42,7 @@ import ( "github.com/harness/gitness/internal/pipeline/commit" "github.com/harness/gitness/internal/pipeline/file" "github.com/harness/gitness/internal/pipeline/manager" + plugin2 "github.com/harness/gitness/internal/pipeline/plugin" "github.com/harness/gitness/internal/pipeline/runner" "github.com/harness/gitness/internal/pipeline/scheduler" "github.com/harness/gitness/internal/pipeline/triggerer" @@ -223,7 +224,8 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro serverServer := server2.ProvideServer(config, routerRouter) executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, streamer, fileService, logStore, logStream, checkStore, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore) client := manager.ProvideExecutionClient(executionManager, config) - runtimeRunner, err := runner.ProvideExecutionRunner(config, client, executionManager) + pluginManager := plugin2.ProvidePluginManager(config, pluginStore) + runtimeRunner, err := runner.ProvideExecutionRunner(config, client, pluginManager, executionManager) if err != nil { return nil, err } @@ -253,6 +255,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro return nil, err } servicesServices := services.ProvideServices(webhookService, pullreqService, triggerService, jobScheduler, collector) - serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, poller, grpcServer, cronManager, servicesServices) + serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, poller, grpcServer, pluginManager, cronManager, servicesServices) return serverSystem, nil } diff --git a/go.mod b/go.mod index 7ad6dfc28..a31111587 100644 --- a/go.mod +++ b/go.mod @@ -12,15 +12,15 @@ require ( github.com/coreos/go-semver v0.3.0 github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 github.com/dgrijalva/jwt-go v3.2.0+incompatible - github.com/drone-runners/drone-runner-docker v1.8.4-0.20230918204859-ce45c209ba31 + github.com/drone-runners/drone-runner-docker v1.8.4-0.20230919202034-23803f6b38c2 github.com/drone/drone-go v1.7.1 github.com/drone/drone-yaml v1.2.3 github.com/drone/funcmap v0.0.0-20190918184546-d4ef6e88376d - github.com/drone/go-convert v0.0.0-20230913194237-2228eb829d0e - github.com/drone/go-generate v0.0.0-20230916194845-e98bb2a7f93a + github.com/drone/go-convert v0.0.0-20230919093251-7104c3bcc635 + github.com/drone/go-generate v0.0.0-20230919103306-db4551429a31 github.com/drone/go-scm v1.31.2 github.com/drone/runner-go v1.12.0 - github.com/drone/spec v0.0.0-20230918031032-07d0f2594649 + github.com/drone/spec v0.0.0-20230919004456-7455b8913ff5 github.com/go-chi/chi v1.5.4 github.com/go-chi/cors v1.2.1 github.com/go-redis/redis/v8 v8.11.5 @@ -70,6 +70,7 @@ require ( cloud.google.com/go/compute v1.18.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect dario.cat/mergo v1.0.0 // indirect + github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d // indirect github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e // indirect github.com/antonmedv/expr v1.15.2 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -83,6 +84,7 @@ require ( github.com/docker/go-connections v0.3.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/drone/envsubst v1.0.3 // indirect + github.com/drone/signal v1.0.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/go.sum b/go.sum index 23644e036..de8f293f6 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,7 @@ code.gitea.io/gitea v1.17.2/go.mod h1:sovminOoSsc8IC2T29rX9+MmaboHTu8QDEvJjaSqIX dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= docker.io/go-docker v1.0.0/go.mod h1:7tiAn5a0LFmjbPDbyTPOaTTOuG1ZRNXdPA6RvKY+fpY= +github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d h1:j6oB/WPCigdOkxtuPl1VSIiLpy7Mdsu6phQffbF19Ng= github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs= github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e h1:rl2Aq4ZODqTDkeSqQBy+fzpZPamacO1Srp8zq7jf2Sc= github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e/go.mod h1:Xa6lInWHNQnuWoF0YPSsx+INFA9qk7/7pTjwb3PInkY= @@ -137,6 +138,10 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/drone-runners/drone-runner-docker v1.8.4-0.20230918204859-ce45c209ba31 h1:lq8ysXgygXtA6QCL3GCM+3VrSSi+IToF5HSMkA1DNyY= github.com/drone-runners/drone-runner-docker v1.8.4-0.20230918204859-ce45c209ba31/go.mod h1:mlVgx9GWL+BvvgKiroR09XupiP+DEP8sWr/M8wDsH8Q= +github.com/drone-runners/drone-runner-docker v1.8.4-0.20230919103848-e39029551c7e h1:zajv6sRMfij4QzvQfT4UHV1If6AhLa/6dSuQcvqktEo= +github.com/drone-runners/drone-runner-docker v1.8.4-0.20230919103848-e39029551c7e/go.mod h1:iXTCJv+tESfI/ggWZwinI2ZAzHTGS+Ic5A9gcUElTns= +github.com/drone-runners/drone-runner-docker v1.8.4-0.20230919202034-23803f6b38c2 h1:NmYT2bCmacG9mFWql0hKlJ7KOjfXDlusrpWbjUDYKoA= +github.com/drone-runners/drone-runner-docker v1.8.4-0.20230919202034-23803f6b38c2/go.mod h1:iXTCJv+tESfI/ggWZwinI2ZAzHTGS+Ic5A9gcUElTns= github.com/drone/drone-go v1.7.1 h1:ZX+3Rs8YHUSUQ5mkuMLmm1zr1ttiiE2YGNxF3AnyDKw= github.com/drone/drone-go v1.7.1/go.mod h1:fxCf9jAnXDZV1yDr0ckTuWd1intvcQwfJmTRpTZ1mXg= github.com/drone/drone-runtime v1.0.7-0.20190729202838-87c84080f4a1/go.mod h1:+osgwGADc/nyl40J0fdsf8Z09bgcBZXvXXnLOY48zYs= @@ -149,15 +154,20 @@ github.com/drone/funcmap v0.0.0-20190918184546-d4ef6e88376d h1:/IO7UVVu191Jc0Daj github.com/drone/funcmap v0.0.0-20190918184546-d4ef6e88376d/go.mod h1:Hph0/pT6ZxbujnE1Z6/08p5I0XXuOsppqF6NQlGOK0E= github.com/drone/go-convert v0.0.0-20230913194237-2228eb829d0e h1:H0eAzGW6vj4XCHnrAlmrcyQbg6g3yVcjfobUHzjNxGM= github.com/drone/go-convert v0.0.0-20230913194237-2228eb829d0e/go.mod h1:ciC4CjDBSrpHGWmodGdUQshx7P5lAqL6DFJBnf3zVWU= +github.com/drone/go-convert v0.0.0-20230919093251-7104c3bcc635 h1:qQX+U2iEm4X2FcmBzxZwZgz8gLpUTa6lBB1vBBCV9Oo= +github.com/drone/go-convert v0.0.0-20230919093251-7104c3bcc635/go.mod h1:PyCDcuAhGF6W0VJ6qMmlM47dsSyGv/zDiMqeJxMFuGM= github.com/drone/go-generate v0.0.0-20230916194845-e98bb2a7f93a h1:kL6IAktFq7P3sRtbnMC/L6GBwJe+IzPKezk0HGq22nE= github.com/drone/go-generate v0.0.0-20230916194845-e98bb2a7f93a/go.mod h1:WbpwzOT2+x4z59zMUA2lRzuqzZ87/QPIeLfkEEi1FUI= +github.com/drone/go-generate v0.0.0-20230919103306-db4551429a31 h1:/z0vz1tJ5t5D6RPug2IZm8QwoYPrXYt/p4Le6B5hJl4= +github.com/drone/go-generate v0.0.0-20230919103306-db4551429a31/go.mod h1:eTfy716efMJgVvk/ZkRvitaXY2UuytfqDjxclFMeLdQ= github.com/drone/go-scm v1.31.2 h1:6hZxf0aETV17830fMCPrgcA4y8j/8Gdfy0xEdInUeqQ= github.com/drone/go-scm v1.31.2/go.mod h1:DFIJJjhMj0TSXPz+0ni4nyZ9gtTtC40Vh/TGRugtyWw= github.com/drone/runner-go v1.12.0 h1:zUjDj9ylsJ4n4Mvy4znddq/Z4EBzcUXzTltpzokKtgs= github.com/drone/runner-go v1.12.0/go.mod h1:vu4pPPYDoeN6vdYQAY01GGGsAIW4aLganJNaa8Fx8zE= +github.com/drone/signal v1.0.0 h1:NrnM2M/4yAuU/tXs6RP1a1ZfxnaHwYkd0kJurA1p6uI= github.com/drone/signal v1.0.0/go.mod h1:S8t92eFT0g4WUgEc/LxG+LCuiskpMNsG0ajAMGnyZpc= -github.com/drone/spec v0.0.0-20230918031032-07d0f2594649 h1:e22spVEjx7WNwRnKXIZ4eUVEmICWb/L92JZxezregQs= -github.com/drone/spec v0.0.0-20230918031032-07d0f2594649/go.mod h1:KyQZA9qwuscbbM7yTrtZg25Wammoc5GKwaRem8kDA5k= +github.com/drone/spec v0.0.0-20230919004456-7455b8913ff5 h1:NgAseJNQpJE3XtgJUPu4x7x5fcBjqZ3oKHDJfwBYdWk= +github.com/drone/spec v0.0.0-20230919004456-7455b8913ff5/go.mod h1:KyQZA9qwuscbbM7yTrtZg25Wammoc5GKwaRem8kDA5k= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= diff --git a/internal/pipeline/manager/teardown.go b/internal/pipeline/manager/teardown.go index 98319e8c2..8988269c6 100644 --- a/internal/pipeline/manager/teardown.go +++ b/internal/pipeline/manager/teardown.go @@ -17,8 +17,8 @@ import ( gitness_store "github.com/harness/gitness/store" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" - "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/rs/zerolog/log" ) diff --git a/internal/pipeline/plugin/manager.go b/internal/pipeline/plugin/manager.go new file mode 100644 index 000000000..f018c6ba1 --- /dev/null +++ b/internal/pipeline/plugin/manager.go @@ -0,0 +1,230 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package plugin + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + + "github.com/harness/gitness/internal/store" + "github.com/harness/gitness/types" + + v1yaml "github.com/drone/spec/dist/go" + "github.com/drone/spec/dist/go/parse" + "github.com/rs/zerolog/log" +) + +// Lookup returns a resource by name, kind and type. +type LookupFunc func(name, kind, typ, version string) (*v1yaml.Config, error) + +type PluginManager struct { + config *types.Config + pluginStore store.PluginStore +} + +func NewPluginManager( + config *types.Config, + pluginStore store.PluginStore, +) *PluginManager { + return &PluginManager{ + config: config, + pluginStore: pluginStore, + } +} + +// GetLookupFn returns a lookup function for plugins which can be used in the resolver. +func (m *PluginManager) GetLookupFn() LookupFunc { + return func(name, kind, typ, version string) (*v1yaml.Config, error) { + if kind != "plugin" { + return nil, fmt.Errorf("only plugin kind supported") + } + if typ != "step" { + return nil, fmt.Errorf("only step plugins supported") + } + plugin, err := m.pluginStore.Find(context.Background(), name, version) + if err != nil { + return nil, fmt.Errorf("could not lookup plugin: %w", err) + } + // Convert plugin to v1yaml spec + config, err := parse.ParseString(plugin.Spec) + if err != nil { + return nil, fmt.Errorf("could not unmarshal plugin to v1yaml spec: %w", err) + } + + return config, nil + } +} + +// Populate fetches plugins information from an external source or a local zip +// and populates in the DB. +func (m *PluginManager) Populate(ctx context.Context) error { + path := m.config.CI.PluginsZipPath + if path == "" { + return fmt.Errorf("plugins path not provided to read schemas from") + } + + var zipFile *zip.ReadCloser + if _, err := os.Stat(path); err != nil { // local path doesn't exist - must be a remote link + // Download zip file locally + f, err := os.CreateTemp(os.TempDir(), "plugins.zip") + if err != nil { + return fmt.Errorf("could not create temp file: %w", err) + } + defer os.Remove(f.Name()) + err = downloadZip(path, f.Name()) + if err != nil { + return fmt.Errorf("could not download remote zip: %w", err) + } + path = f.Name() + } + // open up a zip reader for the file + zipFile, err := zip.OpenReader(path) + if err != nil { + return fmt.Errorf("could not open zip for reading: %w", err) + } + defer zipFile.Close() + + // upsert any new plugins. + err = m.traverseAndUpsertPlugins(ctx, zipFile) + if err != nil { + return fmt.Errorf("could not upsert plugins: %w", err) + } + + return nil +} + +// downloadZip is a helper function that downloads a zip from a URL and +// writes it to a path in the local filesystem. +func downloadZip(url, path string) error { + response, err := http.Get(url) + if err != nil { + return fmt.Errorf("could not get zip from url: %w", err) + } + defer response.Body.Close() + + // Create the file on the local FS. If it exists, it will be truncated. + output, err := os.Create(path) + if err != nil { + return fmt.Errorf("could not create output file: %w", err) + } + defer output.Close() + + // Copy the zip output to the file. + _, err = io.Copy(output, response.Body) + if err != nil { + return fmt.Errorf("could not copy response body output to file: %w", err) + } + + return nil +} + +// traverseAndUpsertPlugins traverses through the zip and upserts plugins into the database +// if they are not present. +func (m *PluginManager) traverseAndUpsertPlugins(ctx context.Context, rc *zip.ReadCloser) error { + plugins, err := m.pluginStore.ListAll(ctx) + if err != nil { + return fmt.Errorf("could not list plugins: %w", err) + } + // Put the plugins in a map so we don't have to perform frequent DB queries. + pluginMap := map[string]*types.Plugin{} + for _, p := range plugins { + pluginMap[p.UID] = p + } + cnt := 0 + for _, file := range rc.File { + matched, err := filepath.Match("**/plugins/*/*.yaml", file.Name) + if err != nil { // only returns BadPattern error which shouldn't happen + return fmt.Errorf("could not glob pattern: %w", err) + } + if !matched { + continue + } + fc, err := file.Open() + if err != nil { + log.Warn().Err(err).Str("name", file.Name).Msg("could not open file") + continue + } + defer fc.Close() + var buf bytes.Buffer + _, err = io.Copy(&buf, fc) + if err != nil { + log.Warn().Err(err).Str("name", file.Name).Msg("could not read file contents") + continue + } + // schema should be a valid config - if not log an error and continue. + config, err := parse.ParseBytes(buf.Bytes()) + if err != nil { + log.Warn().Err(err).Str("name", file.Name).Msg("could not parse schema into valid config") + continue + } + + var desc string + switch vv := config.Spec.(type) { + case *v1yaml.PluginStep: + desc = vv.Description + case *v1yaml.PluginStage: + desc = vv.Description + default: + log.Warn().Str("name", file.Name).Msg("schema did not match a valid plugin schema") + continue + } + + plugin := &types.Plugin{ + Description: desc, + UID: config.Name, + Type: config.Type, + Spec: buf.String(), + } + + // Try to read the logo if it exists in the same directory + dir := filepath.Dir(file.Name) + logoFile := filepath.Join(dir, "logo.svg") + if lf, err := rc.Open(logoFile); err == nil { // if we can open the logo file + var lbuf bytes.Buffer + _, err = io.Copy(&lbuf, lf) + if err != nil { + log.Warn().Err(err).Str("name", file.Name).Msg("could not copy logo file") + } else { + plugin.Logo = lbuf.String() + } + } + + // If plugin already exists in the database, skip upsert + if p, ok := pluginMap[plugin.UID]; ok { + if p.Matches(plugin) { + continue + } + + } + + // If plugin name exists with a different spec, call update - otherwise call create. + // TODO: Once we start using versions, we can think of whether we want to + // keep different schemas for each version in the database. For now, we will + // simply overwrite the existing version with the new version. + if _, ok := pluginMap[plugin.UID]; ok { + err = m.pluginStore.Update(ctx, plugin) + if err != nil { + log.Warn().Str("name", file.Name).Err(err).Msg("could not update plugin") + continue + } + log.Info().Str("name", file.Name).Msg("detected changes: updated existing plugin entry") + } else { + err = m.pluginStore.Create(ctx, plugin) + if err != nil { + log.Warn().Str("name", file.Name).Err(err).Msg("could not create plugin in DB") + continue + } + cnt++ + } + } + log.Info().Msgf("added %d new entries to plugins", cnt) + return nil +} diff --git a/internal/pipeline/plugin/wire.go b/internal/pipeline/plugin/wire.go new file mode 100644 index 000000000..646d5ec62 --- /dev/null +++ b/internal/pipeline/plugin/wire.go @@ -0,0 +1,25 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package plugin + +import ( + "github.com/harness/gitness/internal/store" + "github.com/harness/gitness/types" + + "github.com/google/wire" +) + +// WireSet provides a wire set for this package. +var WireSet = wire.NewSet( + ProvidePluginManager, +) + +// ProvidePluginManager provides an execution runner. +func ProvidePluginManager( + config *types.Config, + pluginStore store.PluginStore, +) *PluginManager { + return NewPluginManager(config, pluginStore) +} diff --git a/internal/pipeline/runner/runner.go b/internal/pipeline/runner/runner.go index 067503db6..8d468361b 100644 --- a/internal/pipeline/runner/runner.go +++ b/internal/pipeline/runner/runner.go @@ -6,6 +6,7 @@ package runner import ( "github.com/harness/gitness/internal/pipeline/manager" + "github.com/harness/gitness/internal/pipeline/plugin" "github.com/harness/gitness/types" "github.com/drone-runners/drone-runner-docker/engine" @@ -29,6 +30,7 @@ import ( func NewExecutionRunner( config *types.Config, client runnerclient.Client, + pluginManager *plugin.PluginManager, m manager.ExecutionManager, ) (*runtime2.Runner, error) { // For linux, containers need to have extra hosts set in order to interact with @@ -79,6 +81,7 @@ func NewExecutionRunner( runner := &runtime2.Runner{ Machine: config.InstanceID, Client: client, + Resolver: pluginManager.GetLookupFn(), Reporter: tracer, Compiler: compiler2, Exec: exec2.Exec, diff --git a/internal/pipeline/runner/wire.go b/internal/pipeline/runner/wire.go index 474a9c721..09c03bdb2 100644 --- a/internal/pipeline/runner/wire.go +++ b/internal/pipeline/runner/wire.go @@ -5,10 +5,11 @@ package runner import ( - runtime2 "github.com/drone-runners/drone-runner-docker/engine2/runtime" "github.com/harness/gitness/internal/pipeline/manager" + "github.com/harness/gitness/internal/pipeline/plugin" "github.com/harness/gitness/types" + runtime2 "github.com/drone-runners/drone-runner-docker/engine2/runtime" runnerclient "github.com/drone/runner-go/client" "github.com/drone/runner-go/poller" "github.com/google/wire" @@ -24,9 +25,10 @@ var WireSet = wire.NewSet( func ProvideExecutionRunner( config *types.Config, client runnerclient.Client, + pluginManager *plugin.PluginManager, manager manager.ExecutionManager, ) (*runtime2.Runner, error) { - return NewExecutionRunner(config, client, manager) + return NewExecutionRunner(config, client, pluginManager, manager) } // ProvideExecutionPoller provides a poller which can poll the manager diff --git a/internal/pipeline/triggerer/trigger.go b/internal/pipeline/triggerer/trigger.go index db3b23552..f10e4726f 100644 --- a/internal/pipeline/triggerer/trigger.go +++ b/internal/pipeline/triggerer/trigger.go @@ -11,10 +11,6 @@ import ( "runtime/debug" "time" - v1yaml "github.com/drone/spec/dist/go" - "github.com/drone/spec/dist/go/parse/expand" - "github.com/drone/spec/dist/go/parse/normalize" - "github.com/drone/spec/dist/go/parse/script" "github.com/harness/gitness/internal/pipeline/checks" "github.com/harness/gitness/internal/pipeline/file" "github.com/harness/gitness/internal/pipeline/scheduler" @@ -24,8 +20,12 @@ import ( "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" + "github.com/drone-runners/drone-runner-docker/engine2/script" "github.com/drone/drone-yaml/yaml" "github.com/drone/drone-yaml/yaml/linter" + v1yaml "github.com/drone/spec/dist/go" + "github.com/drone/spec/dist/go/parse/expand" + "github.com/drone/spec/dist/go/parse/normalize" "github.com/jmoiron/sqlx" "github.com/rs/zerolog/log" ) diff --git a/internal/store/database.go b/internal/store/database.go index bfb13c8f1..1dfcef2f2 100644 --- a/internal/store/database.go +++ b/internal/store/database.go @@ -698,10 +698,19 @@ type ( // along with their associated schemas. List(ctx context.Context, filter types.ListQueryFilter) ([]*types.Plugin, error) + // ListAll returns back the full list of plugins. + ListAll(ctx context.Context) ([]*types.Plugin, error) + // Create creates a new entry in the plugin datastore. Create(ctx context.Context, plugin *types.Plugin) error + // Update tries to update an trigger. + Update(ctx context.Context, plugin *types.Plugin) error + // Count counts the number of plugins matching the given filter. Count(ctx context.Context, filter types.ListQueryFilter) (int64, error) + + // Find returns a plugin given a name and a version. + Find(ctx context.Context, name, version string) (*types.Plugin, error) } ) diff --git a/internal/store/database/migrate/postgres/0033_alter_ci_tables.up.sql b/internal/store/database/migrate/postgres/0033_alter_ci_tables.up.sql new file mode 100644 index 000000000..68980e2a7 --- /dev/null +++ b/internal/store/database/migrate/postgres/0033_alter_ci_tables.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE plugins + ADD COLUMN plugin_type TEXT NOT NULL, + ADD COLUMN plugin_version TEXT NOT NULL; \ No newline at end of file diff --git a/internal/store/database/migrate/sqlite/0033_alter_ci_tables.up.sql b/internal/store/database/migrate/sqlite/0033_alter_ci_tables.up.sql new file mode 100644 index 000000000..9dae2d417 --- /dev/null +++ b/internal/store/database/migrate/sqlite/0033_alter_ci_tables.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE plugins ADD COLUMN plugin_type TEXT NOT NULL; +ALTER TABLE plugins ADD COLUMN plugin_version TEXT NOT NULL; \ No newline at end of file diff --git a/internal/store/database/plugin.go b/internal/store/database/plugin.go index 4030954a3..6415622ef 100644 --- a/internal/store/database/plugin.go +++ b/internal/store/database/plugin.go @@ -24,6 +24,8 @@ const ( pluginColumns = ` plugin_uid ,plugin_description + ,plugin_type + ,plugin_version ,plugin_logo ,plugin_spec ` @@ -46,14 +48,18 @@ func (s *pluginStore) Create(ctx context.Context, plugin *types.Plugin) error { INSERT INTO plugins ( plugin_uid ,plugin_description + ,plugin_type + ,plugin_version ,plugin_logo ,plugin_spec ) VALUES ( :plugin_uid ,:plugin_description + ,:plugin_type + ,:plugin_version ,:plugin_logo ,:plugin_spec - )` + ) RETURNING plugin_uid` db := dbtx.GetAccessor(ctx, s.db) @@ -69,6 +75,23 @@ func (s *pluginStore) Create(ctx context.Context, plugin *types.Plugin) error { return nil } +// Find finds a version of a plugin +func (s *pluginStore) Find(ctx context.Context, name, version string) (*types.Plugin, error) { + const pluginFindStmt = ` + SELECT` + pluginColumns + + `FROM plugins + WHERE plugin_uid = $1 AND plugin_version = $2 + ` + db := dbtx.GetAccessor(ctx, s.db) + + dst := new(types.Plugin) + if err := db.GetContext(ctx, dst, pluginFindStmt, name, version); err != nil { + return nil, database.ProcessSQLErrorf(err, "Failed to find pipeline") + } + + return dst, nil +} + // List returns back the list of plugins along with their associated schemas. func (s *pluginStore) List( ctx context.Context, @@ -100,6 +123,29 @@ func (s *pluginStore) List( return dst, nil } +// ListAll returns back the full list of plugins in the database. +func (s *pluginStore) ListAll( + ctx context.Context, +) ([]*types.Plugin, error) { + stmt := database.Builder. + Select(pluginColumns). + From("plugins") + + sql, args, err := stmt.ToSql() + if err != nil { + return nil, errors.Wrap(err, "Failed to convert query to sql") + } + + db := dbtx.GetAccessor(ctx, s.db) + + dst := []*types.Plugin{} + if err = db.SelectContext(ctx, &dst, sql, args...); err != nil { + return nil, database.ProcessSQLErrorf(err, "Failed executing custom list query") + } + + return dst, nil +} + // Count of plugins matching the filter criteria. func (s *pluginStore) Count(ctx context.Context, filter types.ListQueryFilter) (int64, error) { stmt := database.Builder. @@ -124,3 +170,29 @@ func (s *pluginStore) Count(ctx context.Context, filter types.ListQueryFilter) ( } return count, nil } + +// Update updates a plugin row. +func (s *pluginStore) Update(ctx context.Context, p *types.Plugin) error { + const pluginUpdateStmt = ` + UPDATE plugins + SET + plugin_description = :plugin_description + ,plugin_type = :plugin_type + ,plugin_version = :plugin_version + ,plugin_logo = :plugin_logo + ,plugin_spec = :plugin_spec + WHERE plugin_uid = :plugin_uid` + db := dbtx.GetAccessor(ctx, s.db) + + query, arg, err := db.BindNamed(pluginUpdateStmt, p) + if err != nil { + return database.ProcessSQLErrorf(err, "Failed to bind plugin object") + } + + _, err = db.ExecContext(ctx, query, arg...) + if err != nil { + return database.ProcessSQLErrorf(err, "Failed to update plugin") + } + + return nil +} diff --git a/types/config.go b/types/config.go index d0c76dd76..5bbd2ecae 100644 --- a/types/config.go +++ b/types/config.go @@ -89,6 +89,9 @@ type Config struct { // CI defines configuration related to build executions. CI struct { ParallelWorkers int `envconfig:"GITNESS_CI_PARALLEL_WORKERS" default:"2"` + // PluginsZipPath is a pointer to a zip containing all the plugins schemas. + // This could be a local path or an external location. + PluginsZipPath string `envconfig:"GITNESS_CI_PLUGINS_ZIP_PATH" default:"https://github.com/bradrydzewski/plugins/archive/refs/heads/master.zip"` } // Database defines the database configuration parameters. diff --git a/types/plugin.go b/types/plugin.go index c58a6e54f..367cc36b7 100644 --- a/types/plugin.go +++ b/types/plugin.go @@ -7,9 +7,34 @@ package types // in the spec field. The spec is used by the UI to provide a smart visual // editor for adding plugins to YAML schema. type Plugin struct { - UID string `db:"plugin_uid" json:"uid"` + UID string `db:"plugin_uid" json:"uid"` Description string `db:"plugin_description" json:"description"` - Logo string `db:"plugin_logo" json:"logo"` + // Currently we only support step level plugins but more can be added in the future. + Type string `db:"plugin_type" json:"type"` + Version string `db:"plugin_version" json:"version"` + Logo string `db:"plugin_logo" json:"logo"` // Spec is a YAML template to be used for the plugin. Spec string `db:"plugin_spec" json:"spec"` } + +// Matches checks whether two plugins are identical. +// We can use reflection here, this is just easier to add on to +// when needed. +func (plugin *Plugin) Matches(v *Plugin) bool { + if plugin.UID != v.UID { + return false + } + if plugin.Description != v.Description { + return false + } + if plugin.Spec != v.Spec { + return false + } + if plugin.Version != v.Version { + return false + } + if plugin.Logo != v.Logo { + return false + } + return true +}