mirror of https://github.com/harness/drone.git
add plugins support for v1 YAML (#552)
Vistaar Juneja
Harness
parent
40501293d9
commit
40af7bfa33
|
|
@ -89,6 +89,14 @@ func (c *command) run(*kingpin.ParseContext) error {
|
|||
gHTTP, shutdownHTTP := system.server.ListenAndServe()
|
||||
g.Go(gHTTP.Wait)
|
||||
if c.enableCI {
|
||||
// start populating plugins
|
||||
g.Go(func() error {
|
||||
err := system.pluginManager.Populate(ctx)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("could not populate plugins")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// start poller for CI build executions.
|
||||
g.Go(func() error {
|
||||
log := logrus.New()
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
gitrpcserver "github.com/harness/gitness/gitrpc/server"
|
||||
gitrpccron "github.com/harness/gitness/gitrpc/server/cron"
|
||||
"github.com/harness/gitness/internal/bootstrap"
|
||||
"github.com/harness/gitness/internal/pipeline/plugin"
|
||||
"github.com/harness/gitness/internal/server"
|
||||
"github.com/harness/gitness/internal/services"
|
||||
|
||||
|
|
@ -19,19 +20,22 @@ type System struct {
|
|||
bootstrap bootstrap.Bootstrap
|
||||
server *server.Server
|
||||
gitRPCServer *gitrpcserver.GRPCServer
|
||||
pluginManager *plugin.PluginManager
|
||||
poller *poller.Poller
|
||||
services services.Services
|
||||
gitRPCCronMngr *gitrpccron.Manager
|
||||
}
|
||||
|
||||
// NewSystem returns a new system structure.
|
||||
func NewSystem(bootstrap bootstrap.Bootstrap, server *server.Server, poller *poller.Poller, gitRPCServer *gitrpcserver.GRPCServer,
|
||||
func NewSystem(bootstrap bootstrap.Bootstrap, server *server.Server, poller *poller.Poller,
|
||||
gitRPCServer *gitrpcserver.GRPCServer, pluginManager *plugin.PluginManager,
|
||||
gitrpccron *gitrpccron.Manager, services services.Services) *System {
|
||||
return &System{
|
||||
bootstrap: bootstrap,
|
||||
server: server,
|
||||
poller: poller,
|
||||
gitRPCServer: gitRPCServer,
|
||||
pluginManager: pluginManager,
|
||||
services: services,
|
||||
gitRPCCronMngr: gitrpccron,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,8 +10,6 @@ package main
|
|||
import (
|
||||
"context"
|
||||
|
||||
"github.com/harness/gitness/internal/services/exporter"
|
||||
|
||||
cliserver "github.com/harness/gitness/cli/server"
|
||||
"github.com/harness/gitness/encrypt"
|
||||
"github.com/harness/gitness/events"
|
||||
|
|
@ -46,6 +44,7 @@ import (
|
|||
"github.com/harness/gitness/internal/pipeline/commit"
|
||||
"github.com/harness/gitness/internal/pipeline/file"
|
||||
"github.com/harness/gitness/internal/pipeline/manager"
|
||||
pluginmanager "github.com/harness/gitness/internal/pipeline/plugin"
|
||||
"github.com/harness/gitness/internal/pipeline/runner"
|
||||
"github.com/harness/gitness/internal/pipeline/scheduler"
|
||||
"github.com/harness/gitness/internal/pipeline/triggerer"
|
||||
|
|
@ -53,6 +52,7 @@ import (
|
|||
"github.com/harness/gitness/internal/server"
|
||||
"github.com/harness/gitness/internal/services"
|
||||
"github.com/harness/gitness/internal/services/codecomments"
|
||||
"github.com/harness/gitness/internal/services/exporter"
|
||||
"github.com/harness/gitness/internal/services/importer"
|
||||
"github.com/harness/gitness/internal/services/job"
|
||||
"github.com/harness/gitness/internal/services/metric"
|
||||
|
|
@ -138,6 +138,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e
|
|||
commit.WireSet,
|
||||
controllertrigger.WireSet,
|
||||
plugin.WireSet,
|
||||
pluginmanager.WireSet,
|
||||
importer.WireSet,
|
||||
canceler.WireSet,
|
||||
exporter.WireSet,
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import (
|
|||
"github.com/harness/gitness/internal/pipeline/commit"
|
||||
"github.com/harness/gitness/internal/pipeline/file"
|
||||
"github.com/harness/gitness/internal/pipeline/manager"
|
||||
plugin2 "github.com/harness/gitness/internal/pipeline/plugin"
|
||||
"github.com/harness/gitness/internal/pipeline/runner"
|
||||
"github.com/harness/gitness/internal/pipeline/scheduler"
|
||||
"github.com/harness/gitness/internal/pipeline/triggerer"
|
||||
|
|
@ -223,7 +224,8 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
|
|||
serverServer := server2.ProvideServer(config, routerRouter)
|
||||
executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, streamer, fileService, logStore, logStream, checkStore, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore)
|
||||
client := manager.ProvideExecutionClient(executionManager, config)
|
||||
runtimeRunner, err := runner.ProvideExecutionRunner(config, client, executionManager)
|
||||
pluginManager := plugin2.ProvidePluginManager(config, pluginStore)
|
||||
runtimeRunner, err := runner.ProvideExecutionRunner(config, client, pluginManager, executionManager)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -253,6 +255,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
|
|||
return nil, err
|
||||
}
|
||||
servicesServices := services.ProvideServices(webhookService, pullreqService, triggerService, jobScheduler, collector)
|
||||
serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, poller, grpcServer, cronManager, servicesServices)
|
||||
serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, poller, grpcServer, pluginManager, cronManager, servicesServices)
|
||||
return serverSystem, nil
|
||||
}
|
||||
|
|
|
|||
10
go.mod
10
go.mod
|
|
@ -12,15 +12,15 @@ require (
|
|||
github.com/coreos/go-semver v0.3.0
|
||||
github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible
|
||||
github.com/drone-runners/drone-runner-docker v1.8.4-0.20230918204859-ce45c209ba31
|
||||
github.com/drone-runners/drone-runner-docker v1.8.4-0.20230919202034-23803f6b38c2
|
||||
github.com/drone/drone-go v1.7.1
|
||||
github.com/drone/drone-yaml v1.2.3
|
||||
github.com/drone/funcmap v0.0.0-20190918184546-d4ef6e88376d
|
||||
github.com/drone/go-convert v0.0.0-20230913194237-2228eb829d0e
|
||||
github.com/drone/go-generate v0.0.0-20230916194845-e98bb2a7f93a
|
||||
github.com/drone/go-convert v0.0.0-20230919093251-7104c3bcc635
|
||||
github.com/drone/go-generate v0.0.0-20230919103306-db4551429a31
|
||||
github.com/drone/go-scm v1.31.2
|
||||
github.com/drone/runner-go v1.12.0
|
||||
github.com/drone/spec v0.0.0-20230918031032-07d0f2594649
|
||||
github.com/drone/spec v0.0.0-20230919004456-7455b8913ff5
|
||||
github.com/go-chi/chi v1.5.4
|
||||
github.com/go-chi/cors v1.2.1
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
|
|
@ -70,6 +70,7 @@ require (
|
|||
cloud.google.com/go/compute v1.18.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
dario.cat/mergo v1.0.0 // indirect
|
||||
github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d // indirect
|
||||
github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e // indirect
|
||||
github.com/antonmedv/expr v1.15.2 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
|
|
@ -83,6 +84,7 @@ require (
|
|||
github.com/docker/go-connections v0.3.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/drone/envsubst v1.0.3 // indirect
|
||||
github.com/drone/signal v1.0.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/ghodss/yaml v1.0.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
|
|
|
|||
14
go.sum
14
go.sum
|
|
@ -16,6 +16,7 @@ code.gitea.io/gitea v1.17.2/go.mod h1:sovminOoSsc8IC2T29rX9+MmaboHTu8QDEvJjaSqIX
|
|||
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||
docker.io/go-docker v1.0.0/go.mod h1:7tiAn5a0LFmjbPDbyTPOaTTOuG1ZRNXdPA6RvKY+fpY=
|
||||
github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d h1:j6oB/WPCigdOkxtuPl1VSIiLpy7Mdsu6phQffbF19Ng=
|
||||
github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs=
|
||||
github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e h1:rl2Aq4ZODqTDkeSqQBy+fzpZPamacO1Srp8zq7jf2Sc=
|
||||
github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e/go.mod h1:Xa6lInWHNQnuWoF0YPSsx+INFA9qk7/7pTjwb3PInkY=
|
||||
|
|
@ -137,6 +138,10 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
|
|||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/drone-runners/drone-runner-docker v1.8.4-0.20230918204859-ce45c209ba31 h1:lq8ysXgygXtA6QCL3GCM+3VrSSi+IToF5HSMkA1DNyY=
|
||||
github.com/drone-runners/drone-runner-docker v1.8.4-0.20230918204859-ce45c209ba31/go.mod h1:mlVgx9GWL+BvvgKiroR09XupiP+DEP8sWr/M8wDsH8Q=
|
||||
github.com/drone-runners/drone-runner-docker v1.8.4-0.20230919103848-e39029551c7e h1:zajv6sRMfij4QzvQfT4UHV1If6AhLa/6dSuQcvqktEo=
|
||||
github.com/drone-runners/drone-runner-docker v1.8.4-0.20230919103848-e39029551c7e/go.mod h1:iXTCJv+tESfI/ggWZwinI2ZAzHTGS+Ic5A9gcUElTns=
|
||||
github.com/drone-runners/drone-runner-docker v1.8.4-0.20230919202034-23803f6b38c2 h1:NmYT2bCmacG9mFWql0hKlJ7KOjfXDlusrpWbjUDYKoA=
|
||||
github.com/drone-runners/drone-runner-docker v1.8.4-0.20230919202034-23803f6b38c2/go.mod h1:iXTCJv+tESfI/ggWZwinI2ZAzHTGS+Ic5A9gcUElTns=
|
||||
github.com/drone/drone-go v1.7.1 h1:ZX+3Rs8YHUSUQ5mkuMLmm1zr1ttiiE2YGNxF3AnyDKw=
|
||||
github.com/drone/drone-go v1.7.1/go.mod h1:fxCf9jAnXDZV1yDr0ckTuWd1intvcQwfJmTRpTZ1mXg=
|
||||
github.com/drone/drone-runtime v1.0.7-0.20190729202838-87c84080f4a1/go.mod h1:+osgwGADc/nyl40J0fdsf8Z09bgcBZXvXXnLOY48zYs=
|
||||
|
|
@ -149,15 +154,20 @@ github.com/drone/funcmap v0.0.0-20190918184546-d4ef6e88376d h1:/IO7UVVu191Jc0Daj
|
|||
github.com/drone/funcmap v0.0.0-20190918184546-d4ef6e88376d/go.mod h1:Hph0/pT6ZxbujnE1Z6/08p5I0XXuOsppqF6NQlGOK0E=
|
||||
github.com/drone/go-convert v0.0.0-20230913194237-2228eb829d0e h1:H0eAzGW6vj4XCHnrAlmrcyQbg6g3yVcjfobUHzjNxGM=
|
||||
github.com/drone/go-convert v0.0.0-20230913194237-2228eb829d0e/go.mod h1:ciC4CjDBSrpHGWmodGdUQshx7P5lAqL6DFJBnf3zVWU=
|
||||
github.com/drone/go-convert v0.0.0-20230919093251-7104c3bcc635 h1:qQX+U2iEm4X2FcmBzxZwZgz8gLpUTa6lBB1vBBCV9Oo=
|
||||
github.com/drone/go-convert v0.0.0-20230919093251-7104c3bcc635/go.mod h1:PyCDcuAhGF6W0VJ6qMmlM47dsSyGv/zDiMqeJxMFuGM=
|
||||
github.com/drone/go-generate v0.0.0-20230916194845-e98bb2a7f93a h1:kL6IAktFq7P3sRtbnMC/L6GBwJe+IzPKezk0HGq22nE=
|
||||
github.com/drone/go-generate v0.0.0-20230916194845-e98bb2a7f93a/go.mod h1:WbpwzOT2+x4z59zMUA2lRzuqzZ87/QPIeLfkEEi1FUI=
|
||||
github.com/drone/go-generate v0.0.0-20230919103306-db4551429a31 h1:/z0vz1tJ5t5D6RPug2IZm8QwoYPrXYt/p4Le6B5hJl4=
|
||||
github.com/drone/go-generate v0.0.0-20230919103306-db4551429a31/go.mod h1:eTfy716efMJgVvk/ZkRvitaXY2UuytfqDjxclFMeLdQ=
|
||||
github.com/drone/go-scm v1.31.2 h1:6hZxf0aETV17830fMCPrgcA4y8j/8Gdfy0xEdInUeqQ=
|
||||
github.com/drone/go-scm v1.31.2/go.mod h1:DFIJJjhMj0TSXPz+0ni4nyZ9gtTtC40Vh/TGRugtyWw=
|
||||
github.com/drone/runner-go v1.12.0 h1:zUjDj9ylsJ4n4Mvy4znddq/Z4EBzcUXzTltpzokKtgs=
|
||||
github.com/drone/runner-go v1.12.0/go.mod h1:vu4pPPYDoeN6vdYQAY01GGGsAIW4aLganJNaa8Fx8zE=
|
||||
github.com/drone/signal v1.0.0 h1:NrnM2M/4yAuU/tXs6RP1a1ZfxnaHwYkd0kJurA1p6uI=
|
||||
github.com/drone/signal v1.0.0/go.mod h1:S8t92eFT0g4WUgEc/LxG+LCuiskpMNsG0ajAMGnyZpc=
|
||||
github.com/drone/spec v0.0.0-20230918031032-07d0f2594649 h1:e22spVEjx7WNwRnKXIZ4eUVEmICWb/L92JZxezregQs=
|
||||
github.com/drone/spec v0.0.0-20230918031032-07d0f2594649/go.mod h1:KyQZA9qwuscbbM7yTrtZg25Wammoc5GKwaRem8kDA5k=
|
||||
github.com/drone/spec v0.0.0-20230919004456-7455b8913ff5 h1:NgAseJNQpJE3XtgJUPu4x7x5fcBjqZ3oKHDJfwBYdWk=
|
||||
github.com/drone/spec v0.0.0-20230919004456-7455b8913ff5/go.mod h1:KyQZA9qwuscbbM7yTrtZg25Wammoc5GKwaRem8kDA5k=
|
||||
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ import (
|
|||
gitness_store "github.com/harness/gitness/store"
|
||||
"github.com/harness/gitness/types"
|
||||
"github.com/harness/gitness/types/enum"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,230 @@
|
|||
// Copyright 2022 Harness Inc. All rights reserved.
|
||||
// Use of this source code is governed by the Polyform Free Trial License
|
||||
// that can be found in the LICENSE.md file for this repository.
|
||||
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/harness/gitness/internal/store"
|
||||
"github.com/harness/gitness/types"
|
||||
|
||||
v1yaml "github.com/drone/spec/dist/go"
|
||||
"github.com/drone/spec/dist/go/parse"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// Lookup returns a resource by name, kind and type.
|
||||
type LookupFunc func(name, kind, typ, version string) (*v1yaml.Config, error)
|
||||
|
||||
type PluginManager struct {
|
||||
config *types.Config
|
||||
pluginStore store.PluginStore
|
||||
}
|
||||
|
||||
func NewPluginManager(
|
||||
config *types.Config,
|
||||
pluginStore store.PluginStore,
|
||||
) *PluginManager {
|
||||
return &PluginManager{
|
||||
config: config,
|
||||
pluginStore: pluginStore,
|
||||
}
|
||||
}
|
||||
|
||||
// GetLookupFn returns a lookup function for plugins which can be used in the resolver.
|
||||
func (m *PluginManager) GetLookupFn() LookupFunc {
|
||||
return func(name, kind, typ, version string) (*v1yaml.Config, error) {
|
||||
if kind != "plugin" {
|
||||
return nil, fmt.Errorf("only plugin kind supported")
|
||||
}
|
||||
if typ != "step" {
|
||||
return nil, fmt.Errorf("only step plugins supported")
|
||||
}
|
||||
plugin, err := m.pluginStore.Find(context.Background(), name, version)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not lookup plugin: %w", err)
|
||||
}
|
||||
// Convert plugin to v1yaml spec
|
||||
config, err := parse.ParseString(plugin.Spec)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not unmarshal plugin to v1yaml spec: %w", err)
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Populate fetches plugins information from an external source or a local zip
|
||||
// and populates in the DB.
|
||||
func (m *PluginManager) Populate(ctx context.Context) error {
|
||||
path := m.config.CI.PluginsZipPath
|
||||
if path == "" {
|
||||
return fmt.Errorf("plugins path not provided to read schemas from")
|
||||
}
|
||||
|
||||
var zipFile *zip.ReadCloser
|
||||
if _, err := os.Stat(path); err != nil { // local path doesn't exist - must be a remote link
|
||||
// Download zip file locally
|
||||
f, err := os.CreateTemp(os.TempDir(), "plugins.zip")
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create temp file: %w", err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
err = downloadZip(path, f.Name())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not download remote zip: %w", err)
|
||||
}
|
||||
path = f.Name()
|
||||
}
|
||||
// open up a zip reader for the file
|
||||
zipFile, err := zip.OpenReader(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open zip for reading: %w", err)
|
||||
}
|
||||
defer zipFile.Close()
|
||||
|
||||
// upsert any new plugins.
|
||||
err = m.traverseAndUpsertPlugins(ctx, zipFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not upsert plugins: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// downloadZip is a helper function that downloads a zip from a URL and
|
||||
// writes it to a path in the local filesystem.
|
||||
func downloadZip(url, path string) error {
|
||||
response, err := http.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get zip from url: %w", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
// Create the file on the local FS. If it exists, it will be truncated.
|
||||
output, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create output file: %w", err)
|
||||
}
|
||||
defer output.Close()
|
||||
|
||||
// Copy the zip output to the file.
|
||||
_, err = io.Copy(output, response.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not copy response body output to file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// traverseAndUpsertPlugins traverses through the zip and upserts plugins into the database
|
||||
// if they are not present.
|
||||
func (m *PluginManager) traverseAndUpsertPlugins(ctx context.Context, rc *zip.ReadCloser) error {
|
||||
plugins, err := m.pluginStore.ListAll(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not list plugins: %w", err)
|
||||
}
|
||||
// Put the plugins in a map so we don't have to perform frequent DB queries.
|
||||
pluginMap := map[string]*types.Plugin{}
|
||||
for _, p := range plugins {
|
||||
pluginMap[p.UID] = p
|
||||
}
|
||||
cnt := 0
|
||||
for _, file := range rc.File {
|
||||
matched, err := filepath.Match("**/plugins/*/*.yaml", file.Name)
|
||||
if err != nil { // only returns BadPattern error which shouldn't happen
|
||||
return fmt.Errorf("could not glob pattern: %w", err)
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
fc, err := file.Open()
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("name", file.Name).Msg("could not open file")
|
||||
continue
|
||||
}
|
||||
defer fc.Close()
|
||||
var buf bytes.Buffer
|
||||
_, err = io.Copy(&buf, fc)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("name", file.Name).Msg("could not read file contents")
|
||||
continue
|
||||
}
|
||||
// schema should be a valid config - if not log an error and continue.
|
||||
config, err := parse.ParseBytes(buf.Bytes())
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("name", file.Name).Msg("could not parse schema into valid config")
|
||||
continue
|
||||
}
|
||||
|
||||
var desc string
|
||||
switch vv := config.Spec.(type) {
|
||||
case *v1yaml.PluginStep:
|
||||
desc = vv.Description
|
||||
case *v1yaml.PluginStage:
|
||||
desc = vv.Description
|
||||
default:
|
||||
log.Warn().Str("name", file.Name).Msg("schema did not match a valid plugin schema")
|
||||
continue
|
||||
}
|
||||
|
||||
plugin := &types.Plugin{
|
||||
Description: desc,
|
||||
UID: config.Name,
|
||||
Type: config.Type,
|
||||
Spec: buf.String(),
|
||||
}
|
||||
|
||||
// Try to read the logo if it exists in the same directory
|
||||
dir := filepath.Dir(file.Name)
|
||||
logoFile := filepath.Join(dir, "logo.svg")
|
||||
if lf, err := rc.Open(logoFile); err == nil { // if we can open the logo file
|
||||
var lbuf bytes.Buffer
|
||||
_, err = io.Copy(&lbuf, lf)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("name", file.Name).Msg("could not copy logo file")
|
||||
} else {
|
||||
plugin.Logo = lbuf.String()
|
||||
}
|
||||
}
|
||||
|
||||
// If plugin already exists in the database, skip upsert
|
||||
if p, ok := pluginMap[plugin.UID]; ok {
|
||||
if p.Matches(plugin) {
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// If plugin name exists with a different spec, call update - otherwise call create.
|
||||
// TODO: Once we start using versions, we can think of whether we want to
|
||||
// keep different schemas for each version in the database. For now, we will
|
||||
// simply overwrite the existing version with the new version.
|
||||
if _, ok := pluginMap[plugin.UID]; ok {
|
||||
err = m.pluginStore.Update(ctx, plugin)
|
||||
if err != nil {
|
||||
log.Warn().Str("name", file.Name).Err(err).Msg("could not update plugin")
|
||||
continue
|
||||
}
|
||||
log.Info().Str("name", file.Name).Msg("detected changes: updated existing plugin entry")
|
||||
} else {
|
||||
err = m.pluginStore.Create(ctx, plugin)
|
||||
if err != nil {
|
||||
log.Warn().Str("name", file.Name).Err(err).Msg("could not create plugin in DB")
|
||||
continue
|
||||
}
|
||||
cnt++
|
||||
}
|
||||
}
|
||||
log.Info().Msgf("added %d new entries to plugins", cnt)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
// Copyright 2022 Harness Inc. All rights reserved.
|
||||
// Use of this source code is governed by the Polyform Free Trial License
|
||||
// that can be found in the LICENSE.md file for this repository.
|
||||
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"github.com/harness/gitness/internal/store"
|
||||
"github.com/harness/gitness/types"
|
||||
|
||||
"github.com/google/wire"
|
||||
)
|
||||
|
||||
// WireSet provides a wire set for this package.
|
||||
var WireSet = wire.NewSet(
|
||||
ProvidePluginManager,
|
||||
)
|
||||
|
||||
// ProvidePluginManager provides an execution runner.
|
||||
func ProvidePluginManager(
|
||||
config *types.Config,
|
||||
pluginStore store.PluginStore,
|
||||
) *PluginManager {
|
||||
return NewPluginManager(config, pluginStore)
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ package runner
|
|||
|
||||
import (
|
||||
"github.com/harness/gitness/internal/pipeline/manager"
|
||||
"github.com/harness/gitness/internal/pipeline/plugin"
|
||||
"github.com/harness/gitness/types"
|
||||
|
||||
"github.com/drone-runners/drone-runner-docker/engine"
|
||||
|
|
@ -29,6 +30,7 @@ import (
|
|||
func NewExecutionRunner(
|
||||
config *types.Config,
|
||||
client runnerclient.Client,
|
||||
pluginManager *plugin.PluginManager,
|
||||
m manager.ExecutionManager,
|
||||
) (*runtime2.Runner, error) {
|
||||
// For linux, containers need to have extra hosts set in order to interact with
|
||||
|
|
@ -79,6 +81,7 @@ func NewExecutionRunner(
|
|||
runner := &runtime2.Runner{
|
||||
Machine: config.InstanceID,
|
||||
Client: client,
|
||||
Resolver: pluginManager.GetLookupFn(),
|
||||
Reporter: tracer,
|
||||
Compiler: compiler2,
|
||||
Exec: exec2.Exec,
|
||||
|
|
|
|||
|
|
@ -5,10 +5,11 @@
|
|||
package runner
|
||||
|
||||
import (
|
||||
runtime2 "github.com/drone-runners/drone-runner-docker/engine2/runtime"
|
||||
"github.com/harness/gitness/internal/pipeline/manager"
|
||||
"github.com/harness/gitness/internal/pipeline/plugin"
|
||||
"github.com/harness/gitness/types"
|
||||
|
||||
runtime2 "github.com/drone-runners/drone-runner-docker/engine2/runtime"
|
||||
runnerclient "github.com/drone/runner-go/client"
|
||||
"github.com/drone/runner-go/poller"
|
||||
"github.com/google/wire"
|
||||
|
|
@ -24,9 +25,10 @@ var WireSet = wire.NewSet(
|
|||
func ProvideExecutionRunner(
|
||||
config *types.Config,
|
||||
client runnerclient.Client,
|
||||
pluginManager *plugin.PluginManager,
|
||||
manager manager.ExecutionManager,
|
||||
) (*runtime2.Runner, error) {
|
||||
return NewExecutionRunner(config, client, manager)
|
||||
return NewExecutionRunner(config, client, pluginManager, manager)
|
||||
}
|
||||
|
||||
// ProvideExecutionPoller provides a poller which can poll the manager
|
||||
|
|
|
|||
|
|
@ -11,10 +11,6 @@ import (
|
|||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
v1yaml "github.com/drone/spec/dist/go"
|
||||
"github.com/drone/spec/dist/go/parse/expand"
|
||||
"github.com/drone/spec/dist/go/parse/normalize"
|
||||
"github.com/drone/spec/dist/go/parse/script"
|
||||
"github.com/harness/gitness/internal/pipeline/checks"
|
||||
"github.com/harness/gitness/internal/pipeline/file"
|
||||
"github.com/harness/gitness/internal/pipeline/scheduler"
|
||||
|
|
@ -24,8 +20,12 @@ import (
|
|||
"github.com/harness/gitness/types"
|
||||
"github.com/harness/gitness/types/enum"
|
||||
|
||||
"github.com/drone-runners/drone-runner-docker/engine2/script"
|
||||
"github.com/drone/drone-yaml/yaml"
|
||||
"github.com/drone/drone-yaml/yaml/linter"
|
||||
v1yaml "github.com/drone/spec/dist/go"
|
||||
"github.com/drone/spec/dist/go/parse/expand"
|
||||
"github.com/drone/spec/dist/go/parse/normalize"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -698,10 +698,19 @@ type (
|
|||
// along with their associated schemas.
|
||||
List(ctx context.Context, filter types.ListQueryFilter) ([]*types.Plugin, error)
|
||||
|
||||
// ListAll returns back the full list of plugins.
|
||||
ListAll(ctx context.Context) ([]*types.Plugin, error)
|
||||
|
||||
// Create creates a new entry in the plugin datastore.
|
||||
Create(ctx context.Context, plugin *types.Plugin) error
|
||||
|
||||
// Update tries to update an trigger.
|
||||
Update(ctx context.Context, plugin *types.Plugin) error
|
||||
|
||||
// Count counts the number of plugins matching the given filter.
|
||||
Count(ctx context.Context, filter types.ListQueryFilter) (int64, error)
|
||||
|
||||
// Find returns a plugin given a name and a version.
|
||||
Find(ctx context.Context, name, version string) (*types.Plugin, error)
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
ALTER TABLE plugins
|
||||
ADD COLUMN plugin_type TEXT NOT NULL,
|
||||
ADD COLUMN plugin_version TEXT NOT NULL;
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE plugins ADD COLUMN plugin_type TEXT NOT NULL;
|
||||
ALTER TABLE plugins ADD COLUMN plugin_version TEXT NOT NULL;
|
||||
|
|
@ -24,6 +24,8 @@ const (
|
|||
pluginColumns = `
|
||||
plugin_uid
|
||||
,plugin_description
|
||||
,plugin_type
|
||||
,plugin_version
|
||||
,plugin_logo
|
||||
,plugin_spec
|
||||
`
|
||||
|
|
@ -46,14 +48,18 @@ func (s *pluginStore) Create(ctx context.Context, plugin *types.Plugin) error {
|
|||
INSERT INTO plugins (
|
||||
plugin_uid
|
||||
,plugin_description
|
||||
,plugin_type
|
||||
,plugin_version
|
||||
,plugin_logo
|
||||
,plugin_spec
|
||||
) VALUES (
|
||||
:plugin_uid
|
||||
,:plugin_description
|
||||
,:plugin_type
|
||||
,:plugin_version
|
||||
,:plugin_logo
|
||||
,:plugin_spec
|
||||
)`
|
||||
) RETURNING plugin_uid`
|
||||
|
||||
db := dbtx.GetAccessor(ctx, s.db)
|
||||
|
||||
|
|
@ -69,6 +75,23 @@ func (s *pluginStore) Create(ctx context.Context, plugin *types.Plugin) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Find finds a version of a plugin
|
||||
func (s *pluginStore) Find(ctx context.Context, name, version string) (*types.Plugin, error) {
|
||||
const pluginFindStmt = `
|
||||
SELECT` + pluginColumns +
|
||||
`FROM plugins
|
||||
WHERE plugin_uid = $1 AND plugin_version = $2
|
||||
`
|
||||
db := dbtx.GetAccessor(ctx, s.db)
|
||||
|
||||
dst := new(types.Plugin)
|
||||
if err := db.GetContext(ctx, dst, pluginFindStmt, name, version); err != nil {
|
||||
return nil, database.ProcessSQLErrorf(err, "Failed to find pipeline")
|
||||
}
|
||||
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// List returns back the list of plugins along with their associated schemas.
|
||||
func (s *pluginStore) List(
|
||||
ctx context.Context,
|
||||
|
|
@ -100,6 +123,29 @@ func (s *pluginStore) List(
|
|||
return dst, nil
|
||||
}
|
||||
|
||||
// ListAll returns back the full list of plugins in the database.
|
||||
func (s *pluginStore) ListAll(
|
||||
ctx context.Context,
|
||||
) ([]*types.Plugin, error) {
|
||||
stmt := database.Builder.
|
||||
Select(pluginColumns).
|
||||
From("plugins")
|
||||
|
||||
sql, args, err := stmt.ToSql()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Failed to convert query to sql")
|
||||
}
|
||||
|
||||
db := dbtx.GetAccessor(ctx, s.db)
|
||||
|
||||
dst := []*types.Plugin{}
|
||||
if err = db.SelectContext(ctx, &dst, sql, args...); err != nil {
|
||||
return nil, database.ProcessSQLErrorf(err, "Failed executing custom list query")
|
||||
}
|
||||
|
||||
return dst, nil
|
||||
}
|
||||
|
||||
// Count of plugins matching the filter criteria.
|
||||
func (s *pluginStore) Count(ctx context.Context, filter types.ListQueryFilter) (int64, error) {
|
||||
stmt := database.Builder.
|
||||
|
|
@ -124,3 +170,29 @@ func (s *pluginStore) Count(ctx context.Context, filter types.ListQueryFilter) (
|
|||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// Update updates a plugin row.
|
||||
func (s *pluginStore) Update(ctx context.Context, p *types.Plugin) error {
|
||||
const pluginUpdateStmt = `
|
||||
UPDATE plugins
|
||||
SET
|
||||
plugin_description = :plugin_description
|
||||
,plugin_type = :plugin_type
|
||||
,plugin_version = :plugin_version
|
||||
,plugin_logo = :plugin_logo
|
||||
,plugin_spec = :plugin_spec
|
||||
WHERE plugin_uid = :plugin_uid`
|
||||
db := dbtx.GetAccessor(ctx, s.db)
|
||||
|
||||
query, arg, err := db.BindNamed(pluginUpdateStmt, p)
|
||||
if err != nil {
|
||||
return database.ProcessSQLErrorf(err, "Failed to bind plugin object")
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(ctx, query, arg...)
|
||||
if err != nil {
|
||||
return database.ProcessSQLErrorf(err, "Failed to update plugin")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -89,6 +89,9 @@ type Config struct {
|
|||
// CI defines configuration related to build executions.
|
||||
CI struct {
|
||||
ParallelWorkers int `envconfig:"GITNESS_CI_PARALLEL_WORKERS" default:"2"`
|
||||
// PluginsZipPath is a pointer to a zip containing all the plugins schemas.
|
||||
// This could be a local path or an external location.
|
||||
PluginsZipPath string `envconfig:"GITNESS_CI_PLUGINS_ZIP_PATH" default:"https://github.com/bradrydzewski/plugins/archive/refs/heads/master.zip"`
|
||||
}
|
||||
|
||||
// Database defines the database configuration parameters.
|
||||
|
|
|
|||
|
|
@ -7,9 +7,34 @@ package types
|
|||
// in the spec field. The spec is used by the UI to provide a smart visual
|
||||
// editor for adding plugins to YAML schema.
|
||||
type Plugin struct {
|
||||
UID string `db:"plugin_uid" json:"uid"`
|
||||
UID string `db:"plugin_uid" json:"uid"`
|
||||
Description string `db:"plugin_description" json:"description"`
|
||||
Logo string `db:"plugin_logo" json:"logo"`
|
||||
// Currently we only support step level plugins but more can be added in the future.
|
||||
Type string `db:"plugin_type" json:"type"`
|
||||
Version string `db:"plugin_version" json:"version"`
|
||||
Logo string `db:"plugin_logo" json:"logo"`
|
||||
// Spec is a YAML template to be used for the plugin.
|
||||
Spec string `db:"plugin_spec" json:"spec"`
|
||||
}
|
||||
|
||||
// Matches checks whether two plugins are identical.
|
||||
// We can use reflection here, this is just easier to add on to
|
||||
// when needed.
|
||||
func (plugin *Plugin) Matches(v *Plugin) bool {
|
||||
if plugin.UID != v.UID {
|
||||
return false
|
||||
}
|
||||
if plugin.Description != v.Description {
|
||||
return false
|
||||
}
|
||||
if plugin.Spec != v.Spec {
|
||||
return false
|
||||
}
|
||||
if plugin.Version != v.Version {
|
||||
return false
|
||||
}
|
||||
if plugin.Logo != v.Logo {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue