USEFULL
/
drone
mirror of https://github.com/harness/drone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Templates support for v1 pipelines (#955)

eb/code-1016-2
Vistaar Juneja 2024-01-12 11:02:31 +00:00 committed by Harness
parent 182e6bb174
commit 76e3c06b6b
32 changed files with 500 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
app/api/controller/template/create.go
View File

@ -39,7 +39,6 @@ type CreateInput struct {
Description string `json:"description"`
SpaceRef string `json:"space_ref"` // Ref of the parent space
UID string `json:"uid"`
Type string `json:"type"`
Data string `json:"data"`
}
@ -58,6 +57,10 @@ func (c *Controller) Create(ctx context.Context, session *auth.Session, in *Crea
return nil, err
}
// Try to parse template data into valid v1 config. Ignore error as it's
// already validated in sanitize function.
resolverType, _ := parseResolverType(in.Data)
var template *types.Template
now := time.Now().UnixMilli()
template = &types.Template{
@ -65,6 +68,7 @@ func (c *Controller) Create(ctx context.Context, session *auth.Session, in *Crea
Data: in.Data,
SpaceID: parentSpace.ID,
UID: in.UID,
Type: resolverType,
Created: now,
Updated: now,
Version: 0,
@ -88,6 +92,11 @@ func (c *Controller) sanitizeCreateInput(in *CreateInput) error {
return err
}
_, err = parseResolverType(in.Data)
if err != nil {
return err
}
in.Description = strings.TrimSpace(in.Description)
return check.Description(in.Description)
}

10
app/api/controller/template/delete.go
View File

@ -23,7 +23,13 @@ import (
"github.com/harness/gitness/types/enum"
)
func (c *Controller) Delete(ctx context.Context, session *auth.Session, spaceRef string, uid string) error {
func (c *Controller) Delete(
ctx context.Context,
session *auth.Session,
spaceRef string,
uid string,
resolverType enum.ResolverType,
) error {
space, err := c.spaceStore.FindByRef(ctx, spaceRef)
if err != nil {
return fmt.Errorf("failed to find space: %w", err)
@ -33,7 +39,7 @@ func (c *Controller) Delete(ctx context.Context, session *auth.Session, spaceRef
if err != nil {
return fmt.Errorf("failed to authorize: %w", err)
}
err = c.templateStore.DeleteByUID(ctx, space.ID, uid)
err = c.templateStore.DeleteByUIDAndType(ctx, space.ID, uid, resolverType)
if err != nil {
return fmt.Errorf("could not delete template: %w", err)
}

3
app/api/controller/template/find.go
View File

@ -29,6 +29,7 @@ func (c *Controller) Find(
session *auth.Session,
spaceRef string,
uid string,
resolverType enum.ResolverType,
) (*types.Template, error) {
space, err := c.spaceStore.FindByRef(ctx, spaceRef)
if err != nil {
@ -38,7 +39,7 @@ func (c *Controller) Find(
if err != nil {
return nil, fmt.Errorf("failed to authorize: %w", err)
}
template, err := c.templateStore.FindByUID(ctx, space.ID, uid)
template, err := c.templateStore.FindByUIDAndType(ctx, space.ID, uid, resolverType)
if err != nil {
return nil, fmt.Errorf("failed to find template: %w", err)
}

13
app/api/controller/template/update.go
View File

@ -38,6 +38,7 @@ func (c *Controller) Update(
session *auth.Session,
spaceRef string,
uid string,
resolverType enum.ResolverType,
in *UpdateInput,
) (*types.Template, error) {
space, err := c.spaceStore.FindByRef(ctx, spaceRef)
@ -54,7 +55,7 @@ func (c *Controller) Update(
return nil, fmt.Errorf("failed to sanitize input: %w", err)
}
template, err := c.templateStore.FindByUID(ctx, space.ID, uid)
template, err := c.templateStore.FindByUIDAndType(ctx, space.ID, uid, resolverType)
if err != nil {
return nil, fmt.Errorf("failed to find template: %w", err)
}
@ -67,7 +68,10 @@ func (c *Controller) Update(
original.Description = *in.Description
}
if in.Data != nil {
// ignore error as it's already validated in sanitize function
t, _ := parseResolverType(*in.Data)
original.Data = *in.Data
original.Type = t
}
return nil
@ -88,7 +92,12 @@ func (c *Controller) sanitizeUpdateInput(in *UpdateInput) error {
}
}
// TODO: Validate Data
if in.Data != nil {
_, err := parseResolverType(*in.Data)
if err != nil {
return err
}
}
return nil
}

38
app/api/controller/template/validate.go Normal file
View File

@ -0,0 +1,38 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package template
import (
"fmt"
"github.com/harness/gitness/types/check"
"github.com/harness/gitness/types/enum"
"github.com/drone/spec/dist/go/parse"
)
// parseResolverType parses and validates the input yaml. It returns back the parsed
// template type.
func parseResolverType(data string) (enum.ResolverType, error) {
config, err := parse.ParseString(data)
if err != nil {
return "", check.NewValidationError(fmt.Sprintf("could not parse template data: %s", err))
}
resolverTypeEnum, err := enum.ParseResolverType(config.Type)
if err != nil {
return "", check.NewValidationError(fmt.Sprintf("could not parse template type: %s", config.Type))
}
return resolverTypeEnum, nil
}

15
app/api/handler/template/delete.go
View File

@ -21,6 +21,7 @@ import (
"github.com/harness/gitness/app/api/render"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/app/paths"
"github.com/harness/gitness/types/enum"
)
func HandleDelete(templateCtrl *template.Controller) http.HandlerFunc {
@ -38,7 +39,19 @@ func HandleDelete(templateCtrl *template.Controller) http.HandlerFunc {
return
}
err = templateCtrl.Delete(ctx, session, spaceRef, templateUID)
resolverType, err := request.GetTemplateTypeFromPath(r)
if err != nil {
render.TranslatedUserError(w, err)
return
}
tempalateTypeEnum, err := enum.ParseResolverType(resolverType)
if err != nil {
render.TranslatedUserError(w, err)
return
}
err = templateCtrl.Delete(ctx, session, spaceRef, templateUID, tempalateTypeEnum)
if err != nil {
render.TranslatedUserError(w, err)
return

15
app/api/handler/template/find.go
View File

@ -21,6 +21,7 @@ import (
"github.com/harness/gitness/app/api/render"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/app/paths"
"github.com/harness/gitness/types/enum"
)
// HandleFind finds a template from the database.
@ -39,7 +40,19 @@ func HandleFind(templateCtrl *template.Controller) http.HandlerFunc {
return
}
template, err := templateCtrl.Find(ctx, session, spaceRef, templateUID)
resolverType, err := request.GetTemplateTypeFromPath(r)
if err != nil {
render.TranslatedUserError(w, err)
return
}
tempalateTypeEnum, err := enum.ParseResolverType(resolverType)
if err != nil {
render.TranslatedUserError(w, err)
return
}
template, err := templateCtrl.Find(ctx, session, spaceRef, templateUID, tempalateTypeEnum)
if err != nil {
render.TranslatedUserError(w, err)
return

15
app/api/handler/template/update.go
View File

@ -22,6 +22,7 @@ import (
"github.com/harness/gitness/app/api/render"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/app/paths"
"github.com/harness/gitness/types/enum"
)
func HandleUpdate(templateCtrl *template.Controller) http.HandlerFunc {
@ -47,7 +48,19 @@ func HandleUpdate(templateCtrl *template.Controller) http.HandlerFunc {
return
}
template, err := templateCtrl.Update(ctx, session, spaceRef, templateUID, in)
resolverType, err := request.GetTemplateTypeFromPath(r)
if err != nil {
render.TranslatedUserError(w, err)
}
resolverTypeEnum, err := enum.ParseResolverType(resolverType)
if err != nil {
render.TranslatedUserError(w, err)
return
}
template, err := templateCtrl.Update(ctx, session, spaceRef, templateUID,
resolverTypeEnum, in)
if err != nil {
render.TranslatedUserError(w, err)
return

12
app/api/request/template.go
View File

@ -20,7 +20,8 @@ import (
)
const (
PathParamTemplateRef = "template_ref"
PathParamTemplateRef = "template_ref"
PathParamTemplateType = "template_type"
)
func GetTemplateRefFromPath(r *http.Request) (string, error) {
@ -32,3 +33,12 @@ func GetTemplateRefFromPath(r *http.Request) (string, error) {
// paths are unescaped
return url.PathUnescape(rawRef)
}
func GetTemplateTypeFromPath(r *http.Request) (string, error) {
templateType, err := PathParamOrError(r, PathParamTemplateType)
if err != nil {
return "", err
}
return templateType, nil
}

57
app/pipeline/plugin/manager.go → app/pipeline/resolver/manager.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package plugin
package resolver
import (
"archive/zip"
@ -32,44 +32,55 @@ import (
"github.com/rs/zerolog/log"
)
// Lookup returns a resource by name, kind and type.
type LookupFunc func(name, kind, typ, version string) (*v1yaml.Config, error)
// Lookup returns a resource by name, kind and type. It also sends in the
// execution ID.
type LookupFunc func(name, kind, typ, version string, id int64) (*v1yaml.Config, error)
type Manager struct {
config *types.Config
pluginStore store.PluginStore
config *types.Config
pluginStore store.PluginStore
templateStore store.TemplateStore
executionStore store.ExecutionStore
repoStore store.RepoStore
}
func NewManager(
config *types.Config,
pluginStore store.PluginStore,
templateStore store.TemplateStore,
executionStore store.ExecutionStore,
repoStore store.RepoStore,
) *Manager {
return &Manager{
config: config,
pluginStore: pluginStore,
config: config,
pluginStore: pluginStore,
templateStore: templateStore,
executionStore: executionStore,
repoStore: repoStore,
}
}
// GetLookupFn returns a lookup function for plugins which can be used in the resolver.
// GetLookupFn returns a lookup function for plugins and templates which can be used in the resolver
// passed to the drone runner.
//
//nolint:gocognit
func (m *Manager) GetLookupFn() LookupFunc {
return func(name, kind, typ, version string) (*v1yaml.Config, error) {
if kind != "plugin" {
return nil, fmt.Errorf("only plugin kind supported")
}
if typ != "step" {
return nil, fmt.Errorf("only step plugins supported")
}
plugin, err := m.pluginStore.Find(context.Background(), name, version)
noContext := context.Background()
return func(name, kind, typ, version string, executionID int64) (*v1yaml.Config, error) {
// Find space ID corresponding to the executionID
execution, err := m.executionStore.Find(noContext, executionID)
if err != nil {
return nil, fmt.Errorf("could not lookup plugin: %w", err)
}
// Convert plugin to v1yaml spec
config, err := parse.ParseString(plugin.Spec)
if err != nil {
return nil, fmt.Errorf("could not unmarshal plugin to v1yaml spec: %w", err)
return nil, fmt.Errorf("could not find relevant execution: %w", err)
}
return config, nil
// Find the repo so we know in which space templates should be searched
repo, err := m.repoStore.Find(noContext, execution.RepoID)
if err != nil {
return nil, fmt.Errorf("could not find relevant repo: %w", err)
}
f := Resolve(noContext, m.pluginStore, m.templateStore, repo.ParentID)
return f(name, kind, typ, version)
}
}

76
app/pipeline/resolver/resolve.go Normal file
View File

@ -0,0 +1,76 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package resolver
import (
"context"
"fmt"
"github.com/harness/gitness/app/store"
"github.com/harness/gitness/types/enum"
v1yaml "github.com/drone/spec/dist/go"
"github.com/drone/spec/dist/go/parse"
)
// Resolve returns a resolve function which resolves plugins and templates.
// It searches for plugins globally and for templates in the same space and substitutes
// them in the pipeline yaml.
func Resolve(
ctx context.Context,
pluginStore store.PluginStore,
templateStore store.TemplateStore,
spaceID int64,
) func(name, kind, typ, version string) (*v1yaml.Config, error) {
return func(name, kind, typ, version string) (*v1yaml.Config, error) {
k, err := enum.ParseResolverKind(kind)
if err != nil {
return nil, err
}
t, err := enum.ParseResolverType(typ)
if err != nil {
return nil, err
}
if k == enum.ResolverKindPlugin && t != enum.ResolverTypeStep {
return nil, fmt.Errorf("only step level plugins are currently supported")
}
if k == enum.ResolverKindPlugin {
plugin, err := pluginStore.Find(ctx, name, version)
if err != nil {
return nil, fmt.Errorf("could not lookup plugin: %w", err)
}
// Convert plugin to v1yaml spec
config, err := parse.ParseString(plugin.Spec)
if err != nil {
return nil, fmt.Errorf("could not unmarshal plugin to v1yaml spec: %w", err)
}
return config, nil
}
// Search for templates in the space
template, err := templateStore.FindByUIDAndType(ctx, spaceID, name, t)
if err != nil {
return nil, fmt.Errorf("could not find template: %w", err)
}
// Try to parse the template into v1 yaml
config, err := parse.ParseString(template.Data)
if err != nil {
return nil, fmt.Errorf("could not unmarshal template to v1yaml spec: %w", err)
}
return config, nil
}
}

13
app/pipeline/plugin/wire.go → app/pipeline/resolver/wire.go
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package plugin
package resolver
import (
"github.com/harness/gitness/app/store"
@ -23,13 +23,16 @@ import (
// WireSet provides a wire set for this package.
var WireSet = wire.NewSet(
ProvidePluginManager,
ProvideResolver,
)
// ProvidePluginManager provides an execution runner.
func ProvidePluginManager(
// ProvideResolver provides a resolver which can resolve templates and plugins.
func ProvideResolver(
config *types.Config,
pluginStore store.PluginStore,
templateStore store.TemplateStore,
executionStore store.ExecutionStore,
repoStore store.RepoStore,
) *Manager {
return NewManager(config, pluginStore)
return NewManager(config, pluginStore, templateStore, executionStore, repoStore)
}

6
app/pipeline/runner/runner.go
View File

@ -15,7 +15,7 @@
package runner
import (
"github.com/harness/gitness/app/pipeline/plugin"
"github.com/harness/gitness/app/pipeline/resolver"
"github.com/harness/gitness/types"
"github.com/drone-runners/drone-runner-docker/engine"
@ -50,7 +50,7 @@ var Privileged = []string{
func NewExecutionRunner(
config *types.Config,
client runnerclient.Client,
pluginManager *plugin.Manager,
resolver *resolver.Manager,
) (*runtime2.Runner, error) {
// For linux, containers need to have extra hosts set in order to interact with
// the gitness container.
@ -102,7 +102,7 @@ func NewExecutionRunner(
runner := &runtime2.Runner{
Machine: config.InstanceID,
Client: client,
Resolver: pluginManager.GetLookupFn(),
Resolver: resolver.GetLookupFn(),
Reporter: tracer,
Compiler: compiler2,
Exec: exec2.Exec,

6
app/pipeline/runner/wire.go
View File

@ -15,7 +15,7 @@
package runner
import (
"github.com/harness/gitness/app/pipeline/plugin"
"github.com/harness/gitness/app/pipeline/resolver"
"github.com/harness/gitness/types"
runtime2 "github.com/drone-runners/drone-runner-docker/engine2/runtime"
@ -34,9 +34,9 @@ var WireSet = wire.NewSet(
func ProvideExecutionRunner(
config *types.Config,
client runnerclient.Client,
pluginManager *plugin.Manager,
resolver *resolver.Manager,
) (*runtime2.Runner, error) {
return NewExecutionRunner(config, client, pluginManager)
return NewExecutionRunner(config, client, resolver)
}
// ProvideExecutionPoller provides a poller which can poll the manager

32
app/pipeline/triggerer/trigger.go
View File

@ -25,6 +25,7 @@ import (
"github.com/harness/gitness/app/pipeline/converter"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/app/pipeline/manager"
"github.com/harness/gitness/app/pipeline/resolver"
"github.com/harness/gitness/app/pipeline/scheduler"
"github.com/harness/gitness/app/pipeline/triggerer/dag"
"github.com/harness/gitness/app/store"
@ -39,6 +40,7 @@ import (
"github.com/drone/drone-yaml/yaml/linter"
v1yaml "github.com/drone/spec/dist/go"
"github.com/drone/spec/dist/go/parse/normalize"
specresolver "github.com/drone/spec/dist/go/parse/resolver"
"github.com/rs/zerolog/log"
)
@ -88,6 +90,8 @@ type triggerer struct {
urlProvider url.Provider
scheduler scheduler.Scheduler
repoStore store.RepoStore
templateStore store.TemplateStore
pluginStore store.PluginStore
}
func New(
@ -101,6 +105,8 @@ func New(
scheduler scheduler.Scheduler,
fileService file.Service,
converterService converter.Service,
templateStore store.TemplateStore,
pluginStore store.PluginStore,
) Triggerer {
return &triggerer{
executionStore: executionStore,
@ -113,6 +119,8 @@ func New(
fileService: fileService,
converterService: converterService,
repoStore: repoStore,
templateStore: templateStore,
pluginStore: pluginStore,
}
}
@ -322,7 +330,8 @@ func (t *triggerer) Trigger(
}
}
} else {
stages, err = parseV1Stages(file.Data, repo, execution)
stages, err = parseV1Stages(
ctx, file.Data, repo, execution, t.templateStore, t.pluginStore)
if err != nil {
return nil, fmt.Errorf("could not parse v1 YAML into stages: %w", err)
}
@ -379,7 +388,14 @@ func trunc(s string, i int) string {
// Once we have depends on in v1, this will be changed to use the DAG.
//
//nolint:gocognit // refactor if needed.
func parseV1Stages(data []byte, repo *types.Repository, execution *types.Execution) ([]*types.Stage, error) {
func parseV1Stages(
ctx context.Context,
data []byte,
repo *types.Repository,
execution *types.Execution,
templateStore store.TemplateStore,
pluginStore store.PluginStore,
) ([]*types.Stage, error) {
stages := []*types.Stage{}
// For V1 YAML, just go through the YAML and create stages serially for now
config, err := v1yaml.ParseBytes(data)
@ -403,6 +419,16 @@ func parseV1Stages(data []byte, repo *types.Repository, execution *types.Executi
var prevStage string
// expand stage level templates and plugins
lookupFunc := func(name, kind, typ, version string) (*v1yaml.Config, error) {
f := resolver.Resolve(ctx, pluginStore, templateStore, repo.ParentID)
return f(name, kind, typ, version)
}
if err := specresolver.Resolve(config, lookupFunc); err != nil {
return nil, fmt.Errorf("could not resolve yaml plugins/templates: %w", err)
}
switch v := config.Spec.(type) {
case *v1yaml.Pipeline:
// Expand expressions in strings and matrices
@ -448,7 +474,7 @@ func parseV1Stages(data []byte, repo *types.Repository, execution *types.Executi
prevStage = temp.Name
stages = append(stages, temp)
default:
return nil, fmt.Errorf("only CI stage supported in v1 at the moment")
return nil, fmt.Errorf("only CI and template stages are supported in v1 at the moment")
}
}
default:

5
app/pipeline/triggerer/wire.go
View File

@ -42,7 +42,10 @@ func ProvideTriggerer(
scheduler scheduler.Scheduler,
repoStore store.RepoStore,
urlProvider url.Provider,
templateStore store.TemplateStore,
pluginStore store.PluginStore,
) Triggerer {
return New(executionStore, checkStore, stageStore, pipelineStore,
tx, repoStore, urlProvider, scheduler, fileService, converterService)
tx, repoStore, urlProvider, scheduler, fileService, converterService,
templateStore, pluginStore)
}

13
app/router/api.go
View File

@ -87,7 +87,7 @@ type APIHandler interface {
var (
// terminatedPathPrefixesAPI is the list of prefixes that will require resolving terminated paths.
terminatedPathPrefixesAPI = []string{"/v1/spaces/", "/v1/repos/",
"/v1/secrets/", "/v1/connectors", "/v1/templates"}
"/v1/secrets/", "/v1/connectors", "/v1/templates/step", "/v1/templates/stage"}
)
// NewAPIHandler returns a new APIHandler.
@ -394,11 +394,12 @@ func setupTemplates(
r.Route("/templates", func(r chi.Router) {
// Create takes path and parentId via body, not uri
r.Post("/", handlertemplate.HandleCreate(templateCtrl))
r.Route(fmt.Sprintf("/{%s}", request.PathParamTemplateRef), func(r chi.Router) {
r.Get("/", handlertemplate.HandleFind(templateCtrl))
r.Patch("/", handlertemplate.HandleUpdate(templateCtrl))
r.Delete("/", handlertemplate.HandleDelete(templateCtrl))
})
r.Route(fmt.Sprintf("/{%s}/{%s}", request.PathParamTemplateType, request.PathParamTemplateRef),
func(r chi.Router) {
r.Get("/", handlertemplate.HandleFind(templateCtrl))
r.Patch("/", handlertemplate.HandleUpdate(templateCtrl))
r.Delete("/", handlertemplate.HandleDelete(templateCtrl))
})
})
}

9
app/store/database.go
View File

@ -664,8 +664,9 @@ type (
// Find returns a template given an ID.
Find(ctx context.Context, id int64) (*types.Template, error)
// FindByUID returns a template given a space ID and a UID.
FindByUID(ctx context.Context, spaceID int64, uid string) (*types.Template, error)
// FindByUIDAndType returns a template given a space ID, UID and a type
FindByUIDAndType(ctx context.Context, spaceID int64,
uid string, resolverType enum.ResolverType) (*types.Template, error)
// Create creates a new template.
Create(ctx context.Context, template *types.Template) error
@ -683,8 +684,8 @@ type (
// Delete deletes a template given an ID.
Delete(ctx context.Context, id int64) error
// DeleteByUID deletes a template given a space ID and a uid.
DeleteByUID(ctx context.Context, spaceID int64, uid string) error
// DeleteByUIDAndType deletes a template given a space ID, uid and a type.
DeleteByUIDAndType(ctx context.Context, spaceID int64, uid string, resolverType enum.ResolverType) error
// List lists the templates in a given space.
List(ctx context.Context, spaceID int64, filter types.ListQueryFilter) ([]*types.Template, error)

1
app/store/database/migrate/postgres/0040_alter_table_templates_add_type.down.sql Normal file
View File

@ -0,0 +1 @@
DROP TABLE templates;

22
app/store/database/migrate/postgres/0040_alter_table_templates_add_type.up.sql Normal file
View File

@ -0,0 +1,22 @@
DROP TABLE IF exists templates;
CREATE TABLE templates (
template_id SERIAL PRIMARY KEY
,template_uid TEXT NOT NULL
,template_type TEXT NOT NULL
,template_description TEXT NOT NULL
,template_space_id INTEGER NOT NULL
,template_data BYTEA NOT NULL
,template_created BIGINT NOT NULL
,template_updated BIGINT NOT NULL
,template_version INTEGER NOT NULL
-- Ensure unique combination of space ID, UID and template type
,UNIQUE (template_space_id, template_uid, template_type)
-- Foreign key to spaces table
,CONSTRAINT fk_templates_space_id FOREIGN KEY (template_space_id)
REFERENCES spaces (space_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE CASCADE
);

1
app/store/database/migrate/sqlite/0040_alter_table_templates_add_type.down.sql Normal file
View File

@ -0,0 +1 @@
DROP TABLE templates;

21
app/store/database/migrate/sqlite/0040_alter_table_templates_add_type.up.sql Normal file
View File

@ -0,0 +1,21 @@
DROP TABLE IF exists templates;
CREATE TABLE templates (
template_id INTEGER PRIMARY KEY AUTOINCREMENT
,template_uid TEXT NOT NULL
,template_type TEXT NOT NULL
,template_description TEXT NOT NULL
,template_space_id INTEGER NOT NULL
,template_data BLOB NOT NULL
,template_created INTEGER NOT NULL
,template_updated INTEGER NOT NULL
,template_version INTEGER NOT NULL
-- Ensure unique combination of space ID, UID and template type
,UNIQUE (template_space_id, template_uid, template_type)
-- Foreign key to spaces table
,CONSTRAINT fk_templates_space_id FOREIGN KEY (template_space_id)
REFERENCES spaces (space_id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE CASCADE
);

28
app/store/database/template.go
View File

@ -25,6 +25,7 @@ import (
"github.com/harness/gitness/store/database"
"github.com/harness/gitness/store/database/dbtx"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
@ -40,6 +41,7 @@ const (
templateColumns = `
template_id,
template_description,
template_type,
template_space_id,
template_uid,
template_data,
@ -73,14 +75,18 @@ func (s *templateStore) Find(ctx context.Context, id int64) (*types.Template, er
return dst, nil
}
// FindByUID returns a template in a given space with a given UID.
func (s *templateStore) FindByUID(ctx context.Context, spaceID int64, uid string) (*types.Template, error) {
// FindByUIDAndType returns a template in a space with a given UID and a given type.
func (s *templateStore) FindByUIDAndType(
ctx context.Context,
spaceID int64,
uid string,
resolverType enum.ResolverType) (*types.Template, error) {
const findQueryStmt = templateQueryBase + `
WHERE template_space_id = $1 AND template_uid = $2`
WHERE template_space_id = $1 AND template_uid = $2 AND template_type = $3`
db := dbtx.GetAccessor(ctx, s.db)
dst := new(types.Template)
if err := db.GetContext(ctx, dst, findQueryStmt, spaceID, uid); err != nil {
if err := db.GetContext(ctx, dst, findQueryStmt, spaceID, uid, resolverType.String()); err != nil {
return nil, database.ProcessSQLErrorf(err, "Failed to find template")
}
return dst, nil
@ -94,6 +100,7 @@ func (s *templateStore) Create(ctx context.Context, template *types.Template) er
template_space_id,
template_uid,
template_data,
template_type,
template_created,
template_updated,
template_version
@ -102,6 +109,7 @@ func (s *templateStore) Create(ctx context.Context, template *types.Template) er
:template_space_id,
:template_uid,
:template_data,
:template_type,
:template_created,
:template_updated,
:template_version
@ -127,6 +135,7 @@ func (s *templateStore) Update(ctx context.Context, p *types.Template) error {
template_description = :template_description,
template_uid = :template_uid,
template_data = :template_data,
template_type = :template_type,
template_updated = :template_updated,
template_version = :template_version
WHERE template_id = :template_id AND template_version = :template_version - 1`
@ -239,14 +248,19 @@ func (s *templateStore) Delete(ctx context.Context, id int64) error {
}
// DeleteByUID deletes a template with a given UID in a space.
func (s *templateStore) DeleteByUID(ctx context.Context, spaceID int64, uid string) error {
func (s *templateStore) DeleteByUIDAndType(
ctx context.Context,
spaceID int64,
uid string,
resolverType enum.ResolverType,
) error {
const templateDeleteStmt = `
DELETE FROM templates
WHERE template_space_id = $1 AND template_uid = $2`
WHERE template_space_id = $1 AND template_uid = $2 AND template_type = $3`
db := dbtx.GetAccessor(ctx, s.db)
if _, err := db.ExecContext(ctx, templateDeleteStmt, spaceID, uid); err != nil {
if _, err := db.ExecContext(ctx, templateDeleteStmt, spaceID, uid, resolverType.String()); err != nil {
return database.ProcessSQLErrorf(err, "Could not delete template")
}

2
cli/server/server.go
View File

@ -114,7 +114,7 @@ func (c *command) run(*kingpin.ParseContext) error {
if c.enableCI {
// start populating plugins
g.Go(func() error {
err := system.pluginManager.Populate(ctx)
err := system.resolverManager.Populate(ctx)
if err != nil {
log.Error().Err(err).Msg("could not populate plugins")
}

24
cli/server/system.go
View File

@ -16,7 +16,7 @@ package server
import (
"github.com/harness/gitness/app/bootstrap"
"github.com/harness/gitness/app/pipeline/plugin"
"github.com/harness/gitness/app/pipeline/resolver"
"github.com/harness/gitness/app/server"
"github.com/harness/gitness/app/services"
@ -25,21 +25,21 @@ import (
// System stores high level System sub-routines.
type System struct {
bootstrap bootstrap.Bootstrap
server *server.Server
pluginManager *plugin.Manager
poller *poller.Poller
services services.Services
bootstrap bootstrap.Bootstrap
server *server.Server
resolverManager *resolver.Manager
poller *poller.Poller
services services.Services
}
// NewSystem returns a new system structure.
func NewSystem(bootstrap bootstrap.Bootstrap, server *server.Server, poller *poller.Poller,
pluginManager *plugin.Manager, services services.Services) *System {
resolverManager *resolver.Manager, services services.Services) *System {
return &System{
bootstrap: bootstrap,
server: server,
poller: poller,
pluginManager: pluginManager,
services: services,
bootstrap: bootstrap,
server: server,
poller: poller,
resolverManager: resolverManager,
services: services,
}
}

4
cmd/gitness/wire.go
View File

@ -43,7 +43,7 @@ import (
"github.com/harness/gitness/app/pipeline/converter"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/app/pipeline/manager"
pluginmanager "github.com/harness/gitness/app/pipeline/plugin"
"github.com/harness/gitness/app/pipeline/resolver"
"github.com/harness/gitness/app/pipeline/runner"
"github.com/harness/gitness/app/pipeline/scheduler"
"github.com/harness/gitness/app/pipeline/triggerer"
@ -167,7 +167,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e
commit.WireSet,
controllertrigger.WireSet,
plugin.WireSet,
pluginmanager.WireSet,
resolver.WireSet,
importer.WireSet,
canceler.WireSet,
exporter.WireSet,

18
cmd/gitness/wire_gen.go
View File

@ -42,7 +42,7 @@ import (
"github.com/harness/gitness/app/pipeline/converter"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/app/pipeline/manager"
plugin2 "github.com/harness/gitness/app/pipeline/plugin"
"github.com/harness/gitness/app/pipeline/resolver"
"github.com/harness/gitness/app/pipeline/runner"
"github.com/harness/gitness/app/pipeline/scheduler"
"github.com/harness/gitness/app/pipeline/triggerer"
@ -172,8 +172,8 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
return nil, err
}
codeownersConfig := server.ProvideCodeOwnerConfig(config)
resolver := usergroup.ProvideUserGroupResolver()
codeownersService := codeowners.ProvideCodeOwners(gitInterface, repoStore, codeownersConfig, principalStore, resolver)
usergroupResolver := usergroup.ProvideUserGroupResolver()
codeownersService := codeowners.ProvideCodeOwners(gitInterface, repoStore, codeownersConfig, principalStore, usergroupResolver)
eventsConfig := server.ProvideEventsConfig(config)
eventsSystem, err := events.ProvideSystem(eventsConfig, universalClient)
if err != nil {
@ -200,14 +200,15 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
commitService := commit.ProvideService(gitInterface)
fileService := file.ProvideService(gitInterface)
converterService := converter.ProvideService(fileService)
triggererTriggerer := triggerer.ProvideTriggerer(executionStore, checkStore, stageStore, transactor, pipelineStore, fileService, converterService, schedulerScheduler, repoStore, provider)
templateStore := database.ProvideTemplateStore(db)
pluginStore := database.ProvidePluginStore(db)
triggererTriggerer := triggerer.ProvideTriggerer(executionStore, checkStore, stageStore, transactor, pipelineStore, fileService, converterService, schedulerScheduler, repoStore, provider, templateStore, pluginStore)
executionController := execution.ProvideController(transactor, authorizer, executionStore, checkStore, cancelerCanceler, commitService, triggererTriggerer, repoStore, stageStore, pipelineStore)
logStore := logs.ProvideLogStore(db, config)
logStream := livelog.ProvideLogStream()
logsController := logs2.ProvideController(authorizer, executionStore, repoStore, pipelineStore, stageStore, stepStore, logStore, logStream)
secretStore := database.ProvideSecretStore(db)
connectorStore := database.ProvideConnectorStore(db)
templateStore := database.ProvideTemplateStore(db)
exporterRepository, err := exporter.ProvideSpaceExporter(provider, gitInterface, repoStore, jobScheduler, executor, encrypter, streamer)
if err != nil {
return nil, err
@ -218,7 +219,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
triggerController := trigger.ProvideController(authorizer, triggerStore, pathUID, pipelineStore, repoStore)
connectorController := connector.ProvideController(pathUID, connectorStore, authorizer, spaceStore)
templateController := template.ProvideController(pathUID, templateStore, authorizer, spaceStore)
pluginStore := database.ProvidePluginStore(db)
pluginController := plugin.ProvideController(pluginStore)
pullReqStore := database.ProvidePullReqStore(db, principalInfoCache)
pullReqActivityStore := database.ProvidePullReqActivityStore(db, principalInfoCache)
@ -282,8 +282,8 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
serverServer := server2.ProvideServer(config, routerRouter)
executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, streamer, fileService, converterService, logStore, logStream, checkStore, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore)
client := manager.ProvideExecutionClient(executionManager, provider, config)
pluginManager := plugin2.ProvidePluginManager(config, pluginStore)
runtimeRunner, err := runner.ProvideExecutionRunner(config, client, pluginManager)
resolverManager := resolver.ProvideResolver(config, pluginStore, templateStore, executionStore, repoStore)
runtimeRunner, err := runner.ProvideExecutionRunner(config, client, resolverManager)
if err != nil {
return nil, err
}
@ -319,6 +319,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
return nil, err
}
servicesServices := services.ProvideServices(webhookService, pullreqService, triggerService, jobScheduler, collector, calculator, cleanupService, notificationService, keywordsearchService)
serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, poller, pluginManager, servicesServices)
serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, poller, resolverManager, servicesServices)
return serverSystem, nil
}

6
go.mod
View File

@ -13,7 +13,7 @@ require (
github.com/bmatcuk/doublestar/v4 v4.6.0
github.com/coreos/go-semver v0.3.0
github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5
github.com/drone-runners/drone-runner-docker v1.8.4-0.20231106161015-8c0240291f1d
github.com/drone-runners/drone-runner-docker v1.8.4-0.20240109154718-47375e234554
github.com/drone/drone-go v1.7.1
github.com/drone/drone-yaml v1.2.3
github.com/drone/funcmap v0.0.0-20190918184546-d4ef6e88376d
@ -54,6 +54,7 @@ require (
go.uber.org/multierr v1.8.0
golang.org/x/crypto v0.13.0
golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a
golang.org/x/oauth2 v0.10.0
golang.org/x/sync v0.3.0
golang.org/x/term v0.12.0
golang.org/x/text v0.13.0
@ -72,6 +73,7 @@ require (
gitea.com/go-chi/cache v0.2.0 // indirect
gitea.com/lunny/levelqueue v0.4.2-0.20220729054728-f020868cc2f7 // indirect
github.com/42wim/sshsig v0.0.0-20211121163825-841cf5bbc121 // indirect
github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d // indirect
github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e // indirect
github.com/RoaringBitmap/roaring v0.9.4 // indirect
github.com/alecthomas/chroma v0.10.0 // indirect
@ -116,6 +118,7 @@ require (
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/drone/envsubst v1.0.3 // indirect
github.com/drone/signal v1.0.0 // indirect
github.com/duo-labs/webauthn v0.0.0-20220330035159-03696f3d4499 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/editorconfig/editorconfig-core-go/v2 v2.4.4 // indirect
@ -215,7 +218,6 @@ require (
go.opencensus.io v0.24.0 // indirect
go.starlark.net v0.0.0-20231121155337-90ade8b19d09 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/oauth2 v0.10.0 // indirect
golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect

4
go.sum
View File

@ -90,6 +90,7 @@ gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0p
gitee.com/travelliu/dm v1.8.11192/go.mod h1:DHTzyhCrM843x9VdKVbZ+GKXGRbKM2sJ4LxihRxShkE=
github.com/42wim/sshsig v0.0.0-20211121163825-841cf5bbc121 h1:r3qt8PCHnfjOv9PN3H+XXKmDA1dfFMIN1AislhlA/ps=
github.com/42wim/sshsig v0.0.0-20211121163825-841cf5bbc121/go.mod h1:Ock8XgA7pvULhIaHGAk/cDnRfNrF9Jey81nPcc403iU=
github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d h1:j6oB/WPCigdOkxtuPl1VSIiLpy7Mdsu6phQffbF19Ng=
github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs=
github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e h1:rl2Aq4ZODqTDkeSqQBy+fzpZPamacO1Srp8zq7jf2Sc=
github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e/go.mod h1:Xa6lInWHNQnuWoF0YPSsx+INFA9qk7/7pTjwb3PInkY=
@ -404,6 +405,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/drone-runners/drone-runner-docker v1.8.4-0.20231106161015-8c0240291f1d h1:ZJNxTUJzYxRkMzikX3eMbPrUxKGRVC9fNptTykwEN6c=
github.com/drone-runners/drone-runner-docker v1.8.4-0.20231106161015-8c0240291f1d/go.mod h1:iXTCJv+tESfI/ggWZwinI2ZAzHTGS+Ic5A9gcUElTns=
github.com/drone-runners/drone-runner-docker v1.8.4-0.20240109154718-47375e234554 h1:mO6nivUH91tYkP7ME5kkFiwCgRJSk+bXrgA7wXpx2wc=
github.com/drone-runners/drone-runner-docker v1.8.4-0.20240109154718-47375e234554/go.mod h1:iXTCJv+tESfI/ggWZwinI2ZAzHTGS+Ic5A9gcUElTns=
github.com/drone/drone-go v1.7.1 h1:ZX+3Rs8YHUSUQ5mkuMLmm1zr1ttiiE2YGNxF3AnyDKw=
github.com/drone/drone-go v1.7.1/go.mod h1:fxCf9jAnXDZV1yDr0ckTuWd1intvcQwfJmTRpTZ1mXg=
github.com/drone/drone-runtime v1.0.7-0.20190729202838-87c84080f4a1/go.mod h1:+osgwGADc/nyl40J0fdsf8Z09bgcBZXvXXnLOY48zYs=
@ -422,6 +425,7 @@ github.com/drone/go-scm v1.31.2 h1:6hZxf0aETV17830fMCPrgcA4y8j/8Gdfy0xEdInUeqQ=
github.com/drone/go-scm v1.31.2/go.mod h1:DFIJJjhMj0TSXPz+0ni4nyZ9gtTtC40Vh/TGRugtyWw=
github.com/drone/runner-go v1.12.0 h1:zUjDj9ylsJ4n4Mvy4znddq/Z4EBzcUXzTltpzokKtgs=
github.com/drone/runner-go v1.12.0/go.mod h1:vu4pPPYDoeN6vdYQAY01GGGsAIW4aLganJNaa8Fx8zE=
github.com/drone/signal v1.0.0 h1:NrnM2M/4yAuU/tXs6RP1a1ZfxnaHwYkd0kJurA1p6uI=
github.com/drone/signal v1.0.0/go.mod h1:S8t92eFT0g4WUgEc/LxG+LCuiskpMNsG0ajAMGnyZpc=
github.com/drone/spec v0.0.0-20230919004456-7455b8913ff5 h1:NgAseJNQpJE3XtgJUPu4x7x5fcBjqZ3oKHDJfwBYdWk=
github.com/drone/spec v0.0.0-20230919004456-7455b8913ff5/go.mod h1:KyQZA9qwuscbbM7yTrtZg25Wammoc5GKwaRem8kDA5k=

50
types/enum/resolver_kind.go Normal file
View File

@ -0,0 +1,50 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package enum
import "fmt"
// ResolverKind represents the kind of resolver.
type ResolverKind string
const (
// ResolverKindPlugin is a plugin resolver.
ResolverKindPlugin ResolverKind = "plugin"
// ResolverKindTemplate is a template resolver.
ResolverKindTemplate ResolverKind = "template"
)
func ParseResolverKind(r string) (ResolverKind, error) {
switch r {
case "plugin":
return ResolverKindPlugin, nil
case "template":
return ResolverKindTemplate, nil
default:
return "", fmt.Errorf("unknown resolver kind provided: %s", r)
}
}
func (r ResolverKind) String() string {
switch r {
case ResolverKindPlugin:
return "plugin"
case ResolverKindTemplate:
return "template"
default:
return undefined
}
}

50
types/enum/resolver_type.go Normal file
View File

@ -0,0 +1,50 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package enum
import "fmt"
// ResolverType represents the type of resolver.
type ResolverType string
const (
// ResolverTypeStep is a step level resolver.
ResolverTypeStep ResolverType = "step"
// ResolverTypeStage is a stage level resolver.
ResolverTypeStage ResolverType = "stage"
)
func ParseResolverType(s string) (ResolverType, error) {
switch s {
case "step":
return ResolverTypeStep, nil
case "stage":
return ResolverTypeStage, nil
default:
return "", fmt.Errorf("unknown template type provided: %s", s)
}
}
func (t ResolverType) String() string {
switch t {
case ResolverTypeStep:
return "step"
case ResolverTypeStage:
return "stage"
default:
return undefined
}
}

19
types/template.go
View File

@ -14,13 +14,16 @@
package types
import "github.com/harness/gitness/types/enum"
type Template struct {
ID int64 `db:"template_id" json:"id"`
Description string `db:"template_description" json:"description"`
SpaceID int64 `db:"template_space_id" json:"space_id"`
UID string `db:"template_uid" json:"uid"`
Data string `db:"template_data" json:"data"`
Created int64 `db:"template_created" json:"created"`
Updated int64 `db:"template_updated" json:"updated"`
Version int64 `db:"template_version" json:"-"`
ID int64 `db:"template_id" json:"id"`
Description string `db:"template_description" json:"description"`
Type enum.ResolverType `db:"template_type" json:"type"`
SpaceID int64 `db:"template_space_id" json:"space_id"`
UID string `db:"template_uid" json:"uid"`
Data string `db:"template_data" json:"data"`
Created int64 `db:"template_created" json:"created"`
Updated int64 `db:"template_updated" json:"updated"`
Version int64 `db:"template_version" json:"-"`
}