USEFULL
/
drone
mirror of https://github.com/harness/drone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add support for starlark/jsonnet for drone v1 (#940)

eb/code-1016-2
Vistaar Juneja 2024-01-05 14:10:22 +00:00 committed by Harness
parent f314fc1e5f
commit a27a1e4abb
16 changed files with 826 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/api/controller/execution/create.go
View File

@ -75,7 +75,7 @@ func (c *Controller) Create(
AuthorEmail: commit.Author.Identity.Email,
Ref: ref,
Message: commit.Message,
Title: "", // we expect this to be empty.
Title: commit.Title,
Before: commit.SHA,
After: commit.SHA,
Sender: session.Principal.UID,

67
app/pipeline/converter/converter.go Normal file
View File

@ -0,0 +1,67 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package converter
import (
"context"
"strings"
"github.com/harness/gitness/app/pipeline/converter/jsonnet"
"github.com/harness/gitness/app/pipeline/converter/starlark"
"github.com/harness/gitness/app/pipeline/file"
)
const (
jsonnetImportLimit = 1000
starlarkStepLimit = 50000
starlarkSizeLimit = 1000000
)
type converter struct {
fileService file.Service
}
func newConverter(fileService file.Service) Service {
return &converter{fileService: fileService}
}
func (c *converter) Convert(_ context.Context, args *ConvertArgs) (*file.File, error) {
path := args.Pipeline.ConfigPath
if isJSONNet(path) {
str, err := jsonnet.Parse(args.Repo, args.Pipeline, args.Execution, args.File, c.fileService, jsonnetImportLimit)
if err != nil {
return nil, err
}
return &file.File{Data: []byte(str)}, nil
} else if isStarlark(path) {
str, err := starlark.Parse(args.Repo, args.Pipeline, args.Execution, args.File, starlarkStepLimit, starlarkSizeLimit)
if err != nil {
return nil, err
}
return &file.File{Data: []byte(str)}, nil
}
return args.File, nil
}
func isJSONNet(path string) bool {
return strings.HasSuffix(path, ".drone.jsonnet")
}
func isStarlark(path string) bool {
return strings.HasSuffix(path, ".drone.script") ||
strings.HasSuffix(path, ".drone.star") ||
strings.HasSuffix(path, ".drone.starlark")
}

219
app/pipeline/converter/jsonnet/jsonnet.go Normal file
View File

@ -0,0 +1,219 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package jsonnet
import (
"bytes"
"context"
"errors"
"fmt"
"path"
"strconv"
"strings"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/types"
"github.com/google/go-jsonnet"
)
const repo = "repo."
const build = "build."
const param = "param."
var noContext = context.Background()
type importer struct {
repo *types.Repository
execution *types.Execution
// jsonnet does not cache file imports and may request
// the same file multiple times. We cache the files to
// duplicate API calls.
cache map[string]jsonnet.Contents
// limit the number of outbound requests. github limits
// the number of api requests per hour, so we should
// make sure that a single build does not abuse the api
// by importing dozens of files.
limit int
// counts the number of outbound requests. if the count
// exceeds the limit, the importer will return errors.
count int
fileService file.Service
}
func (i *importer) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) {
if i.cache == nil {
i.cache = map[string]jsonnet.Contents{}
}
// the import is relative to the imported from path. the
// imported path must resolve to a filepath relative to
// the root of the repository.
importedPath = path.Join(
path.Dir(importedFrom),
importedPath,
)
if strings.HasPrefix(importedFrom, "../") {
err = fmt.Errorf("jsonnet: cannot resolve import: %s", importedPath)
return contents, foundAt, err
}
// if the contents exist in the cache, return the
// cached item.
var ok bool
if contents, ok = i.cache[importedPath]; ok {
return contents, importedPath, nil
}
defer func() {
i.count++
}()
// if the import limit is exceeded log an error message.
if i.limit > 0 && i.count >= i.limit {
return contents, foundAt, errors.New("jsonnet: import limit exceeded")
}
find, err := i.fileService.Get(noContext, i.repo, importedPath, i.execution.Ref)
if err != nil {
return contents, foundAt, err
}
i.cache[importedPath] = jsonnet.MakeContents(string(find.Data))
return i.cache[importedPath], importedPath, err
}
func Parse(
repo *types.Repository,
pipeline *types.Pipeline,
execution *types.Execution,
file *file.File,
fileService file.Service,
limit int,
) (string, error) {
vm := jsonnet.MakeVM()
vm.MaxStack = 500
vm.StringOutput = false
vm.ErrorFormatter.SetMaxStackTraceSize(20)
if fileService != nil && limit > 0 {
vm.Importer(
&importer{
repo: repo,
execution: execution,
limit: limit,
fileService: fileService,
},
)
}
// map execution/repo/pipeline parameters
if execution != nil {
mapBuild(execution, vm)
}
if repo != nil {
mapRepo(repo, pipeline, vm)
}
jsonnetFile := file
jsonnetFileName := pipeline.ConfigPath
// convert the jsonnet file to yaml
buf := new(bytes.Buffer)
docs, err := vm.EvaluateAnonymousSnippetStream(jsonnetFileName, string(jsonnetFile.Data))
if err != nil {
doc, err2 := vm.EvaluateAnonymousSnippet(jsonnetFileName, string(jsonnetFile.Data))
if err2 != nil {
return "", err
}
docs = append(docs, doc)
}
// the jsonnet vm returns a stream of yaml documents
// that need to be combined into a single yaml file.
for _, doc := range docs {
buf.WriteString("---")
buf.WriteString("\n")
buf.WriteString(doc)
}
return buf.String(), nil
}
// mapBuild populates build variables available to jsonnet templates.
// Since we want to maintain compatibility with drone, the older format
// needs to be maintained (even if the variables do not exist in gitness).
func mapBuild(v *types.Execution, vm *jsonnet.VM) {
vm.ExtVar(build+"event", v.Event)
vm.ExtVar(build+"action", v.Action)
vm.ExtVar(build+"environment", v.Deploy)
vm.ExtVar(build+"link", v.Link)
vm.ExtVar(build+"branch", v.Target)
vm.ExtVar(build+"source", v.Source)
vm.ExtVar(build+"before", v.Before)
vm.ExtVar(build+"after", v.After)
vm.ExtVar(build+"target", v.Target)
vm.ExtVar(build+"ref", v.Ref)
vm.ExtVar(build+"commit", v.After)
vm.ExtVar(build+"ref", v.Ref)
vm.ExtVar(build+"title", v.Title)
vm.ExtVar(build+"message", v.Message)
vm.ExtVar(build+"source_repo", v.Fork)
vm.ExtVar(build+"author_login", v.Author)
vm.ExtVar(build+"author_name", v.AuthorName)
vm.ExtVar(build+"author_email", v.AuthorEmail)
vm.ExtVar(build+"author_avatar", v.AuthorAvatar)
vm.ExtVar(build+"sender", v.Sender)
fromMap(v.Params, vm)
}
// mapBuild populates repo level variables available to jsonnet templates.
// Since we want to maintain compatibility with drone 2.x, the older format
// needs to be maintained (even if the variables do not exist in gitness).
func mapRepo(v *types.Repository, p *types.Pipeline, vm *jsonnet.VM) {
namespace := v.Path
idx := strings.LastIndex(v.Path, "/")
if idx != -1 {
namespace = v.Path[:idx]
}
vm.ExtVar(repo+"uid", v.UID)
vm.ExtVar(repo+"name", v.UID)
vm.ExtVar(repo+"namespace", namespace)
vm.ExtVar(repo+"slug", v.Path)
vm.ExtVar(repo+"git_http_url", v.GitURL)
vm.ExtVar(repo+"git_ssh_url", v.GitURL)
vm.ExtVar(repo+"link", v.GitURL)
vm.ExtVar(repo+"branch", v.DefaultBranch)
vm.ExtVar(repo+"config", p.ConfigPath)
vm.ExtVar(repo+"private", strconv.FormatBool(!v.IsPublic))
vm.ExtVar(repo+"visibility", "internal")
vm.ExtVar(repo+"active", strconv.FormatBool(true))
vm.ExtVar(repo+"trusted", strconv.FormatBool(true))
vm.ExtVar(repo+"protected", strconv.FormatBool(false))
vm.ExtVar(repo+"ignore_forks", strconv.FormatBool(false))
vm.ExtVar(repo+"ignore_pull_requests", strconv.FormatBool(false))
}
func fromMap(m map[string]string, vm *jsonnet.VM) {
for k, v := range m {
vm.ExtVar(build+param+k, v)
}
}

39
app/pipeline/converter/service.go Normal file
View File

@ -0,0 +1,39 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package converter
import (
"context"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/types"
)
type (
// ConvertArgs represents a request to the pipeline
// conversion service.
ConvertArgs struct {
Repo *types.Repository `json:"repository,omitempty"`
Pipeline *types.Pipeline `json:"pipeline,omitempty"`
Execution *types.Execution `json:"execution,omitempty"`
File *file.File `json:"config,omitempty"`
}
// Service converts a file which is in starlark/jsonnet form by looking
// at the extension and calling the appropriate parser.
Service interface {
Convert(ctx context.Context, args *ConvertArgs) (*file.File, error)
}
)

105
app/pipeline/converter/starlark/args.go Normal file
View File

@ -0,0 +1,105 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package starlark
import (
"strings"
"github.com/harness/gitness/types"
"go.starlark.net/starlark"
"go.starlark.net/starlarkstruct"
)
func createArgs(
repo *types.Repository,
pipeline *types.Pipeline,
execution *types.Execution,
) []starlark.Value {
args := []starlark.Value{
starlarkstruct.FromStringDict(
starlark.String("context"),
starlark.StringDict{
"repo": starlarkstruct.FromStringDict(starlark.String("repo"), fromRepo(repo, pipeline)),
"build": starlarkstruct.FromStringDict(starlark.String("build"), fromBuild(execution)),
},
),
}
return args
}
func fromBuild(v *types.Execution) starlark.StringDict {
return starlark.StringDict{
"event": starlark.String(v.Event),
"action": starlark.String(v.Action),
"cron": starlark.String(v.Cron),
"link": starlark.String(v.Link),
"branch": starlark.String(v.Target),
"source": starlark.String(v.Source),
"before": starlark.String(v.Before),
"after": starlark.String(v.After),
"target": starlark.String(v.Target),
"ref": starlark.String(v.Ref),
"commit": starlark.String(v.After),
"title": starlark.String(v.Title),
"message": starlark.String(v.Message),
"source_repo": starlark.String(v.Fork),
"author_login": starlark.String(v.Author),
"author_name": starlark.String(v.AuthorName),
"author_email": starlark.String(v.AuthorEmail),
"author_avatar": starlark.String(v.AuthorAvatar),
"sender": starlark.String(v.Sender),
"debug": starlark.Bool(v.Debug),
"params": fromMap(v.Params),
}
}
func fromRepo(v *types.Repository, p *types.Pipeline) starlark.StringDict {
namespace := v.Path
idx := strings.LastIndex(v.Path, "/")
if idx != -1 {
namespace = v.Path[:idx]
}
return starlark.StringDict{
"uid": starlark.String(v.UID),
"name": starlark.String(v.UID),
"namespace": starlark.String(namespace),
"slug": starlark.String(v.Path),
"git_http_url": starlark.String(v.GitURL),
"git_ssh_url": starlark.String(v.GitURL),
"link": starlark.String(v.GitURL),
"branch": starlark.String(v.DefaultBranch),
"config": starlark.String(p.ConfigPath),
"private": !starlark.Bool(v.IsPublic),
"visibility": starlark.String("internal"),
"active": starlark.Bool(true),
"trusted": starlark.Bool(true),
"protected": starlark.Bool(false),
"ignore_forks": starlark.Bool(false),
"ignore_pull_requests": starlark.Bool(false),
}
}
func fromMap(m map[string]string) *starlark.Dict {
dict := new(starlark.Dict)
for k, v := range m {
//nolint: errcheck
dict.SetKey(
starlark.String(k),
starlark.String(v),
)
}
return dict
}

147
app/pipeline/converter/starlark/starlark.go Normal file
View File

@ -0,0 +1,147 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package starlark
import (
"bytes"
"errors"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/types"
"github.com/sirupsen/logrus"
"go.starlark.net/starlark"
)
const (
separator = "---"
newline = "\n"
)
// default limit for generated configuration file size.
const defaultSizeLimit = 1000000
var (
// ErrMainMissing indicates the starlark script is missing
// the main method.
ErrMainMissing = errors.New("starlark: missing main function")
// ErrMainInvalid indicates the starlark script defines a
// global variable named main, however, it is not callable.
ErrMainInvalid = errors.New("starlark: main must be a function")
// ErrMainReturn indicates the starlark script's main method
// returns an invalid or unexpected type.
ErrMainReturn = errors.New("starlark: main returns an invalid type")
// ErrMaximumSize indicates the starlark script generated a
// file that exceeds the maximum allowed file size.
ErrMaximumSize = errors.New("starlark: maximum file size exceeded")
// ErrCannotLoad indicates the starlark script is attempting to
// load an external file which is currently restricted.
ErrCannotLoad = errors.New("starlark: cannot load external scripts")
)
func Parse(
repo *types.Repository,
pipeline *types.Pipeline,
execution *types.Execution,
file *file.File,
stepLimit uint64,
sizeLimit uint64,
) (string, error) {
thread := &starlark.Thread{
Name: "drone",
Load: noLoad,
Print: func(_ *starlark.Thread, msg string) {
logrus.WithFields(logrus.Fields{
"namespace": repo.Path, // TODO: update to just be the space
"name": repo.UID,
}).Traceln(msg)
},
}
starlarkFile := file.Data
starlarkFileName := pipeline.ConfigPath
globals, err := starlark.ExecFile(thread, starlarkFileName, starlarkFile, nil)
if err != nil {
return "", err
}
// find the main method in the starlark script and
// cast to a callable type. If not callable the script
// is invalid.
mainVal, ok := globals["main"]
if !ok {
return "", ErrMainMissing
}
main, ok := mainVal.(starlark.Callable)
if !ok {
return "", ErrMainInvalid
}
// create the input args and invoke the main method
// using the input args.
args := createArgs(repo, pipeline, execution)
// set the maximum number of operations in the script. this
// mitigates long running scripts.
if stepLimit == 0 {
stepLimit = 50000
}
thread.SetMaxExecutionSteps(stepLimit)
// execute the main method in the script.
mainVal, err = starlark.Call(thread, main, args, nil)
if err != nil {
return "", err
}
buf := new(bytes.Buffer)
switch v := mainVal.(type) {
case *starlark.List:
for i := 0; i < v.Len(); i++ {
item := v.Index(i)
buf.WriteString(separator)
buf.WriteString(newline)
if err := write(buf, item); err != nil {
return "", err
}
buf.WriteString(newline)
}
case *starlark.Dict:
if err := write(buf, v); err != nil {
return "", err
}
default:
return "", ErrMainReturn
}
if sizeLimit == 0 {
sizeLimit = defaultSizeLimit
}
// this is a temporary workaround until we
// implement a LimitWriter.
if b := buf.Bytes(); uint64(len(b)) > sizeLimit {
return "", ErrMaximumSize
}
return buf.String(), nil
}
func noLoad(_ *starlark.Thread, _ string) (starlark.StringDict, error) {
return nil, ErrCannotLoad
}

125
app/pipeline/converter/starlark/write.go Normal file
View File

@ -0,0 +1,125 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package starlark
import (
"encoding/json"
"fmt"
"io"
"go.starlark.net/starlark"
)
type writer interface {
io.Writer
io.ByteWriter
io.StringWriter
}
//nolint:gocognit,cyclop
func write(out writer, v starlark.Value) error {
if marshaler, ok := v.(json.Marshaler); ok {
jsonData, err := marshaler.MarshalJSON()
if err != nil {
return err
}
if _, err = out.Write(jsonData); err != nil {
return err
}
return nil
}
switch v := v.(type) {
case starlark.NoneType:
if _, err := out.WriteString("null"); err != nil {
return err
}
case starlark.Bool:
fmt.Fprintf(out, "%t", v)
case starlark.Int:
if _, err := out.WriteString(v.String()); err != nil {
return err
}
case starlark.Float:
fmt.Fprintf(out, "%g", v)
case starlark.String:
s := string(v)
if isQuoteSafe(s) {
fmt.Fprintf(out, "%q", s)
} else {
data, err := json.Marshal(s)
if err != nil {
return err
}
if _, err = out.Write(data); err != nil {
return err
}
}
case starlark.Indexable:
if err := out.WriteByte('['); err != nil {
return err
}
for i, n := 0, starlark.Len(v); i < n; i++ {
if i > 0 {
if _, err := out.WriteString(", "); err != nil {
return err
}
}
if err := write(out, v.Index(i)); err != nil {
return err
}
}
if err := out.WriteByte(']'); err != nil {
return err
}
case *starlark.Dict:
if err := out.WriteByte('{'); err != nil {
return err
}
for i, itemPair := range v.Items() {
key := itemPair[0]
value := itemPair[1]
if i > 0 {
if _, err := out.WriteString(", "); err != nil {
return err
}
}
if err := write(out, key); err != nil {
return err
}
if _, err := out.WriteString(": "); err != nil {
return err
}
if err := write(out, value); err != nil {
return err
}
}
if err := out.WriteByte('}'); err != nil {
return err
}
default:
return fmt.Errorf("value %s (type `%s') can't be converted to JSON", v.String(), v.Type())
}
return nil
}
func isQuoteSafe(s string) bool {
for _, r := range s {
if r < 0x20 || r >= 0x10000 {
return false
}
}
return true
}

31
app/pipeline/converter/wire.go Normal file
View File

@ -0,0 +1,31 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package converter
import (
"github.com/harness/gitness/app/pipeline/file"
"github.com/google/wire"
)
// WireSet provides a wire set for this package.
var WireSet = wire.NewSet(
ProvideService,
)
// ProvideService provides a service which can convert templates.
func ProvideService(fileService file.Service) Service {
return newConverter(fileService)
}

59
app/pipeline/manager/manager.go
View File

@ -24,6 +24,7 @@ import (
"github.com/harness/gitness/app/bootstrap"
"github.com/harness/gitness/app/jwt"
"github.com/harness/gitness/app/pipeline/converter"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/app/pipeline/scheduler"
"github.com/harness/gitness/app/sse"
@ -125,12 +126,13 @@ type (
// Manager provides a simplified interface to the build runner so that it
// can more easily interact with the server.
type Manager struct {
Executions store.ExecutionStore
Config *types.Config
FileService file.Service
Pipelines store.PipelineStore
urlProvider urlprovider.Provider
Checks store.CheckStore
Executions store.ExecutionStore
Config *types.Config
FileService file.Service
ConverterService converter.Service
Pipelines store.PipelineStore
urlProvider urlprovider.Provider
Checks store.CheckStore
// Converter store.ConvertService
SSEStreamer sse.Streamer
// Globals store.GlobalSecretStore
@ -155,6 +157,7 @@ func New(
urlProvider urlprovider.Provider,
sseStreamer sse.Streamer,
fileService file.Service,
converterService converter.Service,
logStore store.LogStore,
logStream livelog.LogStream,
checkStore store.CheckStore,
@ -166,21 +169,22 @@ func New(
userStore store.PrincipalStore,
) *Manager {
return &Manager{
Config: config,
Executions: executionStore,
Pipelines: pipelineStore,
urlProvider: urlProvider,
SSEStreamer: sseStreamer,
FileService: fileService,
Logs: logStore,
Logz: logStream,
Checks: checkStore,
Repos: repoStore,
Scheduler: scheduler,
Secrets: secretStore,
Stages: stageStore,
Steps: stepStore,
Users: userStore,
Config: config,
Executions: executionStore,
Pipelines: pipelineStore,
urlProvider: urlProvider,
SSEStreamer: sseStreamer,
FileService: fileService,
ConverterService: converterService,
Logs: logStore,
Logz: logStream,
Checks: checkStore,
Repos: repoStore,
Scheduler: scheduler,
Secrets: secretStore,
Stages: stageStore,
Steps: stepStore,
Users: userStore,
}
}
@ -325,6 +329,19 @@ func (m *Manager) Details(_ context.Context, stageID int64) (*ExecutionContext,
return nil, err
}
// Convert file contents in case templates are being used.
args := &converter.ConvertArgs{
Repo: repo,
Pipeline: pipeline,
Execution: execution,
File: file,
}
file, err = m.ConverterService.Convert(noContext, args)
if err != nil {
log.Warn().Err(err).Msg("manager: cannot convert template contents")
return nil, err
}
netrc, err := m.createNetrc(repo)
if err != nil {
log.Warn().Err(err).Msg("manager: failed to create netrc")

6
app/pipeline/manager/wire.go
View File

@ -15,6 +15,7 @@
package manager
import (
"github.com/harness/gitness/app/pipeline/converter"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/app/pipeline/scheduler"
"github.com/harness/gitness/app/sse"
@ -41,6 +42,7 @@ func ProvideExecutionManager(
urlProvider url.Provider,
sseStreamer sse.Streamer,
fileService file.Service,
converterService converter.Service,
logStore store.LogStore,
logStream livelog.LogStream,
checkStore store.CheckStore,
@ -50,8 +52,8 @@ func ProvideExecutionManager(
stageStore store.StageStore,
stepStore store.StepStore,
userStore store.PrincipalStore) ExecutionManager {
return New(config, executionStore, pipelineStore, urlProvider, sseStreamer, fileService, logStore,
logStream, checkStore, repoStore, scheduler, secretStore, stageStore, stepStore, userStore)
return New(config, executionStore, pipelineStore, urlProvider, sseStreamer, fileService, converterService,
logStore, logStream, checkStore, repoStore, scheduler, secretStore, stageStore, stepStore, userStore)
}
// ProvideExecutionClient provides a client implementation to interact with the execution manager.

53
app/pipeline/triggerer/trigger.go
View File

@ -22,6 +22,7 @@ import (
"time"
"github.com/harness/gitness/app/pipeline/checks"
"github.com/harness/gitness/app/pipeline/converter"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/app/pipeline/manager"
"github.com/harness/gitness/app/pipeline/scheduler"
@ -77,15 +78,16 @@ type Triggerer interface {
}
type triggerer struct {
executionStore store.ExecutionStore
checkStore store.CheckStore
stageStore store.StageStore
tx dbtx.Transactor
pipelineStore store.PipelineStore
fileService file.Service
urlProvider url.Provider
scheduler scheduler.Scheduler
repoStore store.RepoStore
executionStore store.ExecutionStore
checkStore store.CheckStore
stageStore store.StageStore
tx dbtx.Transactor
pipelineStore store.PipelineStore
fileService file.Service
converterService converter.Service
urlProvider url.Provider
scheduler scheduler.Scheduler
repoStore store.RepoStore
}
func New(
@ -98,17 +100,19 @@ func New(
urlProvider url.Provider,
scheduler scheduler.Scheduler,
fileService file.Service,
converterService converter.Service,
) Triggerer {
return &triggerer{
executionStore: executionStore,
checkStore: checkStore,
stageStore: stageStore,
scheduler: scheduler,
urlProvider: urlProvider,
tx: tx,
pipelineStore: pipelineStore,
fileService: fileService,
repoStore: repoStore,
executionStore: executionStore,
checkStore: checkStore,
stageStore: stageStore,
scheduler: scheduler,
urlProvider: urlProvider,
tx: tx,
pipelineStore: pipelineStore,
fileService: fileService,
converterService: converterService,
repoStore: repoStore,
}
}
@ -186,6 +190,19 @@ func (t *triggerer) Trigger(
stages := []*types.Stage{}
//nolint:nestif // refactor if needed
if !isV1Yaml(file.Data) {
// Convert from jsonnet/starlark to drone yaml
args := &converter.ConvertArgs{
Repo: repo,
Pipeline: pipeline,
Execution: execution,
File: file,
}
file, err = t.converterService.Convert(ctx, args)
if err != nil {
log.Warn().Err(err).Msg("trigger: cannot convert from template")
return t.createExecutionWithError(ctx, pipeline, base, err.Error())
}
manifest, err := yaml.ParseString(string(file.Data))
if err != nil {
log.Warn().Err(err).Msg("trigger: cannot parse yaml")

4
app/pipeline/triggerer/wire.go
View File

@ -15,6 +15,7 @@
package triggerer
import (
"github.com/harness/gitness/app/pipeline/converter"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/app/pipeline/scheduler"
"github.com/harness/gitness/app/store"
@ -37,10 +38,11 @@ func ProvideTriggerer(
tx dbtx.Transactor,
pipelineStore store.PipelineStore,
fileService file.Service,
converterService converter.Service,
scheduler scheduler.Scheduler,
repoStore store.RepoStore,
urlProvider url.Provider,
) Triggerer {
return New(executionStore, checkStore, stageStore, pipelineStore,
tx, repoStore, urlProvider, scheduler, fileService)
tx, repoStore, urlProvider, scheduler, fileService, converterService)
}

2
cmd/gitness/wire.go
View File

@ -40,6 +40,7 @@ import (
"github.com/harness/gitness/app/githook"
"github.com/harness/gitness/app/pipeline/canceler"
"github.com/harness/gitness/app/pipeline/commit"
"github.com/harness/gitness/app/pipeline/converter"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/app/pipeline/manager"
pluginmanager "github.com/harness/gitness/app/pipeline/plugin"
@ -159,6 +160,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e
manager.WireSet,
triggerer.WireSet,
file.WireSet,
converter.WireSet,
runner.WireSet,
sse.WireSet,
scheduler.WireSet,

6
cmd/gitness/wire_gen.go
View File

@ -39,6 +39,7 @@ import (
"github.com/harness/gitness/app/githook"
"github.com/harness/gitness/app/pipeline/canceler"
"github.com/harness/gitness/app/pipeline/commit"
"github.com/harness/gitness/app/pipeline/converter"
"github.com/harness/gitness/app/pipeline/file"
"github.com/harness/gitness/app/pipeline/manager"
plugin2 "github.com/harness/gitness/app/pipeline/plugin"
@ -198,7 +199,8 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
cancelerCanceler := canceler.ProvideCanceler(executionStore, streamer, repoStore, schedulerScheduler, stageStore, stepStore)
commitService := commit.ProvideService(gitInterface)
fileService := file.ProvideService(gitInterface)
triggererTriggerer := triggerer.ProvideTriggerer(executionStore, checkStore, stageStore, transactor, pipelineStore, fileService, schedulerScheduler, repoStore, provider)
converterService := converter.ProvideService(fileService)
triggererTriggerer := triggerer.ProvideTriggerer(executionStore, checkStore, stageStore, transactor, pipelineStore, fileService, converterService, schedulerScheduler, repoStore, provider)
executionController := execution.ProvideController(transactor, authorizer, executionStore, checkStore, cancelerCanceler, commitService, triggererTriggerer, repoStore, stageStore, pipelineStore)
logStore := logs.ProvideLogStore(db, config)
logStream := livelog.ProvideLogStream()
@ -278,7 +280,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
webHandler := router.ProvideWebHandler(config)
routerRouter := router.ProvideRouter(apiHandler, gitHandler, webHandler, provider)
serverServer := server2.ProvideServer(config, routerRouter)
executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, streamer, fileService, logStore, logStream, checkStore, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore)
executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, streamer, fileService, converterService, logStore, logStream, checkStore, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore)
client := manager.ProvideExecutionClient(executionManager, provider, config)
pluginManager := plugin2.ProvidePluginManager(config, pluginStore)
runtimeRunner, err := runner.ProvideExecutionRunner(config, client, pluginManager)

2
go.mod
View File

@ -138,6 +138,7 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/certificate-transparency-go v1.1.2-0.20210511102531-373a877eec92 // indirect
github.com/google/go-jsonnet v0.20.0 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
@ -212,6 +213,7 @@ require (
go.etcd.io/etcd/tests/v3 v3.5.0-alpha.0 // indirect
go.etcd.io/etcd/v3 v3.5.0-alpha.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.starlark.net v0.0.0-20231121155337-90ade8b19d09 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/oauth2 v0.10.0 // indirect
golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect

4
go.sum
View File

@ -650,6 +650,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM=
github.com/google/go-jsonnet v0.20.0 h1:WG4TTSARuV7bSm4PMB4ohjxe33IHT5WVTrJSU33uT4g=
github.com/google/go-jsonnet v0.20.0/go.mod h1:VbgWF9JX7ztlv770x/TolZNGGFfiHEVx9G6ca2eUmeA=
github.com/google/go-licenses v0.0.0-20210329231322-ce1d9163b77d/go.mod h1:+TYOmkVoJOpwnS0wfdsJCV9CoD5nJYsHoFk/0CrTK4M=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
@ -1488,6 +1490,8 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.starlark.net v0.0.0-20231121155337-90ade8b19d09 h1:hzy3LFnSN8kuQK8h9tHl4ndF6UruMj47OqwqsS+/Ai4=
go.starlark.net v0.0.0-20231121155337-90ade8b19d09/go.mod h1:LcLNIzVOMp4oV+uusnpk+VU+SzXaJakUuBjoCSWH5dM=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=