From d91b3f8ed8baf89e0111d6b41d23f550f50e65df Mon Sep 17 00:00:00 2001 From: Vistaar Juneja Date: Wed, 18 Sep 2024 13:24:48 +0000 Subject: [PATCH] feat: [CI-14316]: Connectors support in gitness (#2699) * fix postgres migration * address comments * fix lint * fix lint issues * fix db schema, remove unnecessary files * normalize schema to enforce secret integrity * connectors fixes * add created_by field * rebase connectors implementation with latest * fix lint * change UID -> identifier * refactor files around and lint * Basic implementation for connectors --- app/api/controller/connector/controller.go | 17 +- app/api/controller/connector/create.go | 51 +-- app/api/controller/connector/test.go | 65 ++++ app/api/controller/connector/update.go | 29 +- app/api/controller/connector/wire.go | 4 +- app/api/handler/connector/test.go | 49 +++ app/api/openapi/connector.go | 12 + app/connector/connector.go | 58 +++ app/connector/scm/provider.go | 112 ++++++ app/connector/scm/scm.go | 50 +++ app/connector/wire.go | 42 +++ app/router/api.go | 1 + app/store/database/connector.go | 356 ++++++++++++++++-- ...le_images_and_alter_table_artifacts.up.sql | 2 +- .../0074_create_table_connectors.down.sql | 1 + .../0074_create_table_connectors.up.sql | 88 +++++ .../0074_create_table_connectors.down.sql | 1 + .../0074_create_table_connectors.up.sql | 88 +++++ app/store/database/wire.go | 4 +- cmd/gitness/wire.go | 2 + cmd/gitness/wire_gen.go | 9 +- types/connector.go | 39 +- types/connector_auth.go | 60 +++ types/connector_config.go | 38 ++ types/connector_test_response.go | 22 ++ types/enum/connector_auth_type.go | 61 +++ types/enum/connector_status.go | 49 +++ types/enum/connector_type.go | 63 ++++ types/github_connector_data.go | 44 +++ types/secret_ref.go | 19 + 30 files changed, 1333 insertions(+), 103 deletions(-) create mode 100644 app/api/controller/connector/test.go create mode 100644 app/api/handler/connector/test.go create mode 100644 app/connector/connector.go create mode 100644 app/connector/scm/provider.go create mode 100644 app/connector/scm/scm.go create mode 100644 app/connector/wire.go create mode 100644 app/store/database/migrate/postgres/0074_create_table_connectors.down.sql create mode 100644 app/store/database/migrate/postgres/0074_create_table_connectors.up.sql create mode 100644 app/store/database/migrate/sqlite/0074_create_table_connectors.down.sql create mode 100644 app/store/database/migrate/sqlite/0074_create_table_connectors.up.sql create mode 100644 types/connector_auth.go create mode 100644 types/connector_config.go create mode 100644 types/connector_test_response.go create mode 100644 types/enum/connector_auth_type.go create mode 100644 types/enum/connector_status.go create mode 100644 types/enum/connector_type.go create mode 100644 types/github_connector_data.go create mode 100644 types/secret_ref.go diff --git a/app/api/controller/connector/controller.go b/app/api/controller/connector/controller.go index 273d59f54..e78f09ba9 100644 --- a/app/api/controller/connector/controller.go +++ b/app/api/controller/connector/controller.go @@ -16,23 +16,28 @@ package connector import ( "github.com/harness/gitness/app/auth/authz" + "github.com/harness/gitness/app/connector" "github.com/harness/gitness/app/store" ) type Controller struct { - connectorStore store.ConnectorStore - authorizer authz.Authorizer - spaceStore store.SpaceStore + connectorStore store.ConnectorStore + connectorService *connector.Service + + authorizer authz.Authorizer + spaceStore store.SpaceStore } func NewController( authorizer authz.Authorizer, connectorStore store.ConnectorStore, + connectorService *connector.Service, spaceStore store.SpaceStore, ) *Controller { return &Controller{ - connectorStore: connectorStore, - authorizer: authorizer, - spaceStore: spaceStore, + connectorStore: connectorStore, + connectorService: connectorService, + authorizer: authorizer, + spaceStore: spaceStore, } } diff --git a/app/api/controller/connector/create.go b/app/api/controller/connector/create.go index c51a1d88d..cadaf3ed2 100644 --- a/app/api/controller/connector/create.go +++ b/app/api/controller/connector/create.go @@ -36,13 +36,11 @@ var ( ) type CreateInput struct { - Description string `json:"description"` - SpaceRef string `json:"space_ref"` // Ref of the parent space - // TODO [CODE-1363]: remove after identifier migration. - UID string `json:"uid" deprecated:"true"` - Identifier string `json:"identifier"` - Type string `json:"type"` - Data string `json:"data"` + Description string `json:"description"` + SpaceRef string `json:"space_ref"` // Ref of the parent space + Identifier string `json:"identifier"` + Type enum.ConnectorType `json:"type"` + types.ConnectorConfig } func (c *Controller) Create( @@ -50,7 +48,7 @@ func (c *Controller) Create( session *auth.Session, in *CreateInput, ) (*types.Connector, error) { - if err := c.sanitizeCreateInput(in); err != nil { + if err := in.validate(); err != nil { return nil, fmt.Errorf("failed to sanitize input: %w", err) } @@ -72,16 +70,19 @@ func (c *Controller) Create( } now := time.Now().UnixMilli() + connector := &types.Connector{ - Description: in.Description, - Data: in.Data, - Type: in.Type, - SpaceID: parentSpace.ID, - Identifier: in.Identifier, - Created: now, - Updated: now, - Version: 0, + Description: in.Description, + CreatedBy: session.Principal.ID, + Type: in.Type, + SpaceID: parentSpace.ID, + Identifier: in.Identifier, + Created: now, + Updated: now, + Version: 0, + ConnectorConfig: in.ConnectorConfig, } + err = c.connectorStore.Create(ctx, connector) if err != nil { return nil, fmt.Errorf("connector creation failed: %w", err) @@ -90,16 +91,20 @@ func (c *Controller) Create( return connector, nil } -func (c *Controller) sanitizeCreateInput(in *CreateInput) error { - // TODO [CODE-1363]: remove after identifier migration. - if in.Identifier == "" { - in.Identifier = in.UID +func (in *CreateInput) validate() error { + parentRefAsID, err := strconv.ParseInt(in.SpaceRef, 10, 64) + if (err == nil && parentRefAsID <= 0) || (len(strings.TrimSpace(in.SpaceRef)) == 0) { + return errConnectorRequiresParent } - parentRefAsID, _ := strconv.ParseInt(in.SpaceRef, 10, 64) + // check that the connector type is valid + if _, ok := in.Type.Sanitize(); !ok { + return usererror.BadRequest("invalid connector type") + } - if parentRefAsID <= 0 || len(strings.TrimSpace(in.SpaceRef)) == 0 { - return errConnectorRequiresParent + // if the connector type is valid, validate the connector config + if err := in.ConnectorConfig.Validate(in.Type); err != nil { + return usererror.BadRequest(fmt.Sprintf("invalid connector config: %s", err.Error())) } if err := check.Identifier(in.Identifier); err != nil { diff --git a/app/api/controller/connector/test.go b/app/api/controller/connector/test.go new file mode 100644 index 000000000..13a5c7478 --- /dev/null +++ b/app/api/controller/connector/test.go @@ -0,0 +1,65 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connector + +import ( + "context" + "fmt" + "time" + + apiauth "github.com/harness/gitness/app/api/auth" + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" + + "github.com/rs/zerolog/log" +) + +func (c *Controller) Test( + ctx context.Context, + session *auth.Session, + spaceRef string, + identifier string, +) (types.ConnectorTestResponse, error) { + space, err := c.spaceStore.FindByRef(ctx, spaceRef) + if err != nil { + return types.ConnectorTestResponse{}, fmt.Errorf("failed to find space: %w", err) + } + err = apiauth.CheckConnector(ctx, c.authorizer, session, space.Path, identifier, enum.PermissionConnectorAccess) + if err != nil { + return types.ConnectorTestResponse{}, fmt.Errorf("failed to authorize: %w", err) + } + connector, err := c.connectorStore.FindByIdentifier(ctx, space.ID, identifier) + if err != nil { + return types.ConnectorTestResponse{}, fmt.Errorf("failed to find connector: %w", err) + } + + resp, err := c.connectorService.Test(ctx, connector) + if err != nil { + return types.ConnectorTestResponse{}, err + } + // Try to update connector last test information in DB. Log but ignore errors + _, err = c.connectorStore.UpdateOptLock(ctx, connector, func(original *types.Connector) error { + original.LastTestErrorMsg = resp.ErrorMsg + original.LastTestStatus = resp.Status + original.LastTestAttempt = time.Now().UnixMilli() + return nil + }) + if err != nil { + log.Ctx(ctx).Warn().Err(err).Msg("failed to update test connection information in connector") + } + + return resp, nil +} diff --git a/app/api/controller/connector/update.go b/app/api/controller/connector/update.go index 5df1c801c..b32da0588 100644 --- a/app/api/controller/connector/update.go +++ b/app/api/controller/connector/update.go @@ -20,6 +20,7 @@ import ( "strings" apiauth "github.com/harness/gitness/app/api/auth" + "github.com/harness/gitness/app/api/usererror" "github.com/harness/gitness/app/auth" "github.com/harness/gitness/types" "github.com/harness/gitness/types/check" @@ -28,11 +29,9 @@ import ( // UpdateInput is used for updating a connector. type UpdateInput struct { - // TODO [CODE-1363]: remove after identifier migration. - UID *string `json:"uid" deprecated:"true"` Identifier *string `json:"identifier"` Description *string `json:"description"` - Data *string `json:"data"` + *types.ConnectorConfig } func (c *Controller) Update( @@ -42,7 +41,7 @@ func (c *Controller) Update( identifier string, in *UpdateInput, ) (*types.Connector, error) { - if err := c.sanitizeUpdateInput(in); err != nil { + if err := in.validate(); err != nil { return nil, fmt.Errorf("failed to sanitize input: %w", err) } @@ -68,20 +67,22 @@ func (c *Controller) Update( if in.Description != nil { original.Description = *in.Description } - if in.Data != nil { - original.Data = *in.Data + // TODO: See if this can be made better. The PATCH API supports partial updates so + // currently we keep all the top level fields the same unless they are explicitly provided. + // The connector config is a nested field so we only check whether it's provided at the top level, and not + // all the fields inside the config. Maybe PUT/POST would be a better option here? + // We can revisit this once we start adding more connectors. + if in.ConnectorConfig != nil { + if err := in.ConnectorConfig.Validate(connector.Type); err != nil { + return usererror.BadRequestf("failed to validate connector config: %s", err.Error()) + } + original.ConnectorConfig = *in.ConnectorConfig } - return nil }) } -func (c *Controller) sanitizeUpdateInput(in *UpdateInput) error { - // TODO [CODE-1363]: remove after identifier migration. - if in.Identifier == nil { - in.Identifier = in.UID - } - +func (in *UpdateInput) validate() error { if in.Identifier != nil { if err := check.Identifier(*in.Identifier); err != nil { return err @@ -95,7 +96,5 @@ func (c *Controller) sanitizeUpdateInput(in *UpdateInput) error { } } - // TODO: Validate Data - return nil } diff --git a/app/api/controller/connector/wire.go b/app/api/controller/connector/wire.go index 8df233d0e..a7a7b9040 100644 --- a/app/api/controller/connector/wire.go +++ b/app/api/controller/connector/wire.go @@ -16,6 +16,7 @@ package connector import ( "github.com/harness/gitness/app/auth/authz" + "github.com/harness/gitness/app/connector" "github.com/harness/gitness/app/store" "github.com/google/wire" @@ -28,8 +29,9 @@ var WireSet = wire.NewSet( func ProvideController( connectorStore store.ConnectorStore, + connectorService *connector.Service, authorizer authz.Authorizer, spaceStore store.SpaceStore, ) *Controller { - return NewController(authorizer, connectorStore, spaceStore) + return NewController(authorizer, connectorStore, connectorService, spaceStore) } diff --git a/app/api/handler/connector/test.go b/app/api/handler/connector/test.go new file mode 100644 index 000000000..e7cdee20f --- /dev/null +++ b/app/api/handler/connector/test.go @@ -0,0 +1,49 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connector + +import ( + "net/http" + + "github.com/harness/gitness/app/api/controller/connector" + "github.com/harness/gitness/app/api/render" + "github.com/harness/gitness/app/api/request" + "github.com/harness/gitness/app/paths" +) + +func HandleTest(connectorCtrl *connector.Controller) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + session, _ := request.AuthSessionFrom(ctx) + connectorRef, err := request.GetConnectorRefFromPath(r) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + spaceRef, connectorIdentifier, err := paths.DisectLeaf(connectorRef) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + resp, err := connectorCtrl.Test(ctx, session, spaceRef, connectorIdentifier) + if err != nil { + render.TranslatedUserError(ctx, w, err) + return + } + + render.JSON(w, http.StatusOK, resp) + } +} diff --git a/app/api/openapi/connector.go b/app/api/openapi/connector.go index ce010eced..4c3403ce8 100644 --- a/app/api/openapi/connector.go +++ b/app/api/openapi/connector.go @@ -86,4 +86,16 @@ func connectorOperations(reflector *openapi3.Reflector) { _ = reflector.SetJSONResponse(&opUpdate, new(usererror.Error), http.StatusForbidden) _ = reflector.SetJSONResponse(&opUpdate, new(usererror.Error), http.StatusNotFound) _ = reflector.Spec.AddOperation(http.MethodPatch, "/connectors/{connector_ref}", opUpdate) + + opTest := openapi3.Operation{} + opTest.WithTags("connector") + opTest.WithMapOfAnything(map[string]interface{}{"operationId": "testConnector"}) + _ = reflector.SetRequest(&opTest, nil, http.MethodPost) + _ = reflector.SetJSONResponse(&opTest, new(types.ConnectorTestResponse), http.StatusOK) + _ = reflector.SetJSONResponse(&opTest, new(usererror.Error), http.StatusBadRequest) + _ = reflector.SetJSONResponse(&opTest, new(usererror.Error), http.StatusInternalServerError) + _ = reflector.SetJSONResponse(&opTest, new(usererror.Error), http.StatusUnauthorized) + _ = reflector.SetJSONResponse(&opTest, new(usererror.Error), http.StatusForbidden) + _ = reflector.SetJSONResponse(&opTest, new(usererror.Error), http.StatusNotFound) + _ = reflector.Spec.AddOperation(http.MethodPost, "/connectors/{connector_ref}/test", opTest) } diff --git a/app/connector/connector.go b/app/connector/connector.go new file mode 100644 index 000000000..4a7188850 --- /dev/null +++ b/app/connector/connector.go @@ -0,0 +1,58 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connector + +import ( + "context" + "time" + + "github.com/harness/gitness/app/connector/scm" + "github.com/harness/gitness/app/store" + "github.com/harness/gitness/types" +) + +var ( + testConnectionTimeout = 5 * time.Second +) + +type Service struct { + secretStore store.SecretStore + // A separate SCM connector service is helpful here since the go-scm library abstracts out all the specific + // SCM interactions, making the interfacing common for all the SCM connectors. + // There might be connectors (eg docker, gcr, etc) in the future which require separate implementations. + // Nevertheless, there should be an attempt to abstract out common functionality for different connector + // types if possible - otherwise separate implementations can be written here. + scmService *scm.Service +} + +func New(secretStore store.SecretStore, scmService *scm.Service) *Service { + return &Service{ + secretStore: secretStore, + scmService: scmService, + } +} + +func (s *Service) Test( + ctx context.Context, + connector *types.Connector, +) (types.ConnectorTestResponse, error) { + // Set a timeout while testing connection. + ctxWithTimeout, cancel := context.WithDeadline(ctx, time.Now().Add(testConnectionTimeout)) + defer cancel() + if connector.Type.IsSCM() { + return s.scmService.Test(ctxWithTimeout, connector) + } + return types.ConnectorTestResponse{}, nil +} diff --git a/app/connector/scm/provider.go b/app/connector/scm/provider.go new file mode 100644 index 000000000..3cf3ec339 --- /dev/null +++ b/app/connector/scm/provider.go @@ -0,0 +1,112 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scm + +import ( + "context" + "fmt" + "net/http" + + "github.com/harness/gitness/app/store" + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" + + "github.com/drone/go-scm/scm" + "github.com/drone/go-scm/scm/driver/github" + "github.com/drone/go-scm/scm/transport/oauth2" +) + +// getSCMProvider returns an SCM client given a connector. +// The SCM client can be used as a common layer for interfacing with any SCM. +func getSCMProvider( + ctx context.Context, + connector *types.Connector, + secretStore store.SecretStore, +) (*scm.Client, error) { + var client *scm.Client + var err error + var transport http.RoundTripper + + switch x := connector.Type; x { + case enum.ConnectorTypeGithub: + if connector.Github == nil { + return nil, fmt.Errorf("github connector is nil") + } + if connector.Github.APIURL == "" { + client = github.NewDefault() + } else { + client, err = github.New(connector.Github.APIURL) + if err != nil { + return nil, err + } + } + if connector.Github.Auth == nil { + return nil, fmt.Errorf("github auth needs to be provided") + } + if connector.Github.Auth.AuthType == enum.ConnectorAuthTypeBearer { + creds := connector.Github.Auth.Bearer + pass, err := resolveSecret(ctx, connector.SpaceID, creds.Token, secretStore) + if err != nil { + return nil, err + } + transport = oauthTransport(pass, oauth2.SchemeBearer) + } else { + return nil, fmt.Errorf("unsupported auth type for github connector: %s", connector.Github.Auth.AuthType) + } + default: + return nil, fmt.Errorf("unsupported scm provider type: %s", x) + } + + // override default transport if available + if transport != nil { + client.Client = &http.Client{Transport: transport} + } + + return client, nil +} + +func oauthTransport(token string, scheme string) http.RoundTripper { + if token == "" { + return nil + } + return &oauth2.Transport{ + Base: defaultTransport(), + Scheme: scheme, + Source: oauth2.StaticTokenSource(&scm.Token{Token: token}), + } +} + +// defaultTransport provides a default http.Transport. +// This can be extended when needed for things like more advanced TLS config, proxies, etc. +func defaultTransport() http.RoundTripper { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + } +} + +// resolveSecret looks into the secret store to find the value of a secret. +func resolveSecret( + ctx context.Context, + spaceID int64, + ref types.SecretRef, + secretStore store.SecretStore, +) (string, error) { + // the secret should be in the same space as the connector + s, err := secretStore.FindByIdentifier(ctx, spaceID, ref.Identifier) + if err != nil { + return "", fmt.Errorf("could not find secret from store: %w", err) + } + return s.Data, nil +} diff --git a/app/connector/scm/scm.go b/app/connector/scm/scm.go new file mode 100644 index 000000000..a5c063e6b --- /dev/null +++ b/app/connector/scm/scm.go @@ -0,0 +1,50 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scm + +import ( + "context" + "fmt" + + "github.com/harness/gitness/app/store" + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" +) + +type Service struct { + secretStore store.SecretStore +} + +func NewService(secretStore store.SecretStore) *Service { + return &Service{ + secretStore: secretStore, + } +} + +func (s *Service) Test(ctx context.Context, c *types.Connector) (types.ConnectorTestResponse, error) { + if !c.Type.IsSCM() { + return types.ConnectorTestResponse{}, fmt.Errorf("connector type: %s is not an SCM connector", c.Type.String()) + } + client, err := getSCMProvider(ctx, c, s.secretStore) + if err != nil { + return types.ConnectorTestResponse{}, err + } + // Check whether a valid user exists - if yes, the connection is successful + _, _, err = client.Users.Find(ctx) + if err != nil { + return types.ConnectorTestResponse{Status: enum.ConnectorStatusFailed, ErrorMsg: err.Error()}, nil //nolint:nilerr + } + return types.ConnectorTestResponse{Status: enum.ConnectorStatusSuccess}, nil +} diff --git a/app/connector/wire.go b/app/connector/wire.go new file mode 100644 index 000000000..90e634390 --- /dev/null +++ b/app/connector/wire.go @@ -0,0 +1,42 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connector + +import ( + "github.com/harness/gitness/app/connector/scm" + "github.com/harness/gitness/app/store" + + "github.com/google/wire" +) + +// WireSet provides a wire set for this package. +var WireSet = wire.NewSet( + ProvideConnectorHandler, + ProvideSCMConnectorHandler, +) + +// ProvideConnectorHandler provides a connector handler for handling connector-related ops. +func ProvideConnectorHandler( + secretStore store.SecretStore, + scmService *scm.Service, +) *Service { + return New(secretStore, scmService) +} + +// ProvideSCMConnectorHandler provides a SCM connector handler for specifically handling +// SCM connector related ops. +func ProvideSCMConnectorHandler(secretStore store.SecretStore) *scm.Service { + return scm.NewService(secretStore) +} diff --git a/app/router/api.go b/app/router/api.go index 59a194e27..475546da6 100644 --- a/app/router/api.go +++ b/app/router/api.go @@ -503,6 +503,7 @@ func setupConnectors( r.Get("/", handlerconnector.HandleFind(connectorCtrl)) r.Patch("/", handlerconnector.HandleUpdate(connectorCtrl)) r.Delete("/", handlerconnector.HandleDelete(connectorCtrl)) + r.Post("/test", handlerconnector.HandleTest(connectorCtrl)) }) }) } diff --git a/app/store/database/connector.go b/app/store/database/connector.go index 8af4e2cd9..0fcb4254a 100644 --- a/app/store/database/connector.go +++ b/app/store/database/connector.go @@ -16,6 +16,7 @@ package database import ( "context" + "database/sql" "fmt" "strings" "time" @@ -25,6 +26,7 @@ import ( "github.com/harness/gitness/store/database" "github.com/harness/gitness/store/database/dbtx" "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" "github.com/jmoiron/sqlx" "github.com/pkg/errors" @@ -40,25 +42,265 @@ const ( connectorColumns = ` connector_id, + connector_identifier, connector_description, + connector_type, + connector_auth_type, + connector_created_by, connector_space_id, - connector_uid, - connector_data, + connector_last_test_attempt, + connector_last_test_error_msg, + connector_last_test_status, connector_created, connector_updated, - connector_version + connector_version, + connector_address, + connector_insecure, + connector_username, + connector_github_app_installation_id, + connector_github_app_application_id, + connector_region, + connector_password, + connector_token, + connector_aws_key, + connector_aws_secret, + connector_github_app_private_key, + connector_token_refresh ` ) +type connector struct { + ID int64 `db:"connector_id"` + Identifier string `db:"connector_identifier"` + Description string `db:"connector_description"` + Type string `db:"connector_type"` + AuthType string `db:"connector_auth_type"` + CreatedBy int64 `db:"connector_created_by"` + SpaceID int64 `db:"connector_space_id"` + LastTestAttempt int64 `db:"connector_last_test_attempt"` + LastTestErrorMsg string `db:"connector_last_test_error_msg"` + LastTestStatus string `db:"connector_last_test_status"` + Created int64 `db:"connector_created"` + Updated int64 `db:"connector_updated"` + Version int64 `db:"connector_version"` + + Address sql.NullString `db:"connector_address"` + Insecure sql.NullBool `db:"connector_insecure"` + Username sql.NullString `db:"connector_username"` + GithubAppInstallationID sql.NullString `db:"connector_github_app_installation_id"` + GithubAppApplicationID sql.NullString `db:"connector_github_app_application_id"` + Region sql.NullString `db:"connector_region"` + // Password fields are stored as reference to secrets table + Password sql.NullInt64 `db:"connector_password"` + Token sql.NullInt64 `db:"connector_token"` + AWSKey sql.NullInt64 `db:"connector_aws_key"` + AWSSecret sql.NullInt64 `db:"connector_aws_secret"` + GithubAppPrivateKey sql.NullInt64 `db:"connector_github_app_private_key"` + TokenRefresh sql.NullInt64 `db:"connector_token_refresh"` +} + // NewConnectorStore returns a new ConnectorStore. -func NewConnectorStore(db *sqlx.DB) store.ConnectorStore { +// The secret store is used to resolve the secret references. +func NewConnectorStore(db *sqlx.DB, secretStore store.SecretStore) store.ConnectorStore { return &connectorStore{ - db: db, + db: db, + secretStore: secretStore, } } +func (s *connectorStore) mapFromDBConnectors(ctx context.Context, src []*connector) ([]*types.Connector, error) { + dst := make([]*types.Connector, len(src)) + for i, v := range src { + m, err := s.mapFromDBConnector(ctx, v) + if err != nil { + return nil, fmt.Errorf("could not map from db connector: %w", err) + } + dst[i] = m + } + return dst, nil +} + +func (s *connectorStore) mapToDBConnector(ctx context.Context, v *types.Connector) (*connector, error) { + to := connector{ + ID: v.ID, + Identifier: v.Identifier, + Description: v.Description, + Type: v.Type.String(), + SpaceID: v.SpaceID, + CreatedBy: v.CreatedBy, + Created: v.Created, + Updated: v.Updated, + Version: v.Version, + LastTestAttempt: v.LastTestAttempt, + LastTestErrorMsg: v.LastTestErrorMsg, + LastTestStatus: v.LastTestStatus.String(), + } + // Parse connector specific configs + err := s.convertConfigToDB(ctx, v, &to) + if err != nil { + return nil, fmt.Errorf("could not convert config to db: %w", err) + } + return &to, nil +} + +func (s *connectorStore) convertConfigToDB( + ctx context.Context, + source *types.Connector, + to *connector, +) error { + switch { + case source.Github != nil: + to.Address = sql.NullString{String: source.Github.APIURL, Valid: true} + to.Insecure = sql.NullBool{Bool: source.Github.Insecure, Valid: true} + if source.Github.Auth == nil { + return fmt.Errorf("auth is required for github connectors") + } + if source.Github.Auth.AuthType != enum.ConnectorAuthTypeBearer { + return fmt.Errorf("only bearer token auth is supported for github connectors") + } + to.AuthType = source.Github.Auth.AuthType.String() + creds := source.Github.Auth.Bearer + // use the same space ID as the connector + tokenID, err := s.secretIdentiferToID(ctx, creds.Token.Identifier, source.SpaceID) + if err != nil { + return fmt.Errorf("could not find secret: %w", err) + } + to.Token = sql.NullInt64{Int64: tokenID, Valid: true} + default: + return fmt.Errorf("no connector config found for type: %s", source.Type) + } + return nil +} + +// secretIdentiferToID finds the secret ID given the space ID and the identifier. +func (s *connectorStore) secretIdentiferToID( + ctx context.Context, + identifier string, + spaceID int64, +) (int64, error) { + secret, err := s.secretStore.FindByIdentifier(ctx, spaceID, identifier) + if err != nil { + return 0, err + } + return secret.ID, nil +} + +func (s *connectorStore) mapFromDBConnector( + ctx context.Context, + dbConnector *connector, +) (*types.Connector, error) { + connector := &types.Connector{ + ID: dbConnector.ID, + Identifier: dbConnector.Identifier, + Description: dbConnector.Description, + Type: enum.ConnectorType(dbConnector.Type), + SpaceID: dbConnector.SpaceID, + CreatedBy: dbConnector.CreatedBy, + LastTestAttempt: dbConnector.LastTestAttempt, + LastTestErrorMsg: dbConnector.LastTestErrorMsg, + LastTestStatus: enum.ConnectorStatus(dbConnector.LastTestStatus), + Created: dbConnector.Created, + Updated: dbConnector.Updated, + Version: dbConnector.Version, + } + err := s.populateConnectorData(ctx, dbConnector, connector) + if err != nil { + return nil, fmt.Errorf("could not populate connector data: %w", err) + } + return connector, nil +} + +func (s *connectorStore) populateConnectorData( + ctx context.Context, + source *connector, + to *types.Connector, +) error { + switch enum.ConnectorType(source.Type) { + case enum.ConnectorTypeGithub: + githubData, err := s.parseGithubConnectorData(ctx, source) + if err != nil { + return fmt.Errorf("could not parse github connector data: %w", err) + } + to.Github = githubData + // Cases for other connectors can be added here + default: + return fmt.Errorf("unsupported connector type: %s", source.Type) + } + return nil +} + +func (s *connectorStore) parseGithubConnectorData( + ctx context.Context, + connector *connector, +) (*types.GithubConnectorData, error) { + auth, err := s.parseAuthenticationData(ctx, connector) + if err != nil { + return nil, fmt.Errorf("could not parse authentication data: %w", err) + } + return &types.GithubConnectorData{ + APIURL: connector.Address.String, + Insecure: connector.Insecure.Bool, + Auth: auth, + }, nil +} + +func (s *connectorStore) parseAuthenticationData( + ctx context.Context, + connector *connector, +) (*types.ConnectorAuth, error) { + authType, err := enum.ParseConnectorAuthType(connector.AuthType) + if err != nil { + return nil, err + } + + switch authType { + case enum.ConnectorAuthTypeBasic: + if !connector.Username.Valid || !connector.Password.Valid { + return nil, fmt.Errorf("basic auth requires both username and password") + } + passwordRef, err := s.convertToRef(ctx, connector.Password.Int64) + if err != nil { + return nil, fmt.Errorf("could not convert basicauth password to ref: %w", err) + } + return &types.ConnectorAuth{ + AuthType: enum.ConnectorAuthTypeBasic, + Basic: &types.BasicAuthCreds{ + Username: connector.Username.String, + Password: passwordRef, + }, + }, nil + case enum.ConnectorAuthTypeBearer: + if !connector.Token.Valid { + return nil, fmt.Errorf("bearer auth requires a token") + } + tokenRef, err := s.convertToRef(ctx, connector.Token.Int64) + if err != nil { + return nil, fmt.Errorf("could not convert bearer token to ref: %w", err) + } + return &types.ConnectorAuth{ + AuthType: enum.ConnectorAuthTypeBearer, + Bearer: &types.BearerTokenCreds{ + Token: tokenRef, + }, + }, nil + default: + return nil, fmt.Errorf("unsupported auth type: %s", connector.AuthType) + } +} + +func (s *connectorStore) convertToRef(ctx context.Context, id int64) (types.SecretRef, error) { + secret, err := s.secretStore.Find(ctx, id) + if err != nil { + return types.SecretRef{}, err + } + return types.SecretRef{ + Identifier: secret.Identifier, + }, nil +} + type connectorStore struct { - db *sqlx.DB + db *sqlx.DB + secretStore store.SecretStore } // Find returns a connector given a connector ID. @@ -67,11 +309,11 @@ func (s *connectorStore) Find(ctx context.Context, id int64) (*types.Connector, WHERE connector_id = $1` db := dbtx.GetAccessor(ctx, s.db) - dst := new(types.Connector) + dst := new(connector) if err := db.GetContext(ctx, dst, findQueryStmt, id); err != nil { return nil, database.ProcessSQLErrorf(ctx, err, "Failed to find connector") } - return dst, nil + return s.mapFromDBConnector(ctx, dst) } // FindByIdentifier returns a connector in a given space with a given identifier. @@ -81,41 +323,77 @@ func (s *connectorStore) FindByIdentifier( identifier string, ) (*types.Connector, error) { const findQueryStmt = connectorQueryBase + ` - WHERE connector_space_id = $1 AND connector_uid = $2` + WHERE connector_space_id = $1 AND connector_identifier = $2` db := dbtx.GetAccessor(ctx, s.db) - dst := new(types.Connector) + dst := new(connector) if err := db.GetContext(ctx, dst, findQueryStmt, spaceID, identifier); err != nil { return nil, database.ProcessSQLErrorf(ctx, err, "Failed to find connector") } - return dst, nil + return s.mapFromDBConnector(ctx, dst) } // Create creates a connector. func (s *connectorStore) Create(ctx context.Context, connector *types.Connector) error { + dbConnector, err := s.mapToDBConnector(ctx, connector) + if err != nil { + return err + } const connectorInsertStmt = ` INSERT INTO connectors ( connector_description ,connector_type + ,connector_created_by ,connector_space_id - ,connector_uid - ,connector_data + ,connector_identifier + ,connector_last_test_attempt + ,connector_last_test_error_msg + ,connector_last_test_status ,connector_created ,connector_updated ,connector_version + ,connector_auth_type + ,connector_address + ,connector_insecure + ,connector_username + ,connector_github_app_installation_id + ,connector_github_app_application_id + ,connector_region + ,connector_password + ,connector_token + ,connector_aws_key + ,connector_aws_secret + ,connector_github_app_private_key + ,connector_token_refresh ) VALUES ( :connector_description ,:connector_type + ,:connector_created_by ,:connector_space_id - ,:connector_uid - ,:connector_data + ,:connector_identifier + ,:connector_last_test_attempt + ,:connector_last_test_error_msg + ,:connector_last_test_status ,:connector_created ,:connector_updated ,:connector_version + ,:connector_auth_type + ,:connector_address + ,:connector_insecure + ,:connector_username + ,:connector_github_app_installation_id + ,:connector_github_app_application_id + ,:connector_region + ,:connector_password + ,:connector_token + ,:connector_aws_key + ,:connector_aws_secret + ,:connector_github_app_private_key + ,:connector_token_refresh ) RETURNING connector_id` db := dbtx.GetAccessor(ctx, s.db) - query, arg, err := db.BindNamed(connectorInsertStmt, connector) + query, arg, err := db.BindNamed(connectorInsertStmt, dbConnector) if err != nil { return database.ProcessSQLErrorf(ctx, err, "Failed to bind connector object") } @@ -128,24 +406,42 @@ func (s *connectorStore) Create(ctx context.Context, connector *types.Connector) } func (s *connectorStore) Update(ctx context.Context, p *types.Connector) error { + conn, err := s.mapToDBConnector(ctx, p) + if err != nil { + return err + } const connectorUpdateStmt = ` UPDATE connectors SET connector_description = :connector_description - ,connector_uid = :connector_uid - ,connector_data = :connector_data - ,connector_type = :connector_type + ,connector_identifier = :connector_identifier + ,connector_last_test_attempt = :connector_last_test_attempt + ,connector_last_test_error_msg = :connector_last_test_error_msg + ,connector_last_test_status = :connector_last_test_status ,connector_updated = :connector_updated ,connector_version = :connector_version + ,connector_auth_type = :connector_auth_type + ,connector_address = :connector_address + ,connector_insecure = :connector_insecure + ,connector_username = :connector_username + ,connector_github_app_installation_id = :connector_github_app_installation_id + ,connector_github_app_application_id = :connector_github_app_application_id + ,connector_region = :connector_region + ,connector_password = :connector_password + ,connector_token = :connector_token + ,connector_aws_key = :connector_aws_key + ,connector_aws_secret = :connector_aws_secret + ,connector_github_app_private_key = :connector_github_app_private_key + ,connector_token_refresh = :connector_token_refresh WHERE connector_id = :connector_id AND connector_version = :connector_version - 1` - connector := *p + o := *conn - connector.Version++ - connector.Updated = time.Now().UnixMilli() + o.Version++ + o.Updated = time.Now().UnixMilli() db := dbtx.GetAccessor(ctx, s.db) - query, arg, err := db.BindNamed(connectorUpdateStmt, connector) + query, arg, err := db.BindNamed(connectorUpdateStmt, o) if err != nil { return database.ProcessSQLErrorf(ctx, err, "Failed to bind connector object") } @@ -164,8 +460,8 @@ func (s *connectorStore) Update(ctx context.Context, p *types.Connector) error { return gitness_store.ErrVersionConflict } - p.Version = connector.Version - p.Updated = connector.Updated + p.Version = o.Version + p.Updated = o.Updated return nil } @@ -209,7 +505,7 @@ func (s *connectorStore) List( Where("connector_space_id = ?", fmt.Sprint(parentID)) if filter.Query != "" { - stmt = stmt.Where("LOWER(connector_uid) LIKE ?", fmt.Sprintf("%%%s%%", strings.ToLower(filter.Query))) + stmt = stmt.Where("LOWER(connector_identifier) LIKE ?", fmt.Sprintf("%%%s%%", strings.ToLower(filter.Query))) } stmt = stmt.Limit(database.Limit(filter.Size)) @@ -222,12 +518,12 @@ func (s *connectorStore) List( db := dbtx.GetAccessor(ctx, s.db) - dst := []*types.Connector{} + dst := []*connector{} if err = db.SelectContext(ctx, &dst, sql, args...); err != nil { return nil, database.ProcessSQLErrorf(ctx, err, "Failed executing custom list query") } - return dst, nil + return s.mapFromDBConnectors(ctx, dst) } // Delete deletes a connector given a connector ID. @@ -249,7 +545,7 @@ func (s *connectorStore) Delete(ctx context.Context, id int64) error { func (s *connectorStore) DeleteByIdentifier(ctx context.Context, spaceID int64, identifier string) error { const connectorDeleteStmt = ` DELETE FROM connectors - WHERE connector_space_id = $1 AND connector_uid = $2` + WHERE connector_space_id = $1 AND connector_identifier = $2` db := dbtx.GetAccessor(ctx, s.db) @@ -268,7 +564,7 @@ func (s *connectorStore) Count(ctx context.Context, parentID int64, filter types Where("connector_space_id = ?", parentID) if filter.Query != "" { - stmt = stmt.Where("LOWER(connector_uid) LIKE ?", fmt.Sprintf("%%%s%%", filter.Query)) + stmt = stmt.Where("LOWER(connector_identifier) LIKE ?", fmt.Sprintf("%%%s%%", filter.Query)) } sql, args, err := stmt.ToSql() diff --git a/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.up.sql b/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.up.sql index 4f663f157..8e3ca1429 100644 --- a/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.up.sql +++ b/app/store/database/migrate/postgres/0072_create_ar_table_images_and_alter_table_artifacts.up.sql @@ -23,7 +23,7 @@ ALTER TABLE artifacts DROP CONSTRAINT fk_registries_registry_id; ALTER TABLE artifacts DROP COLUMN artifact_name; ALTER TABLE artifacts DROP COLUMN artifact_registry_id; -ALTER TABLE artifacts DROP COLUMN artifact_labels;.. +ALTER TABLE artifacts DROP COLUMN artifact_labels; ALTER TABLE artifacts DROP COLUMN artifact_enabled; ALTER TABLE artifacts ADD COLUMN artifact_version TEXT NOT NULL; diff --git a/app/store/database/migrate/postgres/0074_create_table_connectors.down.sql b/app/store/database/migrate/postgres/0074_create_table_connectors.down.sql new file mode 100644 index 000000000..8eecb411c --- /dev/null +++ b/app/store/database/migrate/postgres/0074_create_table_connectors.down.sql @@ -0,0 +1 @@ +DROP TABLE IF exists connectors; \ No newline at end of file diff --git a/app/store/database/migrate/postgres/0074_create_table_connectors.up.sql b/app/store/database/migrate/postgres/0074_create_table_connectors.up.sql new file mode 100644 index 000000000..199309562 --- /dev/null +++ b/app/store/database/migrate/postgres/0074_create_table_connectors.up.sql @@ -0,0 +1,88 @@ +-- Connectors table is not being used so can be dropped and recreated without +-- worrying about a migration +DROP TABLE IF EXISTS connectors; + +CREATE TABLE connectors ( + -- Fields valid for all connectors + connector_id SERIAL PRIMARY KEY, + connector_identifier TEXT NOT NULL, + connector_description TEXT NOT NULL, + connector_type TEXT NOT NULL, + connector_auth_type TEXT NOT NULL, -- basicauth, oidc, oauth, aws + connector_created_by INTEGER NOT NULL, + connector_space_id INTEGER NOT NULL, + connector_last_test_attempt INTEGER NOT NULL, + connector_last_test_error_msg TEXT NOT NULL, + connector_last_test_status TEXT NOT NULL, + connector_created BIGINT NOT NULL, + connector_updated BIGINT NOT NULL, + connector_version INTEGER NOT NULL, + connector_address TEXT, + connector_insecure BOOLEAN, + + -- Fields used by different connectors based on the auth_type + connector_username TEXT, + connector_github_app_installation_id TEXT, + connector_github_app_application_id TEXT, + connector_region TEXT, + + -- secrets (foreign keys to the secrets table and restricted on delete) + connector_password INTEGER, + connector_token INTEGER, + connector_aws_key INTEGER, + connector_aws_secret INTEGER, + connector_github_app_private_key INTEGER, + connector_token_refresh INTEGER, + + -- Foreign key to spaces table + CONSTRAINT fk_connectors_space_id FOREIGN KEY (connector_space_id) + REFERENCES spaces (space_id) + ON UPDATE NO ACTION + ON DELETE CASCADE, + + -- Foreign key to principals table + CONSTRAINT fk_connectors_created_by FOREIGN KEY (connector_created_by) + REFERENCES principals (principal_id) + ON UPDATE NO ACTION + ON DELETE NO ACTION, + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_password FOREIGN KEY (connector_password) + REFERENCES secrets (secret_id) + ON UPDATE NO ACTION + ON DELETE RESTRICT, + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_token FOREIGN KEY (connector_token) + REFERENCES secrets (secret_id) + ON UPDATE NO ACTION + ON DELETE RESTRICT, + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_aws_key FOREIGN KEY (connector_aws_key) + REFERENCES secrets (secret_id) + ON UPDATE NO ACTION + ON DELETE RESTRICT, + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_aws_secret FOREIGN KEY (connector_aws_secret) + REFERENCES secrets (secret_id) + ON UPDATE NO ACTION + ON DELETE RESTRICT, + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_github_app_private_key FOREIGN KEY (connector_github_app_private_key) + REFERENCES secrets (secret_id) + ON UPDATE NO ACTION + ON DELETE RESTRICT, + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_token_refresh FOREIGN KEY (connector_token_refresh) + REFERENCES secrets (secret_id) + ON UPDATE NO ACTION + ON DELETE RESTRICT +); + +-- Creating a unique index for case-insensitive connector identifiers +CREATE UNIQUE INDEX unique_connector_lowercase_identifier +ON connectors(connector_space_id, LOWER(connector_identifier)); diff --git a/app/store/database/migrate/sqlite/0074_create_table_connectors.down.sql b/app/store/database/migrate/sqlite/0074_create_table_connectors.down.sql new file mode 100644 index 000000000..26c91e895 --- /dev/null +++ b/app/store/database/migrate/sqlite/0074_create_table_connectors.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS connectors; \ No newline at end of file diff --git a/app/store/database/migrate/sqlite/0074_create_table_connectors.up.sql b/app/store/database/migrate/sqlite/0074_create_table_connectors.up.sql new file mode 100644 index 000000000..2b38e7424 --- /dev/null +++ b/app/store/database/migrate/sqlite/0074_create_table_connectors.up.sql @@ -0,0 +1,88 @@ +-- Connectors table is not being used so can be dropped and recreated without +-- worrying about a migration +DROP TABLE IF EXISTS connectors; + +CREATE TABLE connectors ( + -- Fields valid for all connectors + connector_id INTEGER PRIMARY KEY AUTOINCREMENT, + connector_identifier TEXT NOT NULL, + connector_description TEXT NOT NULL, + connector_type TEXT NOT NULL, + connector_auth_type TEXT NOT NULL, -- basicauth, oidc, oauth, aws + connector_created_by INTEGER NOT NULL, + connector_space_id INTEGER NOT NULL, + connector_last_test_attempt INTEGER NOT NULL, + connector_last_test_error_msg TEXT NOT NULL, + connector_last_test_status TEXT NOT NULL, + connector_created INTEGER NOT NULL, + connector_updated INTEGER NOT NULL, + connector_version INTEGER NOT NULL, + connector_address TEXT, + connector_insecure BOOLEAN, + + -- Fields used by different connectors based on the auth_type + connector_username TEXT, + connector_github_app_installation_id TEXT, + connector_github_app_application_id TEXT, + connector_region TEXT, + + -- secrets (foreign keys to the secrets table and restricted on delete) + connector_password INTEGER, + connector_token INTEGER, + connector_aws_key INTEGER, + connector_aws_secret INTEGER, + connector_github_app_private_key INTEGER, + connector_token_refresh INTEGER, + + -- Foreign key to spaces table + CONSTRAINT fk_connectors_space_id FOREIGN KEY (connector_space_id) + REFERENCES spaces (space_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE CASCADE, + + -- Foreign key to principals table + CONSTRAINT fk_connectors_created_by FOREIGN KEY (connector_created_by) + REFERENCES principals (principal_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE NO ACTION, + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_password FOREIGN KEY (connector_password) + REFERENCES secrets (secret_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE RESTRICT, + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_token FOREIGN KEY (connector_token) + REFERENCES secrets (secret_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE RESTRICT, + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_aws_key FOREIGN KEY (connector_aws_key) + REFERENCES secrets (secret_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE RESTRICT, + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_aws_secret FOREIGN KEY (connector_aws_secret) + REFERENCES secrets (secret_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE RESTRICT + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_github_app_private_key FOREIGN KEY (connector_github_app_private_key) + REFERENCES secrets (secret_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE RESTRICT + + -- Foreign key to secrets table + CONSTRAINT fk_connectors_token_refresh FOREIGN KEY (connector_token_refresh) + REFERENCES secrets (secret_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE RESTRICT +); + +-- Creating a unique index for case-insensitive connector identifiers +CREATE UNIQUE INDEX unique_connector_lowercase_identifier +ON connectors(connector_space_id, LOWER(connector_identifier)); diff --git a/app/store/database/wire.go b/app/store/database/wire.go index 8d843005d..051f407c7 100644 --- a/app/store/database/wire.go +++ b/app/store/database/wire.go @@ -205,8 +205,8 @@ func ProvideSecretStore(db *sqlx.DB) store.SecretStore { } // ProvideConnectorStore provides a connector store. -func ProvideConnectorStore(db *sqlx.DB) store.ConnectorStore { - return NewConnectorStore(db) +func ProvideConnectorStore(db *sqlx.DB, secretStore store.SecretStore) store.ConnectorStore { + return NewConnectorStore(db, secretStore) } // ProvideTemplateStore provides a template store. diff --git a/cmd/gitness/wire.go b/cmd/gitness/wire.go index 7e717b58d..39bb5941b 100644 --- a/cmd/gitness/wire.go +++ b/cmd/gitness/wire.go @@ -53,6 +53,7 @@ import ( "github.com/harness/gitness/app/auth/authn" "github.com/harness/gitness/app/auth/authz" "github.com/harness/gitness/app/bootstrap" + connectorservice "github.com/harness/gitness/app/connector" gitevents "github.com/harness/gitness/app/events/git" gitspaceevents "github.com/harness/gitness/app/events/gitspace" gitspaceinfraevents "github.com/harness/gitness/app/events/gitspaceinfra" @@ -215,6 +216,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e controllerlogs.WireSet, secret.WireSet, connector.WireSet, + connectorservice.WireSet, template.WireSet, manager.WireSet, triggerer.WireSet, diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index 3ed148e1a..4b0b634dc 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -12,7 +12,7 @@ import ( aiagent2 "github.com/harness/gitness/app/api/controller/aiagent" capabilities2 "github.com/harness/gitness/app/api/controller/capabilities" check2 "github.com/harness/gitness/app/api/controller/check" - "github.com/harness/gitness/app/api/controller/connector" + connector2 "github.com/harness/gitness/app/api/controller/connector" "github.com/harness/gitness/app/api/controller/execution" "github.com/harness/gitness/app/api/controller/githook" gitspace2 "github.com/harness/gitness/app/api/controller/gitspace" @@ -42,6 +42,7 @@ import ( "github.com/harness/gitness/app/auth/authn" "github.com/harness/gitness/app/auth/authz" "github.com/harness/gitness/app/bootstrap" + "github.com/harness/gitness/app/connector" events6 "github.com/harness/gitness/app/events/git" events7 "github.com/harness/gitness/app/events/gitspace" events3 "github.com/harness/gitness/app/events/gitspaceinfra" @@ -269,7 +270,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro logsController := logs2.ProvideController(authorizer, executionStore, repoStore, pipelineStore, stageStore, stepStore, logStore, logStream) spaceIdentifier := check.ProvideSpaceIdentifierCheck() secretStore := database.ProvideSecretStore(db) - connectorStore := database.ProvideConnectorStore(db) + connectorStore := database.ProvideConnectorStore(db, secretStore) repoGitInfoView := database.ProvideRepoGitInfoView(db) repoGitInfoCache := cache.ProvideRepoGitInfoCache(repoGitInfoView) pullReqStore := database.ProvidePullReqStore(db, principalInfoCache) @@ -306,7 +307,9 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro pipelineController := pipeline.ProvideController(repoStore, triggerStore, authorizer, pipelineStore, reporter2) secretController := secret.ProvideController(encrypter, secretStore, authorizer, spaceStore) triggerController := trigger.ProvideController(authorizer, triggerStore, pipelineStore, repoStore) - connectorController := connector.ProvideController(connectorStore, authorizer, spaceStore) + scmService := connector.ProvideSCMConnectorHandler(secretStore) + connectorService := connector.ProvideConnectorHandler(secretStore, scmService) + connectorController := connector2.ProvideController(connectorStore, connectorService, authorizer, spaceStore) templateController := template.ProvideController(templateStore, authorizer, spaceStore) pluginController := plugin.ProvideController(pluginStore) pullReqActivityStore := database.ProvidePullReqActivityStore(db, principalInfoCache) diff --git a/types/connector.go b/types/connector.go index 51fa07a20..9d0924c8d 100644 --- a/types/connector.go +++ b/types/connector.go @@ -14,29 +14,24 @@ package types -import "encoding/json" +import ( + "github.com/harness/gitness/types/enum" +) type Connector struct { - ID int64 `db:"connector_id" json:"-"` - Description string `db:"connector_description" json:"description"` - SpaceID int64 `db:"connector_space_id" json:"space_id"` - Identifier string `db:"connector_uid" json:"identifier"` - Type string `db:"connector_type" json:"type"` - Data string `db:"connector_data" json:"data"` - Created int64 `db:"connector_created" json:"created"` - Updated int64 `db:"connector_updated" json:"updated"` - Version int64 `db:"connector_version" json:"-"` -} + ID int64 `json:"-"` + Description string `json:"description"` + SpaceID int64 `json:"space_id"` + Identifier string `json:"identifier"` + CreatedBy int64 `json:"created_by"` + Type enum.ConnectorType `json:"type"` + LastTestAttempt int64 `json:"last_test_attempt"` + LastTestErrorMsg string `json:"last_test_error_msg"` + LastTestStatus enum.ConnectorStatus `json:"last_test_status"` + Created int64 `json:"created"` + Updated int64 `json:"updated"` + Version int64 `json:"-"` -// TODO [CODE-1363]: remove after identifier migration. -func (s Connector) MarshalJSON() ([]byte, error) { - // alias allows us to embed the original object while avoiding an infinite loop of marshaling. - type alias Connector - return json.Marshal(&struct { - alias - UID string `json:"uid"` - }{ - alias: (alias)(s), - UID: s.Identifier, - }) + // Pointers to connector specific data + ConnectorConfig } diff --git a/types/connector_auth.go b/types/connector_auth.go new file mode 100644 index 000000000..638742fbc --- /dev/null +++ b/types/connector_auth.go @@ -0,0 +1,60 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "fmt" + + "github.com/harness/gitness/types/enum" +) + +// ConnectorAuth represents the authentication configuration for a connector. +type ConnectorAuth struct { + AuthType enum.ConnectorAuthType `json:"type"` + Basic *BasicAuthCreds `json:"basic,omitempty"` + Bearer *BearerTokenCreds `json:"bearer,omitempty"` +} + +// BasicAuthCreds represents credentials for basic authentication. +type BasicAuthCreds struct { + Username string `json:"username"` + Password SecretRef `json:"password"` +} + +type BearerTokenCreds struct { + Token SecretRef `json:"token"` +} + +func (c *ConnectorAuth) Validate() error { + switch c.AuthType { + case enum.ConnectorAuthTypeBasic: + if c.Basic == nil { + return fmt.Errorf("basic auth credentials are required") + } + if c.Basic.Username == "" || c.Basic.Password.Identifier == "" { + return fmt.Errorf("basic auth credentials are required") + } + case enum.ConnectorAuthTypeBearer: + if c.Bearer == nil { + return fmt.Errorf("bearer token credentials are required") + } + if c.Bearer.Token.Identifier == "" { + return fmt.Errorf("bearer token is required") + } + default: + return fmt.Errorf("unsupported auth type: %s", c.AuthType) + } + return nil +} diff --git a/types/connector_config.go b/types/connector_config.go new file mode 100644 index 000000000..787b39d76 --- /dev/null +++ b/types/connector_config.go @@ -0,0 +1,38 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "fmt" + + "github.com/harness/gitness/types/enum" +) + +// ConnectorConfig is a list of all the connector and their associated config. +type ConnectorConfig struct { + Github *GithubConnectorData `json:"github,omitempty"` +} + +func (c ConnectorConfig) Validate(typ enum.ConnectorType) error { + switch typ { + case enum.ConnectorTypeGithub: + if c.Github != nil { + return c.Github.Validate() + } + return fmt.Errorf("github connector config is required") + default: + return fmt.Errorf("connector type %s is not supported", typ) + } +} diff --git a/types/connector_test_response.go b/types/connector_test_response.go new file mode 100644 index 000000000..5f07d7bdb --- /dev/null +++ b/types/connector_test_response.go @@ -0,0 +1,22 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import "github.com/harness/gitness/types/enum" + +type ConnectorTestResponse struct { + Status enum.ConnectorStatus `json:"status"` + ErrorMsg string `json:"error_msg,omitempty"` +} diff --git a/types/enum/connector_auth_type.go b/types/enum/connector_auth_type.go new file mode 100644 index 000000000..95be7ecc6 --- /dev/null +++ b/types/enum/connector_auth_type.go @@ -0,0 +1,61 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enum + +import "fmt" + +// ConnectorAuthType represents the type of connector authentication. +type ConnectorAuthType string + +const ( + ConnectorAuthTypeBasic ConnectorAuthType = "basic" + ConnectorAuthTypeBearer ConnectorAuthType = "bearer" +) + +func ParseConnectorAuthType(s string) (ConnectorAuthType, error) { + switch s { + case "basic": + return ConnectorAuthTypeBasic, nil + case "bearer": + return ConnectorAuthTypeBearer, nil + default: + return "", fmt.Errorf("unknown connector auth type provided: %s", s) + } +} + +func (t ConnectorAuthType) String() string { + switch t { + case ConnectorAuthTypeBasic: + return "basic" + case ConnectorAuthTypeBearer: + return "bearer" + default: + return "undefined" + } +} + +func GetAllConnectorAuthTypes() []ConnectorAuthType { + return []ConnectorAuthType{ + ConnectorAuthTypeBasic, + ConnectorAuthTypeBearer, + } +} + +func (ConnectorAuthType) Enum() []interface{} { return toInterfaceSlice(GetAllConnectorAuthTypes()) } +func (t ConnectorAuthType) Sanitize() (ConnectorAuthType, bool) { + return Sanitize(t, func() ([]ConnectorAuthType, ConnectorAuthType) { + return GetAllConnectorAuthTypes(), "" + }) +} diff --git a/types/enum/connector_status.go b/types/enum/connector_status.go new file mode 100644 index 000000000..5ccbb5e77 --- /dev/null +++ b/types/enum/connector_status.go @@ -0,0 +1,49 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enum + +// ConnectorStatus represents the status of the connector after testing connection. +type ConnectorStatus string + +const ( + ConnectorStatusSuccess ConnectorStatus = "success" + + ConnectorStatusFailed ConnectorStatus = "failed" +) + +func (s ConnectorStatus) String() string { + switch s { + case ConnectorStatusSuccess: + return "success" + case ConnectorStatusFailed: + return "failed" + default: + return undefined + } +} + +func GetAllConnectorStatus() ([]ConnectorStatus, ConnectorStatus) { + return connectorStatus, "" // No default value +} + +var connectorStatus = sortEnum([]ConnectorStatus{ + ConnectorStatusSuccess, + ConnectorStatusFailed, +}) + +func (ConnectorStatus) Enum() []interface{} { return toInterfaceSlice(connectorStatus) } +func (s ConnectorStatus) Sanitize() (ConnectorStatus, bool) { + return Sanitize(s, GetAllConnectorStatus) +} diff --git a/types/enum/connector_type.go b/types/enum/connector_type.go new file mode 100644 index 000000000..4e3fc278c --- /dev/null +++ b/types/enum/connector_type.go @@ -0,0 +1,63 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enum + +import "fmt" + +// ConnectorType represents the type of connector. +type ConnectorType string + +const ( + // ConnectorTypeGithub is a github connector. + ConnectorTypeGithub ConnectorType = "github" +) + +func ParseConnectorType(s string) (ConnectorType, error) { + switch s { + case "github": + return ConnectorTypeGithub, nil + default: + return "", fmt.Errorf("unknown connector type provided: %s", s) + } +} + +func (t ConnectorType) String() string { + switch t { + case ConnectorTypeGithub: + return "github" + default: + return undefined + } +} + +func (t ConnectorType) IsSCM() bool { + switch t { + case ConnectorTypeGithub: + return true + default: + return false + } +} + +func GetAllConnectorTypes() ([]ConnectorType, ConnectorType) { + return connectorTypes, "" // No default value +} + +var connectorTypes = sortEnum([]ConnectorType{ + ConnectorTypeGithub, +}) + +func (ConnectorType) Enum() []interface{} { return toInterfaceSlice(connectorTypes) } +func (t ConnectorType) Sanitize() (ConnectorType, bool) { return Sanitize(t, GetAllConnectorTypes) } diff --git a/types/github_connector_data.go b/types/github_connector_data.go new file mode 100644 index 000000000..4b75b6f09 --- /dev/null +++ b/types/github_connector_data.go @@ -0,0 +1,44 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "fmt" + + "github.com/harness/gitness/types/enum" +) + +type GithubConnectorData struct { + APIURL string `json:"api_url"` + Insecure bool `json:"insecure"` + Auth *ConnectorAuth `json:"auth"` +} + +func (g *GithubConnectorData) Validate() error { + if g.Auth == nil { + return fmt.Errorf("auth is required for github connectors") + } + if g.Auth.AuthType != enum.ConnectorAuthTypeBearer { + return fmt.Errorf("only bearer token auth is supported for github connectors") + } + if err := g.Auth.Validate(); err != nil { + return fmt.Errorf("invalid auth credentials: %w", err) + } + return nil +} + +func (g *GithubConnectorData) Type() enum.ConnectorType { + return enum.ConnectorTypeGithub +} diff --git a/types/secret_ref.go b/types/secret_ref.go new file mode 100644 index 000000000..f14649235 --- /dev/null +++ b/types/secret_ref.go @@ -0,0 +1,19 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +type SecretRef struct { + Identifier string `json:"identifier"` +}