USEFULL
/
drone
mirror of https://github.com/harness/drone.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

repo soft delete improvements (#1045)

pull/3484/head
Atefeh Mohseni-Ejiyeh 2024-02-22 05:25:29 +00:00 committed by Harness
parent e7520a983b
commit 24fbf49168
14 changed files with 104 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/api/controller/repo/purge.go
View File

@ -32,7 +32,12 @@ import (
)
// Purge removes a repo permanently.
func (c *Controller) Purge(ctx context.Context, session *auth.Session, repoRef string, deletedAt int64) error {
func (c *Controller) Purge(
ctx context.Context,
session *auth.Session,
repoRef string,
deletedAt int64,
) error {
repo, err := c.repoStore.FindByRefAndDeletedAt(ctx, repoRef, deletedAt)
if err != nil {
return fmt.Errorf("failed to find the repo (deleted at %d): %w", deletedAt, err)

4
app/api/controller/repo/restore.go
View File

@ -27,16 +27,16 @@ import (
type RestoreInput struct {
NewIdentifier string `json:"new_identifier,omitempty"`
DeletedAt int64 `json:"deleted_at"`
}
func (c *Controller) Restore(
ctx context.Context,
session *auth.Session,
repoRef string,
deletedAt int64,
in *RestoreInput,
) (*types.Repository, error) {
repo, err := c.repoStore.FindByRefAndDeletedAt(ctx, repoRef, in.DeletedAt)
repo, err := c.repoStore.FindByRefAndDeletedAt(ctx, repoRef, deletedAt)
if err != nil {
return nil, fmt.Errorf("failed to find repository: %w", err)
}

31
app/api/controller/repo/soft_delete.go
View File

@ -28,16 +28,24 @@ import (
"github.com/rs/zerolog/log"
)
// SoftDelete soft deletes a repo (aka sets the deleted timestamp while keep the data).
func (c *Controller) SoftDelete(ctx context.Context, session *auth.Session, repoRef string) error {
type SoftDeleteResponse struct {
DeletedAt int64 `json:"deleted_at"`
}
// SoftDelete soft deletes a repo and returns the deletedAt timestamp in epoch format.
func (c *Controller) SoftDelete(
ctx context.Context,
session *auth.Session,
repoRef string,
) (*SoftDeleteResponse, error) {
// note: can't use c.getRepoCheckAccess because import job for repositories being imported must be cancelled.
repo, err := c.repoStore.FindByRef(ctx, repoRef)
if err != nil {
return fmt.Errorf("failed to find the repo for soft delete: %w", err)
return nil, fmt.Errorf("failed to find the repo for soft delete: %w", err)
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoDelete, false); err != nil {
return fmt.Errorf("access check failed: %w", err)
return nil, fmt.Errorf("access check failed: %w", err)
}
log.Ctx(ctx).Info().
@ -46,19 +54,24 @@ func (c *Controller) SoftDelete(ctx context.Context, session *auth.Session, repo
Msg("soft deleting repository")
if repo.Deleted != nil {
return usererror.BadRequest("repository has been already deleted")
return nil, usererror.BadRequest("repository has been already deleted")
}
if repo.Importing {
log.Ctx(ctx).Info().Msg("repository is importing. cancelling the import job and purge the repo.")
err = c.importer.Cancel(ctx, repo)
if err != nil {
return fmt.Errorf("failed to cancel repository import")
return nil, fmt.Errorf("failed to cancel repository import")
}
return c.PurgeNoAuth(ctx, session, repo)
return nil, c.PurgeNoAuth(ctx, session, repo)
}
return c.SoftDeleteNoAuth(ctx, repo, time.Now().UnixMilli())
now := time.Now().UnixMilli()
if err = c.SoftDeleteNoAuth(ctx, repo, now); err != nil {
return nil, fmt.Errorf("failed to soft delete repo: %w", err)
}
return &SoftDeleteResponse{DeletedAt: now}, nil
}
func (c *Controller) SoftDeleteNoAuth(
@ -66,7 +79,7 @@ func (c *Controller) SoftDeleteNoAuth(
repo *types.Repository,
deletedAt int64,
) error {
err := c.repoStore.SoftDelete(ctx, repo, &deletedAt)
err := c.repoStore.SoftDelete(ctx, repo, deletedAt)
if err != nil {
return fmt.Errorf("failed to soft delete repo from db: %w", err)
}

14
app/api/controller/space/delete.go
View File

@ -86,12 +86,12 @@ func (c *Controller) deleteRepositoriesNoAuth(
spaceID int64,
) error {
filter := &types.RepoFilter{
Page: 1,
Size: int(math.MaxInt),
Query: "",
Order: enum.OrderAsc,
Sort: enum.RepoAttrNone,
DeletedBefore: nil,
Page: 1,
Size: int(math.MaxInt),
Query: "",
Order: enum.OrderAsc,
Sort: enum.RepoAttrNone,
DeletedBeforeOrAt: nil,
}
repos, _, err := c.ListRepositoriesNoAuth(ctx, spaceID, filter)
@ -101,7 +101,7 @@ func (c *Controller) deleteRepositoriesNoAuth(
// TEMPORARY until we support space delete/restore CODE-1413
recent := time.Now().Add(+time.Hour * 24).UnixMilli()
filter.DeletedBefore = &recent
filter.DeletedBeforeOrAt = &recent
alreadyDeletedRepos, _, err := c.ListRepositoriesNoAuth(ctx, spaceID, filter)
if err != nil {
return fmt.Errorf("failed to list delete repositories for space %d: %w", spaceID, err)

2
app/api/handler/repo/purge.go
View File

@ -34,7 +34,7 @@ func HandlePurge(repoCtrl *repo.Controller) http.HandlerFunc {
return
}
deletedAt, err := request.GetRepoDeletedAtFromQuery(r)
deletedAt, err := request.GetDeletedAtFromQuery(r)
if err != nil {
render.TranslatedUserError(w, err)
return

8
app/api/handler/repo/restore.go
View File

@ -35,6 +35,12 @@ func HandleRestore(repoCtrl *repo.Controller) http.HandlerFunc {
return
}
deletedAt, err := request.GetDeletedAtFromQuery(r)
if err != nil {
render.TranslatedUserError(w, err)
return
}
in := new(repo.RestoreInput)
err = json.NewDecoder(r.Body).Decode(in)
if err != nil {
@ -42,7 +48,7 @@ func HandleRestore(repoCtrl *repo.Controller) http.HandlerFunc {
return
}
repo, err := repoCtrl.Restore(ctx, session, repoRef, in)
repo, err := repoCtrl.Restore(ctx, session, repoRef, deletedAt, in)
if err != nil {
render.TranslatedUserError(w, err)
return

4
app/api/handler/repo/soft_delete.go
View File

@ -37,12 +37,12 @@ func HandleSoftDelete(repoCtrl *repo.Controller) http.HandlerFunc {
return
}
err = repoCtrl.SoftDelete(ctx, session, repoRef)
softDeleteResponse, err := repoCtrl.SoftDelete(ctx, session, repoRef)
if err != nil {
render.TranslatedUserError(w, err)
return
}
render.DeleteSuccessful(w)
render.JSON(w, http.StatusOK, softDeleteResponse)
}
}

11
app/api/openapi/repo.go
View File

@ -458,11 +458,11 @@ var queryParameterBypassRules = openapi3.ParameterOrRef{
},
}
var queryParameterRepoDeletedAt = openapi3.ParameterOrRef{
var queryParameterDeletedAt = openapi3.ParameterOrRef{
Parameter: &openapi3.Parameter{
Name: request.QueryParamRepoDeletedAt,
Name: request.QueryParamDeletedAt,
In: openapi3.ParameterInQuery,
Description: ptr.String("The time repository was deleted at in epoch format."),
Description: ptr.String("The exact time the resource was delete at in epoch format."),
Required: ptr.Bool(true),
Schema: &openapi3.SchemaOrRef{
Schema: &openapi3.Schema{
@ -525,7 +525,7 @@ func repoOperations(reflector *openapi3.Reflector) {
opDelete.WithTags("repository")
opDelete.WithMapOfAnything(map[string]interface{}{"operationId": "deleteRepository"})
_ = reflector.SetRequest(&opDelete, new(repoRequest), http.MethodDelete)
_ = reflector.SetJSONResponse(&opDelete, nil, http.StatusNoContent)
_ = reflector.SetJSONResponse(&opDelete, new(repo.SoftDeleteResponse), http.StatusOK)
_ = reflector.SetJSONResponse(&opDelete, new(usererror.Error), http.StatusInternalServerError)
_ = reflector.SetJSONResponse(&opDelete, new(usererror.Error), http.StatusUnauthorized)
_ = reflector.SetJSONResponse(&opDelete, new(usererror.Error), http.StatusForbidden)
@ -535,7 +535,7 @@ func repoOperations(reflector *openapi3.Reflector) {
opPurge := openapi3.Operation{}
opPurge.WithTags("repository")
opPurge.WithMapOfAnything(map[string]interface{}{"operationId": "purgeRepository"})
opPurge.WithParameters(queryParameterRepoDeletedAt)
opPurge.WithParameters(queryParameterDeletedAt)
_ = reflector.SetRequest(&opPurge, new(repoRequest), http.MethodPost)
_ = reflector.SetJSONResponse(&opPurge, nil, http.StatusNoContent)
_ = reflector.SetJSONResponse(&opPurge, new(usererror.Error), http.StatusInternalServerError)
@ -547,6 +547,7 @@ func repoOperations(reflector *openapi3.Reflector) {
opRestore := openapi3.Operation{}
opRestore.WithTags("repository")
opRestore.WithMapOfAnything(map[string]interface{}{"operationId": "restoreRepository"})
opRestore.WithParameters(queryParameterDeletedAt)
_ = reflector.SetRequest(&opRestore, new(restoreRequest), http.MethodPost)
_ = reflector.SetJSONResponse(&opRestore, new(types.Repository), http.StatusOK)
_ = reflector.SetJSONResponse(&opRestore, new(usererror.Error), http.StatusBadRequest)

7
app/api/request/common.go
View File

@ -37,6 +37,8 @@ const (
QueryParamAfter = "after"
QueryParamBefore = "before"
QueryParamDeletedAt = "deleted_at"
QueryParamPage = "page"
QueryParamLimit = "limit"
PerPageDefault = 30
@ -118,3 +120,8 @@ func ParseListQueryFilterFromRequest(r *http.Request) types.ListQueryFilter {
func GetContentEncodingFromHeadersOrDefault(r *http.Request, dflt string) string {
return GetHeaderOrDefault(r, HeaderContentEncoding, dflt)
}
// GetDeletedAtFromQuery extracts the resource deleted timestamp from the query.
func GetDeletedAtFromQuery(r *http.Request) (int64, error) {
return QueryParamAsPositiveInt64(r, QueryParamDeletedAt)
}

12
app/api/request/repo.go
View File

@ -23,10 +23,9 @@ import (
)
const (
PathParamRepoRef = "repo_ref"
QueryParamRepoID = "repo_id"
QueryParamRepoDeletedAt = "repo_deleted_at"
QueryParamRecursive = "recursive"
PathParamRepoRef = "repo_ref"
QueryParamRepoID = "repo_id"
QueryParamRecursive = "recursive"
)
func GetRepoRefFromPath(r *http.Request) (string, error) {
@ -57,11 +56,6 @@ func ParseRepoFilter(r *http.Request) *types.RepoFilter {
}
}
// GetRepoDeletedAtFromQuery extracts the repository deleted timestamp from the query.
func GetRepoDeletedAtFromQuery(r *http.Request) (int64, error) {
return QueryParamAsPositiveInt64(r, QueryParamRepoDeletedAt)
}
// ParseRecursiveFromQuery extracts the recursive option from the URL query.
func ParseRecursiveFromQuery(r *http.Request) (bool, error) {
return QueryParamAsBoolOrDefault(r, QueryParamRecursive, false)

14
app/services/cleanup/deleted_repos.go
View File

@ -65,14 +65,14 @@ func (j *deletedReposCleanupJob) Handle(ctx context.Context, _ string, _ job.Pro
j.retentionTime,
olderThan.Format(time.RFC3339Nano))
deletedBefore := olderThan.UnixMilli()
deletedBeforeOrAt := olderThan.UnixMilli()
filter := &types.RepoFilter{
Page: 1,
Size: int(math.MaxInt),
Query: "",
Order: enum.OrderAsc,
Sort: enum.RepoAttrDeleted,
DeletedBefore: &deletedBefore,
Page: 1,
Size: int(math.MaxInt),
Query: "",
Order: enum.OrderAsc,
Sort: enum.RepoAttrDeleted,
DeletedBeforeOrAt: &deletedBeforeOrAt,
}
toBePurgedRepos, err := j.repoStore.List(ctx, 0, filter)
if err != nil {

6
app/store/database.go
View File

@ -207,7 +207,7 @@ type (
mutateFn func(repository *types.Repository) error) (*types.Repository, error)
// SoftDelete a repo.
SoftDelete(ctx context.Context, repo *types.Repository, deletedAt *int64) error
SoftDelete(ctx context.Context, repo *types.Repository, deletedAt int64) error
// Purge the soft deleted repo permanently.
Purge(ctx context.Context, id int64, deletedAt *int64) error
@ -216,10 +216,10 @@ type (
Restore(ctx context.Context, repo *types.Repository,
newIdentifier string) (*types.Repository, error)
// Count of active repos in a space. With "DeletedBefore" filter, counts only deleted repos by opts.DeletedBefore.
// Count of active repos in a space. With "DeletedBeforeOrAt" filter, counts deleted repos.
Count(ctx context.Context, parentID int64, opts *types.RepoFilter) (int64, error)
// List returns a list of repos in a space.
// List returns a list of repos in a space. With "DeletedBeforeOrAt" filter, lists deleted repos.
List(ctx context.Context, parentID int64, opts *types.RepoFilter) ([]*types.Repository, error)
// ListSizeInfos returns a list of all active repo sizes.

36
app/store/database/repo.go
View File

@ -450,9 +450,9 @@ func (s *RepoStore) updateOptLock(
}
// SoftDelete deletes a repo softly by setting the deleted timestamp.
func (s *RepoStore) SoftDelete(ctx context.Context, repo *types.Repository, deletedAt *int64) error {
func (s *RepoStore) SoftDelete(ctx context.Context, repo *types.Repository, deletedAt int64) error {
_, err := s.UpdateOptLock(ctx, repo, func(r *types.Repository) error {
r.Deleted = deletedAt
r.Deleted = &deletedAt
return nil
})
if err != nil {
@ -463,13 +463,25 @@ func (s *RepoStore) SoftDelete(ctx context.Context, repo *types.Repository, dele
// Purge deletes the repo permanently.
func (s *RepoStore) Purge(ctx context.Context, id int64, deletedAt *int64) error {
const repoDelete = `
DELETE FROM repositories
WHERE repo_id = $1 AND repo_deleted = $2`
stmt := database.Builder.
Delete("repositories").
Where("repo_id = ?", id)
if deletedAt != nil {
stmt = stmt.Where("repo_deleted = ?", *deletedAt)
} else {
stmt = stmt.Where("repo_deleted IS NULL")
}
sql, args, err := stmt.ToSql()
if err != nil {
return fmt.Errorf("failed to convert purge repo query to sql: %w", err)
}
db := dbtx.GetAccessor(ctx, s.db)
if _, err := db.ExecContext(ctx, repoDelete, id, deletedAt); err != nil {
_, err = db.ExecContext(ctx, sql, args...)
if err != nil {
return database.ProcessSQLErrorf(err, "the delete query failed")
}
@ -490,13 +502,13 @@ func (s *RepoStore) Restore(
return nil
})
if err != nil {
return nil, database.ProcessSQLErrorf(err, "failed to restore the repo")
return nil, err
}
return repo, nil
}
// Count of active repos in a space. if parentID (space) is zero then it will count all repositories in the system.
// With "DeletedBefore" filter, counts only deleted repos by opts.DeletedBefore.
// Count deleted repos requires opts.DeletedBeforeOrAt filter.
func (s *RepoStore) Count(
ctx context.Context,
parentID int64,
@ -585,7 +597,7 @@ FROM SpaceHierarchy h1;`
}
// List returns a list of active repos in a space.
// With "DeletedBefore" filter, shows deleted repos by opts.DeletedBefore.
// With "DeletedBeforeOrAt" filter, lists deleted repos by opts.DeletedBeforeOrAt.
func (s *RepoStore) List(
ctx context.Context,
parentID int64,
@ -813,8 +825,8 @@ func applyQueryFilter(stmt squirrel.SelectBuilder, filter *types.RepoFilter) squ
if filter.Query != "" {
stmt = stmt.Where("LOWER(repo_uid) LIKE ?", fmt.Sprintf("%%%s%%", strings.ToLower(filter.Query)))
}
if filter.DeletedBefore != nil {
stmt = stmt.Where("repo_deleted < ?", filter.DeletedBefore)
if filter.DeletedBeforeOrAt != nil {
stmt = stmt.Where("repo_deleted <= ?", filter.DeletedBeforeOrAt)
} else {
stmt = stmt.Where("repo_deleted IS NULL")
}
@ -836,8 +848,8 @@ func applySortFilter(stmt squirrel.SelectBuilder, filter *types.RepoFilter) squi
stmt = stmt.OrderBy("repo_created " + filter.Order.String())
case enum.RepoAttrUpdated:
stmt = stmt.OrderBy("repo_updated " + filter.Order.String())
case enum.RepoAttrDeleted:
stmt = stmt.OrderBy("repo_deleted " + filter.Order.String())
}
return stmt

14
types/repo.go
View File

@ -81,13 +81,13 @@ func (r Repository) GetGitUID() string {
// RepoFilter stores repo query parameters.
type RepoFilter struct {
Page int `json:"page"`
Size int `json:"size"`
Query string `json:"query"`
Sort enum.RepoAttr `json:"sort"`
Order enum.Order `json:"order"`
DeletedBefore *int64 `json:"deleted_before,omitempty"`
Recursive bool
Page int `json:"page"`
Size int `json:"size"`
Query string `json:"query"`
Sort enum.RepoAttr `json:"sort"`
Order enum.Order `json:"order"`
DeletedBeforeOrAt *int64 `json:"deleted_before_or_at,omitempty"`
Recursive bool
}
// RepositoryGitInfo holds git info for a repository.