introduce git object cache for gogit

cmd/gitness/wire_gen.go

@@ -232,9 +232,9 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
 	if err != nil {
 		return nil, err
 	}
-	cacheCache := server3.ProvideGoGitRepoCache()
-	cache2 := server3.ProvideLastCommitCache(serverConfig, universalClient, cacheCache)
-	gitAdapter, err := server3.ProvideGITAdapter(cacheCache, cache2)
+	goGitRepoProvider := server3.ProvideGoGitRepoProvider()
+	cacheCache := server3.ProvideLastCommitCache(serverConfig, universalClient, goGitRepoProvider)
+	gitAdapter, err := server3.ProvideGITAdapter(goGitRepoProvider, cacheCache)
 	if err != nil {
 		return nil, err
 	}

gitrpc/internal/gitea/gitea.go

@@ -15,12 +15,12 @@ import (
 )
 
 type Adapter struct {
-	repoCache       cache.Cache[string, *RepoEntryValue]
+	repoProvider    *GoGitRepoProvider
 	lastCommitCache cache.Cache[CommitEntryKey, *types.Commit]
 }
 
 func New(
-	repoCache cache.Cache[string, *RepoEntryValue],
+	repoProvider *GoGitRepoProvider,
 	lastCommitCache cache.Cache[CommitEntryKey, *types.Commit],
 ) (Adapter, error) {
 	// TODO: should be subdir of gitRoot? What is it being used for?
@@ -32,7 +32,7 @@ func New(
 	}
 
 	return Adapter{
-		repoCache:       repoCache,
+		repoProvider:    repoProvider,
 		lastCommitCache: lastCommitCache,
 	}, nil
 }

gitrpc/internal/gitea/gogit.go

@@ -8,25 +8,79 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
+	"time"
 
+	"github.com/harness/gitness/cache"
 	"github.com/harness/gitness/gitrpc/internal/types"
 
+	gogitosfs "github.com/go-git/go-billy/v5/osfs"
 	gogit "github.com/go-git/go-git/v5"
 	gogitplumbing "github.com/go-git/go-git/v5/plumbing"
+	gogitcache "github.com/go-git/go-git/v5/plumbing/cache"
 	gogitobject "github.com/go-git/go-git/v5/plumbing/object"
+	gogitfilesystem "github.com/go-git/go-git/v5/storage/filesystem"
 )
 
+type GoGitRepoProvider struct {
+	gitObjectCache cache.Cache[string, *gogitcache.ObjectLRU]
+}
+
+func NewGoGitRepoProvider(objectCacheMax int, cacheDuration time.Duration) *GoGitRepoProvider {
+	c := cache.New[string, *gogitcache.ObjectLRU](gitObjectCacheGetter{
+		maxSize: objectCacheMax,
+	}, cacheDuration)
+	return &GoGitRepoProvider{
+		gitObjectCache: c,
+	}
+}
+
+func (gr *GoGitRepoProvider) Get(ctx context.Context, path string) (*gogit.Repository, error) {
+	fs := gogitosfs.New(path)
+	stat, err := fs.Stat("")
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, types.ErrRepositoryNotFound
+		}
+
+		return nil, fmt.Errorf("failed to check repository existence: %w", err)
+	}
+	if !stat.IsDir() {
+		return nil, types.ErrRepositoryCorrupted
+	}
+
+	gitObjectCache, err := gr.gitObjectCache.Get(ctx, path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get repository cache: %w", err)
+	}
+
+	s := gogitfilesystem.NewStorage(fs, gitObjectCache)
+
+	repo, err := gogit.Open(s, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return repo, nil
+}
+
+type gitObjectCacheGetter struct {
+	maxSize int
+}
+
+func (r gitObjectCacheGetter) Find(_ context.Context, _ string) (*gogitcache.ObjectLRU, error) {
+	return gogitcache.NewObjectLRU(gogitcache.FileSize(r.maxSize)), nil
+}
+
 func (g Adapter) getGoGitCommit(ctx context.Context,
 	repoPath string,
 	rev string,
 ) (*gogit.Repository, *gogitobject.Commit, error) {
-	repoEntry, err := g.repoCache.Get(ctx, repoPath)
+	repo, err := g.repoProvider.Get(ctx, repoPath)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to open repository: %w", err)
 	}
 
-	repo := repoEntry.Repo()
-
 	var refSHA *gogitplumbing.Hash
 	if rev == "" {
 		var head *gogitplumbing.Reference

gitrpc/internal/gitea/last_commit_cache.go

@@ -23,11 +23,11 @@ import (
 
 func NewInMemoryLastCommitCache(
 	cacheDuration time.Duration,
-	repoCache cache.Cache[string, *RepoEntryValue],
+	repoProvider *GoGitRepoProvider,
 ) cache.Cache[CommitEntryKey, *types.Commit] {
 	return cache.New[CommitEntryKey, *types.Commit](
 		commitEntryGetter{
-			repoCache: repoCache,
+			repoProvider: repoProvider,
 		},
 		cacheDuration)
 }
@@ -35,12 +35,12 @@ func NewInMemoryLastCommitCache(
 func NewRedisLastCommitCache(
 	redisClient redis.UniversalClient,
 	cacheDuration time.Duration,
-	repoCache cache.Cache[string, *RepoEntryValue],
+	repoProvider *GoGitRepoProvider,
 ) cache.Cache[CommitEntryKey, *types.Commit] {
 	return cache.NewRedis[CommitEntryKey, *types.Commit](
 		redisClient,
 		commitEntryGetter{
-			repoCache: repoCache,
+			repoProvider: repoProvider,
 		},
 		func(key CommitEntryKey) string {
 			h := sha256.New()
@@ -52,9 +52,9 @@ func NewRedisLastCommitCache(
 }
 
 func NoLastCommitCache(
-	repoCache cache.Cache[string, *RepoEntryValue],
+	repoProvider *GoGitRepoProvider,
 ) cache.Cache[CommitEntryKey, *types.Commit] {
-	return cache.NewNoCache[CommitEntryKey, *types.Commit](commitEntryGetter{repoCache: repoCache})
+	return cache.NewNoCache[CommitEntryKey, *types.Commit](commitEntryGetter{repoProvider: repoProvider})
 }
 
 type CommitEntryKey string
@@ -96,7 +96,7 @@ func (c commitValueCodec) Decode(s string) (*types.Commit, error) {
 }
 
 type commitEntryGetter struct {
-	repoCache cache.Cache[string, *RepoEntryValue]
+	repoProvider *GoGitRepoProvider
 }
 
 // Find implements the cache.Getter interface.
@@ -119,12 +119,12 @@ func (c commitEntryGetter) Find(ctx context.Context, key CommitEntryKey) (*types
 		return nil, types.ErrNotFound
 	}
 
-	repo, err := c.repoCache.Get(ctx, repoPath)
+	repo, err := c.repoProvider.Get(ctx, repoPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get repository %s from cache: %w", repoPath, err)
 	}
 
-	commit, err := repo.Repo().CommitObject(gogitplumbing.NewHash(commitSHA))
+	commit, err := repo.CommitObject(gogitplumbing.NewHash(commitSHA))
 	if err != nil {
 		return nil, fmt.Errorf("failed to load commit data: %w", err)
 	}

gitrpc/internal/gitea/repo_cache.go

@@ -1,34 +0,0 @@
-// Copyright 2022 Harness Inc. All rights reserved.
-// Use of this source code is governed by the Polyform Free Trial License
-// that can be found in the LICENSE.md file for this repository.
-
-package gitea
-
-import (
-	"context"
-
-	"github.com/harness/gitness/cache"
-
-	gogit "github.com/go-git/go-git/v5"
-)
-
-func NewRepoCache() cache.Cache[string, *RepoEntryValue] {
-	return cache.NewNoCache[string, *RepoEntryValue](repoGetter{})
-}
-
-type repoGetter struct{}
-
-type RepoEntryValue gogit.Repository
-
-func (repo *RepoEntryValue) Repo() *gogit.Repository {
-	return (*gogit.Repository)(repo)
-}
-
-func (r repoGetter) Find(_ context.Context, path string) (*RepoEntryValue, error) {
-	repo, err := gogit.PlainOpen(path)
-	if err != nil {
-		return nil, err
-	}
-
-	return (*RepoEntryValue)(repo), nil
-}

gitrpc/internal/types/errors.go

@@ -14,6 +14,8 @@ import (
 var (
 	ErrAlreadyExists               = errors.New("already exists")
 	ErrInvalidArgument             = errors.New("invalid argument")
+	ErrRepositoryNotFound          = errors.New("repository not found")
+	ErrRepositoryCorrupted         = errors.New("repository corrupted")
 	ErrNotFound                    = errors.New("not found")
 	ErrInvalidPath                 = errors.New("path is invalid")
 	ErrUndefinedAction             = errors.New("undefined action")

gitrpc/server/wire.go

@@ -21,37 +21,38 @@ var WireSet = wire.NewSet(
 	ProvideServer,
 	ProvideHTTPServer,
 	ProvideGITAdapter,
-	ProvideGoGitRepoCache,
+	ProvideGoGitRepoProvider,
 	ProvideLastCommitCache,
 )
 
-func ProvideGoGitRepoCache() cache.Cache[string, *gitea.RepoEntryValue] {
-	return gitea.NewRepoCache()
+func ProvideGoGitRepoProvider() *gitea.GoGitRepoProvider {
+	const objectCacheSize = 16 << 20 // 16MiB
+	return gitea.NewGoGitRepoProvider(objectCacheSize, 15*time.Minute)
 }
 
 func ProvideLastCommitCache(
 	config Config,
 	redisClient redis.UniversalClient,
-	repoCache cache.Cache[string, *gitea.RepoEntryValue],
+	repoProvider *gitea.GoGitRepoProvider,
 ) cache.Cache[gitea.CommitEntryKey, *types.Commit] {
 	cacheDuration := time.Duration(config.LastCommitCache.DurationSeconds) * time.Second
 
 	if config.LastCommitCache.Mode == ModeNone || cacheDuration < time.Second {
-		return gitea.NoLastCommitCache(repoCache)
+		return gitea.NoLastCommitCache(repoProvider)
 	}
 
 	if config.LastCommitCache.Mode == ModeRedis && redisClient != nil {
-		return gitea.NewRedisLastCommitCache(redisClient, cacheDuration, repoCache)
+		return gitea.NewRedisLastCommitCache(redisClient, cacheDuration, repoProvider)
 	}
 
-	return gitea.NewInMemoryLastCommitCache(cacheDuration, repoCache)
+	return gitea.NewInMemoryLastCommitCache(cacheDuration, repoProvider)
 }
 
 func ProvideGITAdapter(
-	repoCache cache.Cache[string, *gitea.RepoEntryValue],
+	repoProvider *gitea.GoGitRepoProvider,
 	lastCommitCache cache.Cache[gitea.CommitEntryKey, *types.Commit],
 ) (service.GitAdapter, error) {
-	return gitea.New(repoCache, lastCommitCache)
+	return gitea.New(repoProvider, lastCommitCache)
 }
 
 func ProvideServer(config Config, adapter service.GitAdapter) (*GRPCServer, error) {