sharpetronics/gitea
1
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2025-05-13 01:12:36 -04:00
Code Issues Projects Releases Wiki Activity

Merge 324ed3263c2d6a20cc278b6380bf5dab33ecaa47 into 355e9a9d544aa2d3f3a17b06cdb2bf1ceb290fd7

Browse Source
This commit is contained in:
Lunny Xiao 2025-05-11 23:20:11 -04:00 committed by GitHub
commit 979dc8342d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 115 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
custom/conf/app.example.ini
View File

@ -1886,9 +1886,6 @@ LEVEL = Info
;; Time to keep items in cache if not used, default is 8760 hours.
;; Setting it to -1 disables caching
;ITEM_TTL = 8760h
;;
;; Only enable the cache when repository's commits count great than
;COMMITS_COUNT = 1000
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

5
modules/git/commit.go
View File

@ -78,10 +78,7 @@ func (c *Commit) ParentCount() int {
// GetCommitByPath return the commit of relative path object.
func (c *Commit) GetCommitByPath(relpath string) (*Commit, error) {
if c.repo.LastCommitCache != nil {
return c.repo.LastCommitCache.GetCommitByPath(c.ID.String(), relpath)
}
return c.repo.getCommitByPathWithID(c.ID, relpath)
return c.repo.lastCommitCache.GetCommitByPath(c.ID.String(), relpath)
}
// AddChanges marks local changes to be ready for commit.

46
modules/git/commit_info_gogit.go
View File

@ -34,28 +34,19 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
return nil, nil, err
}
var revs map[string]*Commit
if commit.repo.LastCommitCache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, commit.repo.LastCommitCache)
revs, unHitPaths, err := commit.repo.lastCommitCache.getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
revs2, err := GetLastCommitForPaths(ctx, commit.repo.lastCommitCache, c, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
revs2, err := GetLastCommitForPaths(ctx, commit.repo.LastCommitCache, c, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
for k, v := range revs2 {
revs[k] = v
}
for k, v := range revs2 {
revs[k] = v
}
} else {
revs, err = GetLastCommitForPaths(ctx, nil, c, treePath, entryPaths)
}
if err != nil {
return nil, nil, err
}
commit.repo.gogitStorage.Close()
@ -153,27 +144,8 @@ func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[
return hashes, nil
}
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) {
var unHitEntryPaths []string
results := make(map[string]*Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
// GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(ctx context.Context, cache *LastCommitCache, c cgobject.CommitNode, treePath string, paths []string) (map[string]*Commit, error) {
func GetLastCommitForPaths(ctx context.Context, cache *lastCommitCache, c cgobject.CommitNode, treePath string, paths []string) (map[string]*Commit, error) {
refSha := c.ID().String()
// We do a tree traversal with nodes sorted by commit time

49
modules/git/commit_info_nogogit.go
View File

@ -9,7 +9,6 @@ import (
"context"
"fmt"
"io"
"path"
"sort"
"code.gitea.io/gitea/modules/log"
@ -27,29 +26,22 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
var err error
var revs map[string]*Commit
if commit.repo.LastCommitCache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, commit.repo.LastCommitCache)
var unHitPaths []string
revs, unHitPaths, err = commit.repo.lastCommitCache.getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
sort.Strings(unHitPaths)
commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
sort.Strings(unHitPaths)
commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
for pth, found := range commits {
revs[pth] = found
}
for pth, found := range commits {
revs[pth] = found
}
} else {
sort.Strings(entryPaths)
revs, err = GetLastCommitForPaths(ctx, commit, treePath, entryPaths)
}
if err != nil {
return nil, nil, err
}
commitsInfo := make([]CommitInfo, len(tes))
@ -97,25 +89,6 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath
return commitsInfo, treeCommit, nil
}
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) {
var unHitEntryPaths []string
results := make(map[string]*Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
// GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(ctx context.Context, commit *Commit, treePath string, paths []string) (map[string]*Commit, error) {
// We read backwards from the commit to obtain all of the commits

72
modules/git/last_commit_cache.go
View File

@ -4,8 +4,10 @@
package git
import (
"context"
"crypto/sha256"
"fmt"
"path"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/log"
@ -17,25 +19,21 @@ func getCacheKey(repoPath, commitID, entryPath string) string {
return fmt.Sprintf("last_commit:%x", hashBytes)
}
// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl func() int64
repo *Repository
commitCache map[string]*Commit
cache cache.StringCache
// lastCommitCache represents a cache to store last commit
type lastCommitCache struct {
repoPath string
repo *Repository
ttl func() int64
cache cache.StringCache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(count int64, repoPath string, gitRepo *Repository, cache cache.StringCache) *LastCommitCache {
// newLastCommitCache creates a new last commit cache for repo
func newLastCommitCache(repoPath string, gitRepo *Repository, cache cache.StringCache) *lastCommitCache {
if cache == nil {
return nil
}
if count < setting.CacheService.LastCommit.CommitsCount {
return nil
}
return &LastCommitCache{
return &lastCommitCache{
repoPath: repoPath,
repo: gitRepo,
ttl: setting.LastCommitCacheTTLSeconds,
@ -44,7 +42,7 @@ func NewLastCommitCache(count int64, repoPath string, gitRepo *Repository, cache
}
// Put put the last commit id with commit and entry path
func (c *LastCommitCache) Put(ref, entryPath, commitID string) error {
func (c *lastCommitCache) Put(ref, entryPath, commitID string) error {
if c == nil || c.cache == nil {
return nil
}
@ -53,7 +51,7 @@ func (c *LastCommitCache) Put(ref, entryPath, commitID string) error {
}
// Get gets the last commit information by commit id and entry path
func (c *LastCommitCache) Get(ref, entryPath string) (*Commit, error) {
func (c *lastCommitCache) Get(ref, entryPath string) (*Commit, error) {
if c == nil || c.cache == nil {
return nil, nil
}
@ -63,27 +61,12 @@ func (c *LastCommitCache) Get(ref, entryPath string) (*Commit, error) {
return nil, nil
}
log.Debug("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, commitID)
if c.commitCache != nil {
if commit, ok := c.commitCache[commitID]; ok {
log.Debug("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, commitID)
return commit, nil
}
}
commit, err := c.repo.GetCommit(commitID)
if err != nil {
return nil, err
}
if c.commitCache == nil {
c.commitCache = make(map[string]*Commit)
}
c.commitCache[commitID] = commit
return commit, nil
log.Debug("LastCommitCache hit: [%s:%s:%s]", ref, entryPath, commitID)
return c.repo.GetCommit(commitID)
}
// GetCommitByPath gets the last commit for the entry in the provided commit
func (c *LastCommitCache) GetCommitByPath(commitID, entryPath string) (*Commit, error) {
func (c *lastCommitCache) GetCommitByPath(commitID, entryPath string) (*Commit, error) {
sha, err := NewIDFromString(commitID)
if err != nil {
return nil, err
@ -105,3 +88,26 @@ func (c *LastCommitCache) GetCommitByPath(commitID, entryPath string) (*Commit,
return lastCommit, nil
}
func (c *lastCommitCache) getLastCommitForPathsByCache(commitID, treePath string, paths []string) (map[string]*Commit, []string, error) {
var unHitEntryPaths []string
results := make(map[string]*Commit)
for _, p := range paths {
lastCommit, err := c.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
func (repo *Repository) CacheCommit(ctx context.Context, commit *Commit) error {
return repo.lastCommitCache.CacheCommit(ctx, commit)
}

15
modules/git/last_commit_cache_gogit.go
View File

@ -13,21 +13,18 @@ import (
)
// CacheCommit will cache the commit from the gitRepository
func (c *Commit) CacheCommit(ctx context.Context) error {
if c.repo.LastCommitCache == nil {
return nil
}
func (c *lastCommitCache) CacheCommit(ctx context.Context, commit *Commit) error {
commitNodeIndex, _ := c.repo.CommitNodeIndex()
index, err := commitNodeIndex.Get(plumbing.Hash(c.ID.RawValue()))
index, err := commitNodeIndex.Get(plumbing.Hash(commit.ID.RawValue()))
if err != nil {
return err
}
return c.recursiveCache(ctx, index, &c.Tree, "", 1)
return c.recursiveCache(ctx, index, commit, &commit.Tree, "", 1)
}
func (c *Commit) recursiveCache(ctx context.Context, index cgobject.CommitNode, tree *Tree, treePath string, level int) error {
func (c *lastCommitCache) recursiveCache(ctx context.Context, index cgobject.CommitNode, commit *Commit, tree *Tree, treePath string, level int) error {
if level == 0 {
return nil
}
@ -44,7 +41,7 @@ func (c *Commit) recursiveCache(ctx context.Context, index cgobject.CommitNode,
entryMap[entry.Name()] = entry
}
commits, err := GetLastCommitForPaths(ctx, c.repo.LastCommitCache, index, treePath, entryPaths)
commits, err := GetLastCommitForPaths(ctx, c, index, treePath, entryPaths)
if err != nil {
return err
}
@ -55,7 +52,7 @@ func (c *Commit) recursiveCache(ctx context.Context, index cgobject.CommitNode,
if err != nil {
return err
}
if err := c.recursiveCache(ctx, index, subTree, entry, level-1); err != nil {
if err := c.recursiveCache(ctx, index, commit, subTree, entry, level-1); err != nil {
return err
}
}

13
modules/git/last_commit_cache_nogogit.go
View File

@ -10,14 +10,11 @@ import (
)
// CacheCommit will cache the commit from the gitRepository
func (c *Commit) CacheCommit(ctx context.Context) error {
if c.repo.LastCommitCache == nil {
return nil
}
return c.recursiveCache(ctx, &c.Tree, "", 1)
func (c *lastCommitCache) CacheCommit(ctx context.Context, commit *Commit) error {
return c.recursiveCache(ctx, commit, &commit.Tree, "", 1)
}
func (c *Commit) recursiveCache(ctx context.Context, tree *Tree, treePath string, level int) error {
func (c *lastCommitCache) recursiveCache(ctx context.Context, commit *Commit, tree *Tree, treePath string, level int) error {
if level == 0 {
return nil
}
@ -32,7 +29,7 @@ func (c *Commit) recursiveCache(ctx context.Context, tree *Tree, treePath string
entryPaths[i] = entry.Name()
}
_, err = WalkGitLog(ctx, c.repo, c, treePath, entryPaths...)
_, err = WalkGitLog(ctx, c.repo, commit, treePath, entryPaths...)
if err != nil {
return err
}
@ -44,7 +41,7 @@ func (c *Commit) recursiveCache(ctx context.Context, tree *Tree, treePath string
if err != nil {
return err
}
if err := c.recursiveCache(ctx, subTree, treeEntry.Name(), level-1); err != nil {
if err := c.recursiveCache(ctx, commit, subTree, treeEntry.Name(), level-1); err != nil {
return err
}
}

4
modules/git/log_name_status.go
View File

@ -387,14 +387,14 @@ heaploop:
changed[i] = false
if results[i] == "" {
results[i] = current.CommitID
if err := repo.LastCommitCache.Put(headRef, path.Join(treepath, paths[i]), current.CommitID); err != nil {
if err := repo.lastCommitCache.Put(headRef, path.Join(treepath, paths[i]), current.CommitID); err != nil {
return nil, err
}
delete(path2idx, paths[i])
remaining--
if results[0] == "" {
results[0] = current.CommitID
if err := repo.LastCommitCache.Put(headRef, treepath, current.CommitID); err != nil {
if err := repo.lastCommitCache.Put(headRef, treepath, current.CommitID); err != nil {
return nil, err
}
delete(path2idx, "")

17
modules/git/repo_base_gogit.go
View File

@ -10,6 +10,7 @@ import (
"context"
"path/filepath"
giteacache "code.gitea.io/gitea/modules/cache"
gitealog "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@ -28,14 +29,15 @@ const isGogit = true
type Repository struct {
Path string
tagCache *ObjectCache[*Tag]
tagCache *ObjectCache[*Tag]
commitCache map[string]*Commit
gogitRepo *gogit.Repository
gogitStorage *filesystem.Storage
gpgSettings *GPGSettings
Ctx context.Context
LastCommitCache *LastCommitCache
lastCommitCache *lastCommitCache
objectFormat ObjectFormat
}
@ -80,14 +82,18 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) {
return nil, err
}
return &Repository{
repo := &Repository{
Path: repoPath,
gogitRepo: gogitRepo,
gogitStorage: storage,
tagCache: newObjectCache[*Tag](),
commitCache: make(map[string]*Commit),
Ctx: ctx,
objectFormat: ParseGogitHash(plumbing.ZeroHash).Type(),
}, nil
}
repo.lastCommitCache = newLastCommitCache(repo.Path, repo, giteacache.GetCache())
return repo, nil
}
// Close this repository, in particular close the underlying gogitStorage if this is not nil
@ -99,8 +105,9 @@ func (repo *Repository) Close() error {
gitealog.Error("Error closing storage: %v", err)
}
repo.gogitStorage = nil
repo.LastCommitCache = nil
repo.lastCommitCache = nil
repo.tagCache = nil
repo.commitCache = nil
return nil
}

22
modules/git/repo_base_nogogit.go
View File

@ -11,6 +11,7 @@ import (
"context"
"path/filepath"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
)
@ -21,7 +22,8 @@ const isGogit = false
type Repository struct {
Path string
tagCache *ObjectCache[*Tag]
tagCache *ObjectCache[*Tag]
commitCache map[string]*Commit
gpgSettings *GPGSettings
@ -32,7 +34,7 @@ type Repository struct {
check *Batch
Ctx context.Context
LastCommitCache *LastCommitCache
lastCommitCache *lastCommitCache
objectFormat ObjectFormat
}
@ -56,11 +58,14 @@ func OpenRepository(ctx context.Context, repoPath string) (*Repository, error) {
return nil, util.NewNotExistErrorf("no such file or directory")
}
return &Repository{
Path: repoPath,
tagCache: newObjectCache[*Tag](),
Ctx: ctx,
}, nil
repo := &Repository{
Path: repoPath,
tagCache: newObjectCache[*Tag](),
commitCache: make(map[string]*Commit),
Ctx: ctx,
}
repo.lastCommitCache = newLastCommitCache(repoPath, repo, cache.GetCache())
return repo, nil
}
// CatFileBatch obtains a CatFileBatch for this repository
@ -127,7 +132,8 @@ func (repo *Repository) Close() error {
repo.check = nil
repo.checkInUse = false
}
repo.LastCommitCache = nil
repo.lastCommitCache = nil
repo.tagCache = nil
repo.commitCache = nil
return nil
}

31
modules/git/repo_commit.go
View File

@ -11,7 +11,7 @@ import (
"strconv"
"strings"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
@ -27,12 +27,22 @@ func (repo *Repository) GetTagCommitID(name string) (string, error) {
// GetCommit returns commit object of by ID string.
func (repo *Repository) GetCommit(commitID string) (*Commit, error) {
if commit, ok := repo.commitCache[commitID]; ok {
log.Debug("repo commitCache hit: [%s:%s:%s]", repo.Path, commitID)
return commit, nil
}
id, err := repo.ConvertToGitID(commitID)
if err != nil {
return nil, err
}
return repo.getCommit(id)
commit, err := repo.getCommit(id)
if err != nil {
return nil, err
}
repo.commitCache[commitID] = commit
return commit, nil
}
// GetBranchCommit returns the last commit of given branch.
@ -502,23 +512,6 @@ func (repo *Repository) IsCommitInBranch(commitID, branch string) (r bool, err e
return len(stdout) > 0, err
}
func (repo *Repository) AddLastCommitCache(cacheKey, fullName, sha string) error {
if repo.LastCommitCache == nil {
commitsCount, err := cache.GetInt64(cacheKey, func() (int64, error) {
commit, err := repo.GetCommit(sha)
if err != nil {
return 0, err
}
return commit.CommitsCount()
})
if err != nil {
return err
}
repo.LastCommitCache = NewLastCommitCache(commitsCount, fullName, repo, cache.GetCache())
}
return nil
}
// GetCommitBranchStart returns the commit where the branch diverged
func (repo *Repository) GetCommitBranchStart(env []string, branch, endCommitID string) (string, error) {
cmd := NewCommand("log", prettyLogFormat)

12
modules/setting/cache.go
View File

@ -23,8 +23,7 @@ var CacheService = struct {
Cache `ini:"cache"`
LastCommit struct {
TTL time.Duration `ini:"ITEM_TTL"`
CommitsCount int64
TTL time.Duration `ini:"ITEM_TTL"`
} `ini:"cache.last_commit"`
}{
Cache: Cache{
@ -33,11 +32,9 @@ var CacheService = struct {
TTL: 16 * time.Hour,
},
LastCommit: struct {
TTL time.Duration `ini:"ITEM_TTL"`
CommitsCount int64
TTL time.Duration `ini:"ITEM_TTL"`
}{
TTL: 8760 * time.Hour,
CommitsCount: 1000,
TTL: 8760 * time.Hour,
},
}
@ -63,9 +60,6 @@ func loadCacheFrom(rootCfg ConfigProvider) {
default:
log.Fatal("Unknown cache adapter: %s", CacheService.Adapter)
}
sec = rootCfg.Section("cache.last_commit")
CacheService.LastCommit.CommitsCount = sec.Key("COMMITS_COUNT").MustInt64(1000)
}
// TTLSeconds returns the TTLSeconds or unix timestamp for memcache

2
routers/web/repo/repo.go
View File

@ -18,7 +18,6 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/optional"
@ -77,7 +76,6 @@ func CommitInfoCache(ctx *context.Context) {
return
}
ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount
ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(ctx.Repo.CommitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache())
}
func checkContextUser(ctx *context.Context, uid int64) *user_model.User {

1
services/context/repo.go
View File

@ -937,7 +937,6 @@ func RepoRefByType(detectRefType git.RefType) func(*Context) {
return
}
ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount
ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(ctx.Repo.CommitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache())
}
}

11
services/repository/cache.go
View File

@ -7,7 +7,6 @@ import (
"context"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
)
@ -18,13 +17,5 @@ func CacheRef(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Rep
return err
}
if gitRepo.LastCommitCache == nil {
commitsCount, err := cache.GetInt64(repo.GetCommitsCountCacheKey(fullRefName.ShortName(), true), commit.CommitsCount)
if err != nil {
return err
}
gitRepo.LastCommitCache = git.NewLastCommitCache(commitsCount, repo.FullName(), gitRepo, cache.GetCache())
}
return commit.CacheCommit(ctx)
return gitRepo.CacheCommit(ctx, commit)
}

5
services/repository/files/content.go
View File

@ -134,11 +134,6 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, refCommit *ut
}
selfURLString := selfURL.String()
err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(refCommit.InputRef, refType != git.RefTypeCommit), repo.FullName(), refCommit.CommitID)
if err != nil {
return nil, err
}
lastCommit, err := commit.GetCommitByPath(treePath)
if err != nil {
return nil, err

4
tests/integration/repo_test.go
View File

@ -129,14 +129,10 @@ func testViewRepoWithCache(t *testing.T) {
// FIXME: these test don't seem quite right, no enough assert
// no last commit cache
testView(t)
// enable last commit cache for all repositories
oldCommitsCount := setting.CacheService.LastCommit.CommitsCount
setting.CacheService.LastCommit.CommitsCount = 0
// first view will not hit the cache
testView(t)
// second view will hit the cache
testView(t)
setting.CacheService.LastCommit.CommitsCount = oldCommitsCount
}
func testViewRepoPrivate(t *testing.T) {