Compare commits

...

6 Commits

Author SHA1 Message Date
Lunny Xiao
e6cb5d4034
Merge 9363b591eda59e4fb6394c58f77d6f564faf2414 into b907b9fb1a1a792b9bc25112fa3eb3f8d2fb4397 2025-09-30 23:43:26 -07:00
silverwind
b907b9fb1a
Enable a few more tsconfig options (#35553)
Enable a few more useful options in tsconfig. `noImplicitReturns` had
two cases which I've fixed. Also, partially sort the file.
2025-09-30 21:43:41 -07:00
dependabot[bot]
c5d74e5869
Bump github.com/wneessen/go-mail from 0.6.2 to 0.7.1 (#35557) 2025-10-01 00:14:53 +00:00
Lunny Xiao
9363b591ed Allow exit when caculating language in gogit version 2025-06-07 22:11:27 -07:00
Lunny Xiao
57ba66822a Fix lint 2025-06-07 22:11:26 -07:00
Lunny Xiao
a68e04c0e0 Reduce memory usage when cacluting repository languages 2025-06-07 22:11:17 -07:00
11 changed files with 122 additions and 64 deletions

6
go.mod
View File

@ -109,7 +109,7 @@ require (
github.com/ulikunitz/xz v0.5.15 github.com/ulikunitz/xz v0.5.15
github.com/urfave/cli-docs/v3 v3.0.0-alpha6 github.com/urfave/cli-docs/v3 v3.0.0-alpha6
github.com/urfave/cli/v3 v3.4.1 github.com/urfave/cli/v3 v3.4.1
github.com/wneessen/go-mail v0.6.2 github.com/wneessen/go-mail v0.7.1
github.com/xeipuuv/gojsonschema v1.2.0 github.com/xeipuuv/gojsonschema v1.2.0
github.com/yohcop/openid-go v1.0.1 github.com/yohcop/openid-go v1.0.1
github.com/yuin/goldmark v1.7.13 github.com/yuin/goldmark v1.7.13
@ -120,9 +120,9 @@ require (
golang.org/x/image v0.30.0 golang.org/x/image v0.30.0
golang.org/x/net v0.43.0 golang.org/x/net v0.43.0
golang.org/x/oauth2 v0.30.0 golang.org/x/oauth2 v0.30.0
golang.org/x/sync v0.16.0 golang.org/x/sync v0.17.0
golang.org/x/sys v0.35.0 golang.org/x/sys v0.35.0
golang.org/x/text v0.28.0 golang.org/x/text v0.29.0
google.golang.org/grpc v1.75.0 google.golang.org/grpc v1.75.0
google.golang.org/protobuf v1.36.8 google.golang.org/protobuf v1.36.8
gopkg.in/ini.v1 v1.67.0 gopkg.in/ini.v1 v1.67.0

17
go.sum
View File

@ -765,8 +765,8 @@ github.com/urfave/cli/v3 v3.4.1/go.mod h1:FJSKtM/9AiiTOJL4fJ6TbMUkxBXn7GO9guZqoZ
github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/wneessen/go-mail v0.6.2 h1:c6V7c8D2mz868z9WJ+8zDKtUyLfZ1++uAZmo2GRFji8= github.com/wneessen/go-mail v0.7.1 h1:rvy63sp14N06/kdGqCYwW8Na5gDCXjTQM1E7So4PuKk=
github.com/wneessen/go-mail v0.6.2/go.mod h1:L/PYjPK3/2ZlNb2/FjEBIn9n1rUWjW+Toy531oVmeb4= github.com/wneessen/go-mail v0.7.1/go.mod h1:+TkW6QP3EVkgTEqHtVmnAE/1MRhmzb8Y9/W3pweuS+k=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
@ -837,7 +837,6 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -930,9 +929,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -974,7 +972,6 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
@ -987,7 +984,6 @@ golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4=
golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -1003,9 +999,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=

View File

@ -7,6 +7,7 @@ package languagestats
import ( import (
"bytes" "bytes"
"context"
"io" "io"
"code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/analyze"
@ -20,8 +21,8 @@ import (
"github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/object"
) )
// GetLanguageStats calculates language stats for git repository at specified commit // CalcLanguageStats calculates language stats for git repository at specified commit
func GetLanguageStats(repo *git_module.Repository, commitID string) (map[string]int64, error) { func CalcLanguageStats(ctx context.Context, repo *git_module.Repository, commitID string) (map[string]int64, error) {
r, err := git.PlainOpen(repo.Path) r, err := git.PlainOpen(repo.Path)
if err != nil { if err != nil {
return nil, err return nil, err
@ -58,6 +59,13 @@ func GetLanguageStats(repo *git_module.Repository, commitID string) (map[string]
firstExcludedLanguageSize := int64(0) firstExcludedLanguageSize := int64(0)
err = tree.Files().ForEach(func(f *object.File) error { err = tree.Files().ForEach(func(f *object.File) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
return nil
}
if f.Size == 0 { if f.Size == 0 {
return nil return nil
} }

View File

@ -7,6 +7,7 @@ package languagestats
import ( import (
"bytes" "bytes"
"context"
"io" "io"
"code.gitea.io/gitea/modules/analyze" "code.gitea.io/gitea/modules/analyze"
@ -18,8 +19,8 @@ import (
"github.com/go-enry/go-enry/v2" "github.com/go-enry/go-enry/v2"
) )
// GetLanguageStats calculates language stats for git repository at specified commit // CalcLanguageStats calculates language stats for git repository at specified commit
func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64, error) { func CalcLanguageStats(ctx context.Context, repo *git.Repository, commitID string) (map[string]int64, error) {
// We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary. // We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary.
// so let's create a batch stdin and stdout // so let's create a batch stdin and stdout
batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx) batchStdinWriter, batchReader, cancel, err := repo.CatFileBatch(repo.Ctx)
@ -59,11 +60,6 @@ func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64,
tree := commit.Tree tree := commit.Tree
entries, err := tree.ListEntriesRecursiveWithSize()
if err != nil {
return nil, err
}
checker, err := attribute.NewBatchChecker(repo, commitID, attribute.LinguistAttributes) checker, err := attribute.NewBatchChecker(repo, commitID, attribute.LinguistAttributes)
if err != nil { if err != nil {
return nil, err return nil, err
@ -82,18 +78,12 @@ func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64,
firstExcludedLanguage := "" firstExcludedLanguage := ""
firstExcludedLanguageSize := int64(0) firstExcludedLanguageSize := int64(0)
for _, f := range entries { if err := tree.IterateEntriesRecursive(ctx, func(ctx context.Context, f *git.TreeEntry) error {
select {
case <-repo.Ctx.Done():
return sizes, repo.Ctx.Err()
default:
}
contentBuf.Reset() contentBuf.Reset()
content = contentBuf.Bytes() content = contentBuf.Bytes()
if f.Size() == 0 { if f.Size() == 0 {
continue return nil
} }
isVendored := optional.None[bool]() isVendored := optional.None[bool]()
@ -104,19 +94,19 @@ func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64,
attrLinguistGenerated := optional.None[bool]() attrLinguistGenerated := optional.None[bool]()
if err == nil { if err == nil {
if isVendored = attrs.GetVendored(); isVendored.ValueOrDefault(false) { if isVendored = attrs.GetVendored(); isVendored.ValueOrDefault(false) {
continue return nil
} }
if attrLinguistGenerated = attrs.GetGenerated(); attrLinguistGenerated.ValueOrDefault(false) { if attrLinguistGenerated = attrs.GetGenerated(); attrLinguistGenerated.ValueOrDefault(false) {
continue return nil
} }
if isDocumentation = attrs.GetDocumentation(); isDocumentation.ValueOrDefault(false) { if isDocumentation = attrs.GetDocumentation(); isDocumentation.ValueOrDefault(false) {
continue return nil
} }
if isDetectable = attrs.GetDetectable(); !isDetectable.ValueOrDefault(true) { if isDetectable = attrs.GetDetectable(); !isDetectable.ValueOrDefault(true) {
continue return nil
} }
if hasLanguage := attrs.GetLanguage(); hasLanguage.Value() != "" { if hasLanguage := attrs.GetLanguage(); hasLanguage.Value() != "" {
@ -130,7 +120,7 @@ func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64,
// this language will always be added to the size // this language will always be added to the size
sizes[language] += f.Size() sizes[language] += f.Size()
continue return nil
} }
} }
@ -138,19 +128,19 @@ func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64,
enry.IsDotFile(f.Name()) || enry.IsDotFile(f.Name()) ||
(!isDocumentation.Has() && enry.IsDocumentation(f.Name())) || (!isDocumentation.Has() && enry.IsDocumentation(f.Name())) ||
enry.IsConfiguration(f.Name()) { enry.IsConfiguration(f.Name()) {
continue return nil
} }
// If content can not be read or file is too big just do detection by filename // If content can not be read or file is too big just do detection by filename
if f.Size() <= bigFileSize { if f.Size() <= bigFileSize {
if err := writeID(f.ID.String()); err != nil { if err := writeID(f.ID.String()); err != nil {
return nil, err return err
} }
_, _, size, err := git.ReadBatchLine(batchReader) _, _, size, err := git.ReadBatchLine(batchReader)
if err != nil { if err != nil {
log.Debug("Error reading blob: %s Err: %v", f.ID.String(), err) log.Debug("Error reading blob: %s Err: %v", f.ID.String(), err)
return nil, err return err
} }
sizeToRead := size sizeToRead := size
@ -162,11 +152,11 @@ func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64,
_, err = contentBuf.ReadFrom(io.LimitReader(batchReader, sizeToRead)) _, err = contentBuf.ReadFrom(io.LimitReader(batchReader, sizeToRead))
if err != nil { if err != nil {
return nil, err return err
} }
content = contentBuf.Bytes() content = contentBuf.Bytes()
if err := git.DiscardFull(batchReader, discard); err != nil { if err := git.DiscardFull(batchReader, discard); err != nil {
return nil, err return err
} }
} }
@ -178,14 +168,14 @@ func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64,
isGenerated = enry.IsGenerated(f.Name(), content) isGenerated = enry.IsGenerated(f.Name(), content)
} }
if isGenerated { if isGenerated {
continue return nil
} }
// FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary? // FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary?
// - eg. do the all the detection tests using filename first before reading content. // - eg. do the all the detection tests using filename first before reading content.
language := analyze.GetCodeLanguage(f.Name(), content) language := analyze.GetCodeLanguage(f.Name(), content)
if language == "" { if language == "" {
continue return nil
} }
// group languages, such as Pug -> HTML; SCSS -> CSS // group languages, such as Pug -> HTML; SCSS -> CSS
@ -206,6 +196,9 @@ func GetLanguageStats(repo *git.Repository, commitID string) (map[string]int64,
firstExcludedLanguage = language firstExcludedLanguage = language
firstExcludedLanguageSize += f.Size() firstExcludedLanguageSize += f.Size()
} }
return nil
}, git.TrustedCmdArgs{"--long"}); err != nil {
return sizes, err
} }
// If there are no included languages add the first excluded language // If there are no included languages add the first excluded language

View File

@ -22,7 +22,7 @@ func TestRepository_GetLanguageStats(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
defer gitRepo.Close() defer gitRepo.Close()
stats, err := GetLanguageStats(gitRepo, "8fee858da5796dfb37704761701bb8e800ad9ef3") stats, err := CalcLanguageStats(t.Context(), gitRepo, "8fee858da5796dfb37704761701bb8e800ad9ef3")
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, map[string]int64{ assert.Equal(t, map[string]int64{

View File

@ -22,6 +22,13 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) {
// parseTreeEntries FIXME this function's design is not right, it should not make the caller read all data into memory // parseTreeEntries FIXME this function's design is not right, it should not make the caller read all data into memory
func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1) entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1)
return entries, iterateTreeEntries(data, ptree, func(entry *TreeEntry) error {
entries = append(entries, entry)
return nil
})
}
func iterateTreeEntries(data []byte, ptree *Tree, f func(entry *TreeEntry) error) error {
for pos := 0; pos < len(data); { for pos := 0; pos < len(data); {
posEnd := bytes.IndexByte(data[pos:], '\n') posEnd := bytes.IndexByte(data[pos:], '\n')
if posEnd == -1 { if posEnd == -1 {
@ -33,7 +40,7 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
line := data[pos:posEnd] line := data[pos:posEnd]
lsTreeLine, err := parseLsTreeLine(line) lsTreeLine, err := parseLsTreeLine(line)
if err != nil { if err != nil {
return nil, err return err
} }
entry := &TreeEntry{ entry := &TreeEntry{
ptree: ptree, ptree: ptree,
@ -44,9 +51,11 @@ func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
sized: lsTreeLine.Size.Has(), sized: lsTreeLine.Size.Has(),
} }
pos = posEnd + 1 pos = posEnd + 1
entries = append(entries, entry) if err := f(entry); err != nil {
return err
}
} }
return entries, nil return nil
} }
func catBatchParseTreeEntries(objectFormat ObjectFormat, ptree *Tree, rd *bufio.Reader, sz int64) ([]*TreeEntry, error) { func catBatchParseTreeEntries(objectFormat ObjectFormat, ptree *Tree, rd *bufio.Reader, sz int64) ([]*TreeEntry, error) {

View File

@ -6,6 +6,8 @@
package git package git
import ( import (
"bufio"
"context"
"io" "io"
"strings" "strings"
@ -124,3 +126,50 @@ func (t *Tree) ListEntriesRecursiveFast() (Entries, error) {
func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) {
return t.listEntriesRecursive(gitcmd.TrustedCmdArgs{"--long"}) return t.listEntriesRecursive(gitcmd.TrustedCmdArgs{"--long"})
} }
// IterateEntriesRecursive returns iterate entries of current tree recursively including all subtrees
// extraArgs could be "-l" to get the size, which is slower
func (t *Tree) IterateEntriesRecursive(ctx context.Context, f func(ctx context.Context, entry *TreeEntry) error, extraArgs TrustedCmdArgs) error {
reader, writer := io.Pipe()
done := make(chan error)
go func(t *Tree, done chan error, writer *io.PipeWriter) {
runErr := NewCommand("ls-tree", "-t", "-r").
AddArguments(extraArgs...).
AddDynamicArguments(t.ID.String()).
Run(ctx, &RunOpts{
Dir: t.repo.Path,
Stdout: writer,
})
_ = writer.Close()
done <- runErr
}(t, done, writer)
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
if err := scanner.Err(); err != nil {
return err
}
data := scanner.Bytes()
if err := iterateTreeEntries(data, t, func(entry *TreeEntry) error {
if err := f(ctx, entry); err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
case runErr := <-done:
return runErr
default:
return nil
}
}); err != nil {
return err
}
}
return nil
}

View File

@ -63,7 +63,7 @@ func (db *DBIndexer) Index(id int64) error {
} }
// Calculate and save language statistics to database // Calculate and save language statistics to database
stats, err := languagestats.GetLanguageStats(gitRepo, commitID) stats, err := languagestats.CalcLanguageStats(ctx, gitRepo, commitID)
if err != nil { if err != nil {
if !setting.IsInTesting { if !setting.IsInTesting {
log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.FullName(), err) log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.FullName(), err)

View File

@ -1,7 +1,7 @@
{ {
"include": [ "include": [
"${configDir}/.*",
"${configDir}/*", "${configDir}/*",
"${configDir}/.*",
"${configDir}/tests/e2e/**/*", "${configDir}/tests/e2e/**/*",
"${configDir}/tests/e2e/**/.*", "${configDir}/tests/e2e/**/.*",
"${configDir}/tools/**/*", "${configDir}/tools/**/*",
@ -17,27 +17,31 @@
"allowImportingTsExtensions": true, "allowImportingTsExtensions": true,
"allowJs": true, "allowJs": true,
"allowSyntheticDefaultImports": true, "allowSyntheticDefaultImports": true,
"allowUnreachableCode": false,
"allowUnusedLabels": false,
"alwaysStrict": true, "alwaysStrict": true,
"erasableSyntaxOnly": true, "erasableSyntaxOnly": true,
"esModuleInterop": true, "esModuleInterop": true,
"exactOptionalPropertyTypes": false,
"isolatedModules": true, "isolatedModules": true,
"libReplacement": false, "libReplacement": false,
"noEmit": true, "noEmit": true,
"noFallthroughCasesInSwitch": true,
"noImplicitAny": true,
"noImplicitReturns": true,
"noImplicitThis": true,
"noPropertyAccessFromIndexSignature": false,
"noUnusedLocals": true,
"noUnusedParameters": true,
"resolveJsonModule": true, "resolveJsonModule": true,
"skipLibCheck": true, "skipLibCheck": true,
"verbatimModuleSyntax": true, "sourceMap": true,
"stripInternal": true,
"strict": false, "strict": false,
"strictBindCallApply": true, "strictBindCallApply": true,
"strictBuiltinIteratorReturn": true, "strictBuiltinIteratorReturn": true,
"strictFunctionTypes": true, "strictFunctionTypes": true,
"noImplicitAny": true, "stripInternal": true,
"noImplicitThis": true, "verbatimModuleSyntax": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noPropertyAccessFromIndexSignature": false,
"exactOptionalPropertyTypes": false,
"sourceMap": true,
"types": [ "types": [
"vitest/globals", "vitest/globals",
"./web_src/js/globals.d.ts", "./web_src/js/globals.d.ts",

View File

@ -10,7 +10,7 @@ export function initTableSort() {
} }
function tableSort(normSort: string, revSort: string, isDefault: string) { function tableSort(normSort: string, revSort: string, isDefault: string) {
if (!normSort) return false; if (!normSort) return;
if (!revSort) revSort = ''; if (!revSort) revSort = '';
const url = new URL(window.location.href); const url = new URL(window.location.href);

View File

@ -43,7 +43,7 @@ type ToastOpts = {
type ToastifyElement = HTMLElement & {_giteaToastifyInstance?: Toast}; type ToastifyElement = HTMLElement & {_giteaToastifyInstance?: Toast};
/** See https://github.com/apvarun/toastify-js#api for options */ /** See https://github.com/apvarun/toastify-js#api for options */
function showToast(message: string, level: Intent, {gravity, position, duration, useHtmlBody, preventDuplicates = true, ...other}: ToastOpts = {}): Toast { function showToast(message: string, level: Intent, {gravity, position, duration, useHtmlBody, preventDuplicates = true, ...other}: ToastOpts = {}): Toast | null {
const body = useHtmlBody ? message : htmlEscape(message); const body = useHtmlBody ? message : htmlEscape(message);
const parent = document.querySelector('.ui.dimmer.active') ?? document.body; const parent = document.querySelector('.ui.dimmer.active') ?? document.body;
const duplicateKey = preventDuplicates ? (preventDuplicates === true ? `${level}-${body}` : preventDuplicates) : ''; const duplicateKey = preventDuplicates ? (preventDuplicates === true ? `${level}-${body}` : preventDuplicates) : '';
@ -56,7 +56,7 @@ function showToast(message: string, level: Intent, {gravity, position, duration,
showElem(toastDupNumEl); showElem(toastDupNumEl);
toastDupNumEl.textContent = String(Number(toastDupNumEl.textContent) + 1); toastDupNumEl.textContent = String(Number(toastDupNumEl.textContent) + 1);
animateOnce(toastDupNumEl, 'pulse-1p5-200'); animateOnce(toastDupNumEl, 'pulse-1p5-200');
return; return null;
} }
} }
@ -83,15 +83,15 @@ function showToast(message: string, level: Intent, {gravity, position, duration,
return toast; return toast;
} }
export function showInfoToast(message: string, opts?: ToastOpts): Toast { export function showInfoToast(message: string, opts?: ToastOpts): Toast | null {
return showToast(message, 'info', opts); return showToast(message, 'info', opts);
} }
export function showWarningToast(message: string, opts?: ToastOpts): Toast { export function showWarningToast(message: string, opts?: ToastOpts): Toast | null {
return showToast(message, 'warning', opts); return showToast(message, 'warning', opts);
} }
export function showErrorToast(message: string, opts?: ToastOpts): Toast { export function showErrorToast(message: string, opts?: ToastOpts): Toast | null {
return showToast(message, 'error', opts); return showToast(message, 'error', opts);
} }