Compare commits

...

3 Commits

Author SHA1 Message Date
KN4CK3R
01acd1eea3
Strip / from relative links (#28932)
Fixes #28915

Restores the old behaviour:

https://github.com/go-gitea/gitea/pull/26745/files#diff-d78a9d361b1fddc12218e4dd42f42d39d6be1fda184041e06bb6fb30f0d94c59L96
2024-01-27 11:36:01 +08:00
wackbyte
a240d5dfa7
Fix non-alphabetic sorting of repo topics (#28938) 2024-01-26 14:15:57 +00:00
Lunny Xiao
534917d576
Don't remove all mirror repository's releases when mirroring (#28817)
Fix #22066

# Purpose

This PR fix the releases will be deleted when mirror repository sync the
tags.

# The problem

In the previous implementation of #19125. All releases record in
databases of one mirror repository will be deleted before sync.
Ref:
https://github.com/go-gitea/gitea/pull/19125/files#diff-2aa04998a791c30e5a02b49a97c07fcd93d50e8b31640ce2ddb1afeebf605d02R481

# The Pros

This PR introduced a new method which will load all releases from
databases and all tags on git data into memory. And detect which tags
needs to be inserted, which tags need to be updated or deleted. Only
tags releases(IsTag=true) which are not included in git data will be
deleted, only tags which sha1 changed will be updated. So it will not
delete any real releases include drafts.

# The Cons

The drawback is the memory usage will be higher than before if there are
many tags on this repository. This PR defined a special release struct
to reduce columns loaded from database to memory.
2024-01-26 06:18:19 +00:00
5 changed files with 179 additions and 11 deletions

View File

@ -366,7 +366,7 @@ func syncTopicsInRepository(sess db.Engine, repoID int64) error {
topicNames := make([]string, 0, 25)
if err := sess.Table("topic").Cols("name").
Join("INNER", "repo_topic", "repo_topic.topic_id = topic.id").
Where("repo_topic.repo_id = ?", repoID).Desc("topic.repo_count").Find(&topicNames); err != nil {
Where("repo_topic.repo_id = ?", repoID).Asc("topic.name").Find(&topicNames); err != nil {
return err
}

View File

@ -85,9 +85,11 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa
// 2. If they're not wrapped with a link they need a link wrapper
// Check if the destination is a real link
link := v.Destination
if len(link) > 0 && !markup.IsLink(link) {
v.Destination = []byte(giteautil.URLJoin(ctx.Links.ResolveMediaLink(ctx.IsWiki), string(link)))
if len(v.Destination) > 0 && !markup.IsLink(v.Destination) {
v.Destination = []byte(giteautil.URLJoin(
ctx.Links.ResolveMediaLink(ctx.IsWiki),
strings.TrimLeft(string(v.Destination), "/"),
))
}
parent := n.Parent()
@ -103,7 +105,7 @@ func (g *ASTTransformer) Transform(node *ast.Document, reader text.Reader, pc pa
// Duplicate the current image node
image := ast.NewImage(ast.NewLink())
image.Destination = link
image.Destination = v.Destination
image.Title = v.Title
for _, attr := range v.Attributes() {
image.SetAttribute(attr.Name, attr.Value)

View File

@ -580,6 +580,8 @@ https://example.com/file.bin
[[local link|file.bin]]
[[remote link|https://example.com]]
![local image](image.jpg)
![local image](path/file)
![local image](/path/file)
![remote image](https://example.com/image.jpg)
[[local image|image.jpg]]
[[remote link|https://example.com/image.jpg]]
@ -609,6 +611,8 @@ mail@domain.com
<a href="/src/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="/image.jpg" target="_blank" rel="nofollow noopener"><img src="/image.jpg" alt="local image"/></a><br/>
<a href="/path/file" target="_blank" rel="nofollow noopener"><img src="/path/file" alt="local image"/></a><br/>
<a href="/path/file" target="_blank" rel="nofollow noopener"><img src="/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="/image.jpg" rel="nofollow"><img src="/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>
@ -634,6 +638,8 @@ space</p>
<a href="/wiki/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="/wiki/raw/image.jpg" alt="local image"/></a><br/>
<a href="/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="/wiki/raw/image.jpg" rel="nofollow"><img src="/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>
@ -661,6 +667,8 @@ space</p>
<a href="https://gitea.io/src/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="https://gitea.io/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/image.jpg" alt="local image"/></a><br/>
<a href="https://gitea.io/path/file" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/path/file" alt="local image"/></a><br/>
<a href="https://gitea.io/path/file" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="https://gitea.io/image.jpg" rel="nofollow"><img src="https://gitea.io/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>
@ -688,6 +696,8 @@ space</p>
<a href="https://gitea.io/wiki/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="https://gitea.io/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/wiki/raw/image.jpg" alt="local image"/></a><br/>
<a href="https://gitea.io/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="https://gitea.io/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="https://gitea.io/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="https://gitea.io/wiki/raw/image.jpg" rel="nofollow"><img src="https://gitea.io/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>
@ -715,6 +725,8 @@ space</p>
<a href="/relative/path/src/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="/relative/path/image.jpg" target="_blank" rel="nofollow noopener"><img src="/relative/path/image.jpg" alt="local image"/></a><br/>
<a href="/relative/path/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/path/file" alt="local image"/></a><br/>
<a href="/relative/path/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="/relative/path/image.jpg" rel="nofollow"><img src="/relative/path/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>
@ -742,6 +754,8 @@ space</p>
<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="/relative/path/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/image.jpg" alt="local image"/></a><br/>
<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="/relative/path/wiki/raw/image.jpg" rel="nofollow"><img src="/relative/path/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>
@ -770,6 +784,8 @@ space</p>
<a href="/user/repo/src/branch/main/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="/user/repo/media/branch/main/image.jpg" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/image.jpg" alt="local image"/></a><br/>
<a href="/user/repo/media/branch/main/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/path/file" alt="local image"/></a><br/>
<a href="/user/repo/media/branch/main/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="/user/repo/media/branch/main/image.jpg" rel="nofollow"><img src="/user/repo/media/branch/main/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>
@ -798,6 +814,8 @@ space</p>
<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="/relative/path/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/image.jpg" alt="local image"/></a><br/>
<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="/relative/path/wiki/raw/image.jpg" rel="nofollow"><img src="/relative/path/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>
@ -826,6 +844,8 @@ space</p>
<a href="/user/repo/src/sub/folder/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="/user/repo/image.jpg" target="_blank" rel="nofollow noopener"><img src="/user/repo/image.jpg" alt="local image"/></a><br/>
<a href="/user/repo/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/path/file" alt="local image"/></a><br/>
<a href="/user/repo/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="/user/repo/image.jpg" rel="nofollow"><img src="/user/repo/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>
@ -854,6 +874,8 @@ space</p>
<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="/relative/path/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/image.jpg" alt="local image"/></a><br/>
<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="/relative/path/wiki/raw/image.jpg" rel="nofollow"><img src="/relative/path/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>
@ -883,6 +905,8 @@ space</p>
<a href="/user/repo/src/branch/main/sub/folder/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="/user/repo/media/branch/main/sub/folder/image.jpg" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/sub/folder/image.jpg" alt="local image"/></a><br/>
<a href="/user/repo/media/branch/main/sub/folder/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/sub/folder/path/file" alt="local image"/></a><br/>
<a href="/user/repo/media/branch/main/sub/folder/path/file" target="_blank" rel="nofollow noopener"><img src="/user/repo/media/branch/main/sub/folder/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="/user/repo/media/branch/main/sub/folder/image.jpg" rel="nofollow"><img src="/user/repo/media/branch/main/sub/folder/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>
@ -912,6 +936,8 @@ space</p>
<a href="/relative/path/wiki/file.bin" rel="nofollow">local link</a><br/>
<a href="https://example.com" rel="nofollow">remote link</a><br/>
<a href="/relative/path/wiki/raw/image.jpg" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/image.jpg" alt="local image"/></a><br/>
<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="/relative/path/wiki/raw/path/file" target="_blank" rel="nofollow noopener"><img src="/relative/path/wiki/raw/path/file" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" target="_blank" rel="nofollow noopener"><img src="https://example.com/image.jpg" alt="remote image"/></a><br/>
<a href="/relative/path/wiki/raw/image.jpg" rel="nofollow"><img src="/relative/path/wiki/raw/image.jpg" title="local image" alt="local image"/></a><br/>
<a href="https://example.com/image.jpg" rel="nofollow"><img src="https://example.com/image.jpg" title="remote link" alt="remote link"/></a><br/>

View File

@ -508,6 +508,18 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re
return nil
}
// shortRelease to reduce load memory, this struct can replace repo_model.Release
type shortRelease struct {
ID int64
TagName string
Sha1 string
IsTag bool
}
func (shortRelease) TableName() string {
return "release"
}
// pullMirrorReleaseSync is a pull-mirror specific tag<->release table
// synchronization which overwrites all Releases from the repository tags. This
// can be relied on since a pull-mirror is always identical to its
@ -521,16 +533,20 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git
return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
}
err = db.WithTx(ctx, func(ctx context.Context) error {
//
// clear out existing releases
//
if _, err := db.DeleteByBean(ctx, &repo_model.Release{RepoID: repo.ID}); err != nil {
return fmt.Errorf("unable to clear releases for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
dbReleases, err := db.Find[shortRelease](ctx, repo_model.FindReleasesOptions{
RepoID: repo.ID,
IncludeDrafts: true,
IncludeTags: true,
})
if err != nil {
return fmt.Errorf("unable to FindReleases in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
}
inserts, deletes, updates := calcSync(tags, dbReleases)
//
// make release set identical to upstream tags
//
for _, tag := range tags {
for _, tag := range inserts {
release := repo_model.Release{
RepoID: repo.ID,
TagName: tag.Name,
@ -547,6 +563,25 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git
return fmt.Errorf("unable insert tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err)
}
}
// only delete tags releases
if len(deletes) > 0 {
if _, err := db.GetEngine(ctx).Where("repo_id=?", repo.ID).
In("id", deletes).
Delete(&repo_model.Release{}); err != nil {
return fmt.Errorf("unable to delete tags for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
}
}
for _, tag := range updates {
if _, err := db.GetEngine(ctx).Where("repo_id = ? AND lower_tag_name = ?", repo.ID, strings.ToLower(tag.Name)).
Cols("sha1").
Update(&repo_model.Release{
Sha1: tag.Object.String(),
}); err != nil {
return fmt.Errorf("unable to update tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err)
}
}
return nil
})
if err != nil {
@ -556,3 +591,32 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git
log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags)
return nil
}
func calcSync(destTags []*git.Tag, dbTags []*shortRelease) ([]*git.Tag, []int64, []*git.Tag) {
destTagMap := make(map[string]*git.Tag)
for _, tag := range destTags {
destTagMap[tag.Name] = tag
}
dbTagMap := make(map[string]*shortRelease)
for _, rel := range dbTags {
dbTagMap[rel.TagName] = rel
}
inserted := make([]*git.Tag, 0, 10)
updated := make([]*git.Tag, 0, 10)
for _, tag := range destTags {
rel := dbTagMap[tag.Name]
if rel == nil {
inserted = append(inserted, tag)
} else if rel.Sha1 != tag.Object.String() {
updated = append(updated, tag)
}
}
deleted := make([]int64, 0, 10)
for _, tag := range dbTags {
if destTagMap[tag.TagName] == nil && tag.IsTag {
deleted = append(deleted, tag.ID)
}
}
return inserted, deleted, updated
}

View File

@ -0,0 +1,76 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"testing"
"code.gitea.io/gitea/modules/git"
"github.com/stretchr/testify/assert"
)
func Test_calcSync(t *testing.T) {
gitTags := []*git.Tag{
/*{
Name: "v0.1.0-beta", //deleted tag
Object: git.MustIDFromString(""),
},
{
Name: "v0.1.1-beta", //deleted tag but release should not be deleted because it's a release
Object: git.MustIDFromString(""),
},
*/
{
Name: "v1.0.0", // keep as before
Object: git.MustIDFromString("1006e6e13c73ad3d9e2d5682ad266b5016523485"),
},
{
Name: "v1.1.0", // retagged with new commit id
Object: git.MustIDFromString("bbdb7df30248e7d4a26a909c8d2598a152e13868"),
},
{
Name: "v1.2.0", // new tag
Object: git.MustIDFromString("a5147145e2f24d89fd6d2a87826384cc1d253267"),
},
}
dbReleases := []*shortRelease{
{
ID: 1,
TagName: "v0.1.0-beta",
Sha1: "244758d7da8dd1d9e0727e8cb7704ed4ba9a17c3",
IsTag: true,
},
{
ID: 2,
TagName: "v0.1.1-beta",
Sha1: "244758d7da8dd1d9e0727e8cb7704ed4ba9a17c3",
IsTag: false,
},
{
ID: 3,
TagName: "v1.0.0",
Sha1: "1006e6e13c73ad3d9e2d5682ad266b5016523485",
},
{
ID: 4,
TagName: "v1.1.0",
Sha1: "53ab18dcecf4152b58328d1f47429510eb414d50",
},
}
inserts, deletes, updates := calcSync(gitTags, dbReleases)
if assert.EqualValues(t, 1, len(inserts), "inserts") {
assert.EqualValues(t, *gitTags[2], *inserts[0], "inserts equal")
}
if assert.EqualValues(t, 1, len(deletes), "deletes") {
assert.EqualValues(t, 1, deletes[0], "deletes equal")
}
if assert.EqualValues(t, 1, len(updates), "updates") {
assert.EqualValues(t, *gitTags[1], *updates[0], "updates equal")
}
}