Support importing comment types (#22510)

This commit adds support for specifying comment types when importing
with `gitea restore-repo`. It makes it possible to import issue changes,
such as "title changed" or "assigned user changed".

An earlier version of this pull request was made by Matti Ranta, in
https://future.projects.blender.org/blender-migration/gitea-bf/pulls/3

There are two changes with regard to Matti's original code:

1. The comment type was an `int64` in Matti's code, and is now using a
string. This makes it possible to use `comment_type: title`, which is
more reliable and future-proof than an index into an internal list in
the Gitea Go code.

2. Matti's code also had support for including labels, but in a way that
would require knowing the database ID of the labels before the import
even starts, which is impossible. This can be solved by using label
names instead of IDs; for simplicity I I left that out of this PR.

models/issues/comment.go

@@ -175,6 +175,15 @@ func (t CommentType) String() string {
 	return commentStrings[t]
 }
 
+func AsCommentType(typeName string) CommentType {
+	for index, name := range commentStrings {
+		if typeName == name {
+			return CommentType(index)
+		}
+	}
+	return CommentTypeUnknown
+}
+
 // RoleDescriptor defines comment tag type
 type RoleDescriptor int
 

models/issues/comment_test.go

@@ -62,3 +62,10 @@ func TestFetchCodeComments(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Len(t, res, 1)
 }
+
+func TestAsCommentType(t *testing.T) {
+	assert.Equal(t, issues_model.CommentTypeUnknown, issues_model.AsCommentType(""))
+	assert.Equal(t, issues_model.CommentTypeUnknown, issues_model.AsCommentType("nonsense"))
+	assert.Equal(t, issues_model.CommentTypeComment, issues_model.AsCommentType("comment"))
+	assert.Equal(t, issues_model.CommentTypePRUnScheduledToAutoMerge, issues_model.AsCommentType("pull_cancel_scheduled_merge"))
+}

models/packages/package_blob.go

@@ -85,7 +85,16 @@ func DeleteBlobByID(ctx context.Context, blobID int64) error {
 }
 
 // GetTotalBlobSize returns the total blobs size in bytes
-func GetTotalBlobSize() (int64, error) {
+func GetTotalBlobSize(ctx context.Context) (int64, error) {
-	return db.GetEngine(db.DefaultContext).
+	return db.GetEngine(ctx).
+		SumInt(&PackageBlob{}, "size")
+}
+
+// GetTotalUnreferencedBlobSize returns the total size of all unreferenced blobs in bytes
+func GetTotalUnreferencedBlobSize(ctx context.Context) (int64, error) {
+	return db.GetEngine(ctx).
+		Table("package_blob").
+		Join("LEFT", "package_file", "package_file.blob_id = package_blob.id").
+		Where("package_file.id IS NULL").
 		SumInt(&PackageBlob{}, "size")
 }

models/packages/package_file.go

@@ -199,9 +199,9 @@ func SearchFiles(ctx context.Context, opts *PackageFileSearchOptions) ([]*Packag
 	return pfs, count, err
 }
 
-// CalculateBlobSize sums up all blob sizes matching the search options.
+// CalculateFileSize sums up all blob sizes matching the search options.
 // It does NOT respect the deduplication of blobs.
-func CalculateBlobSize(ctx context.Context, opts *PackageFileSearchOptions) (int64, error) {
+func CalculateFileSize(ctx context.Context, opts *PackageFileSearchOptions) (int64, error) {
 	return db.GetEngine(ctx).
 		Table("package_file").
 		Where(opts.toConds()).

models/user/user.go

@@ -1233,7 +1233,10 @@ func GetUserByOpenID(uri string) (*User, error) {
 // GetAdminUser returns the first administrator
 func GetAdminUser() (*User, error) {
 	var admin User
-	has, err := db.GetEngine(db.DefaultContext).Where("is_admin=?", true).Get(&admin)
+	has, err := db.GetEngine(db.DefaultContext).
+		Where("is_admin=?", true).
+		Asc("id"). // Reliably get the admin with the lowest ID.
+		Get(&admin)
 	if err != nil {
 		return nil, err
 	} else if !has {

modules/migration/comment.go

@@ -17,6 +17,7 @@ type Commentable interface {
 type Comment struct {
 	IssueIndex  int64 `yaml:"issue_index"`
 	Index       int64
+	CommentType string `yaml:"comment_type"` // see `commentStrings` in models/issues/comment.go
 	PosterID    int64  `yaml:"poster_id"`
 	PosterName  string `yaml:"poster_name"`
 	PosterEmail string `yaml:"poster_email"`
@@ -24,6 +25,7 @@ type Comment struct {
 	Updated     time.Time
 	Content     string
 	Reactions   []*Reaction
+	Meta        map[string]interface{} `yaml:"meta,omitempty"` // see models/issues/comment.go for fields in Comment struct
 }
 
 // GetExternalName ExternalUserMigrated interface

options/locale/locale_en-US.ini

@@ -2645,6 +2645,7 @@ repos.size = Size
 
 packages.package_manage_panel = Package Management
 packages.total_size = Total Size: %s
+packages.unreferenced_size = Unreferenced Size: %s
 packages.owner = Owner
 packages.creator = Creator
 packages.name = Name

routers/web/admin/packages.go

@@ -51,12 +51,18 @@ func Packages(ctx *context.Context) {
 		return
 	}
 
-	totalBlobSize, err := packages_model.GetTotalBlobSize()
+	totalBlobSize, err := packages_model.GetTotalBlobSize(ctx)
 	if err != nil {
 		ctx.ServerError("GetTotalBlobSize", err)
 		return
 	}
 
+	totalUnreferencedBlobSize, err := packages_model.GetTotalUnreferencedBlobSize(ctx)
+	if err != nil {
+		ctx.ServerError("CalculateBlobSize", err)
+		return
+	}
+
 	ctx.Data["Title"] = ctx.Tr("packages.title")
 	ctx.Data["PageIsAdmin"] = true
 	ctx.Data["PageIsAdminPackages"] = true
@@ -65,8 +71,9 @@ func Packages(ctx *context.Context) {
 	ctx.Data["AvailableTypes"] = packages_model.TypeList
 	ctx.Data["SortType"] = sort
 	ctx.Data["PackageDescriptors"] = pds
-	ctx.Data["Total"] = total
+	ctx.Data["TotalCount"] = total
-	ctx.Data["TotalBlobSize"] = totalBlobSize
+	ctx.Data["TotalBlobSize"] = totalBlobSize - totalUnreferencedBlobSize
+	ctx.Data["TotalUnreferencedBlobSize"] = totalUnreferencedBlobSize
 
 	pager := context.NewPagination(int(total), setting.UI.PackagesPagingNum, page, 5)
 	pager.AddParamString("q", query)

routers/web/repo/issue.go

@@ -784,7 +784,8 @@ func setTemplateIfExists(ctx *context.Context, ctxDataKey string, possibleFiles
 			}
 
 		}
-		if !strings.HasPrefix(template.Ref, "refs/") { // Assume that the ref intended is always a branch - for tags users should use refs/tags/<ref>
+
+		if template.Ref != "" && !strings.HasPrefix(template.Ref, "refs/") { // Assume that the ref intended is always a branch - for tags users should use refs/tags/<ref>
 			template.Ref = git.BranchPrefix + template.Ref
 		}
 		ctx.Data["HasSelectedLabel"] = len(labelIDs) > 0

services/issue/commit.go

@@ -18,6 +18,7 @@ import (
 	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/container"
+	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/references"
 	"code.gitea.io/gitea/modules/repository"
@@ -175,7 +176,8 @@ func UpdateIssuesCommit(doer *user_model.User, repo *repo_model.Repository, comm
 			if !repo.CloseIssuesViaCommitInAnyBranch {
 				// If the issue was specified to be in a particular branch, don't allow commits in other branches to close it
 				if refIssue.Ref != "" {
-					if branchName != refIssue.Ref {
+					issueBranchName := strings.TrimPrefix(refIssue.Ref, git.BranchPrefix)
+					if branchName != issueBranchName {
 						continue
 					}
 					// Otherwise, only process commits to the default branch

services/migrations/gitea_uploader.go

@@ -454,15 +454,34 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
 		if comment.Updated.IsZero() {
 			comment.Updated = comment.Created
 		}
-
+		if comment.CommentType == "" {
+			// if type field is missing, then assume a normal comment
+			comment.CommentType = issues_model.CommentTypeComment.String()
+		}
 		cm := issues_model.Comment{
 			IssueID:     issue.ID,
-			Type:        issues_model.CommentTypeComment,
+			Type:        issues_model.AsCommentType(comment.CommentType),
 			Content:     comment.Content,
 			CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()),
 			UpdatedUnix: timeutil.TimeStamp(comment.Updated.Unix()),
 		}
 
+		switch cm.Type {
+		case issues_model.CommentTypeAssignees:
+			cm.AssigneeID = comment.Meta["AssigneeID"].(int64)
+			if comment.Meta["RemovedAssigneeID"] != nil {
+				cm.RemovedAssignee = true
+			}
+		case issues_model.CommentTypeChangeTitle:
+			if comment.Meta["OldTitle"] != nil {
+				cm.OldTitle = fmt.Sprintf("%s", comment.Meta["OldTitle"])
+			}
+			if comment.Meta["NewTitle"] != nil {
+				cm.NewTitle = fmt.Sprintf("%s", comment.Meta["NewTitle"])
+			}
+		default:
+		}
+
 		if err := g.remapUser(comment, &cm); err != nil {
 			return err
 		}

services/packages/packages.go

@@ -361,11 +361,11 @@ func checkSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p
 	}
 
 	if setting.Packages.LimitTotalOwnerSize > -1 {
-		totalSize, err := packages_model.CalculateBlobSize(ctx, &packages_model.PackageFileSearchOptions{
+		totalSize, err := packages_model.CalculateFileSize(ctx, &packages_model.PackageFileSearchOptions{
 			OwnerID: owner.ID,
 		})
 		if err != nil {
-			log.Error("CalculateBlobSize failed: %v", err)
+			log.Error("CalculateFileSize failed: %v", err)
 			return err
 		}
 		if totalSize+uploadSize > setting.Packages.LimitTotalOwnerSize {

services/pull/update.go

@@ -118,6 +118,9 @@ func IsUserAllowedToUpdate(ctx context.Context, pull *issues_model.PullRequest,
 		}
 		prUnit, err := pr.BaseRepo.GetUnit(ctx, unit.TypePullRequests)
 		if err != nil {
+			if repo_model.IsErrUnitTypeNotExist(err) {
+				return false, false, nil
+			}
 			log.Error("pr.BaseRepo.GetUnit(unit.TypePullRequests): %v", err)
 			return false, false, err
 		}

templates/admin/packages/list.tmpl

@@ -4,7 +4,9 @@
 	<div class="ui container">
 		{{template "base/alert" .}}
 		<h4 class="ui top attached header">
-			{{.locale.Tr "admin.packages.package_manage_panel"}} ({{.locale.Tr "admin.total" .Total}}, {{.locale.Tr "admin.packages.total_size" (FileSize .TotalBlobSize)}})
+			{{.locale.Tr "admin.packages.package_manage_panel"}} ({{.locale.Tr "admin.total" .TotalCount}},
+			{{.locale.Tr "admin.packages.total_size" (FileSize .TotalBlobSize)}},
+			{{.locale.Tr "admin.packages.unreferenced_size" (FileSize .TotalUnreferencedBlobSize)}})
 		</h4>
 		<div class="ui attached segment">
 			<form class="ui form ignore-dirty">

web_src/js/markup/asciicast.js

@@ -2,7 +2,10 @@ export async function renderAsciinemaPlayer() {
   const els = document.querySelectorAll('.asciinema-player-container');
   if (!els.length) return;
 
-  const player = await import(/* webpackChunkName: "asciinema-player" */'asciinema-player');
+  const [player] = await Promise.all([
+    import(/* webpackChunkName: "asciinema-player" */'asciinema-player'),
+    import(/* webpackChunkName: "asciinema-player" */'asciinema-player/dist/bundle/asciinema-player.css'),
+  ]);
 
   for (const el of els) {
     player.create(el.getAttribute('data-asciinema-player-src'), el, {

web_src/less/markup/asciicast.less

@@ -1,5 +1,3 @@
-@import "../asciinema-player/dist/bundle/asciinema-player.css";
-
 .asciinema-player-container {
   width: 100%;
   height: auto;