Fix migration failing when importing either issues or PRs but not the other (#8892)

Related to https://codeberg.org/Codeberg/Community/issues/1944 * Allowed the githubdownloaderv3 to know whether issues and, or PRs are requested to migrate * Used this information to decide to filter for "/pulls/" or "/issues" * Or not to filter at all if issues == true && prs == true * Added isolated test for the downloader and for the uploader * Created a new test_repo in github.com/forgejo and set it up properly together with @Gusted * Updated github_downloader_test with the new URLs and test data from the repo * Recorded the API calls for local testing * Added a minimal gitbucket test (which uses the github downloader under the hood) Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/8892 Reviewed-by: Gusted <gusted@noreply.codeberg.org> Co-authored-by: patdyn <patdyn@noreply.codeberg.org> Co-committed-by: patdyn <patdyn@noreply.codeberg.org>
2025-09-29 22:25:59 +00:00 · 2025-09-01 14:05:10 +02:00 · 2025-09-01 14:05:10 +02:00 · 9a423c0e67
commit 9a423c0e67
parent ed3b70cbb9
70 changed files with 966 additions and 980 deletions
--- a/services/migrations/gitea_uploader_test.go
+++ b/services/migrations/gitea_uploader_test.go
@ -19,7 +19,6 @@ import (
 	user_model "forgejo.org/models/user"
 	"forgejo.org/modules/git"
 	"forgejo.org/modules/gitrepo"
-	"forgejo.org/modules/graceful"
 	"forgejo.org/modules/log"
 	base "forgejo.org/modules/migration"
 	"forgejo.org/modules/optional"
@ -30,6 +29,130 @@ import (
 	"github.com/stretchr/testify/require"
 )

+func TestCommentUpload(t *testing.T) {
+	unittest.PrepareTestEnv(t)
+	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1})
+	var (
+		opts = base.MigrateOptions{
+			Issues: true,
+		}
+		repoName = "test_repo"
+		uploader = NewGiteaLocalUploader(t.Context(), user, user.Name, repoName)
+	)
+	defer uploader.Close()
+
+	fixturePath := "./testdata/github/full_download"
+	server := unittest.NewMockWebServer(t, "https://api.github.com", fixturePath, false)
+	defer server.Close()
+
+	// Mock Data
+	repoMock := &base.Repository{
+		Name:          repoName,
+		Owner:         "forgejo",
+		Description:   "Some mock repo",
+		CloneURL:      server.URL + "/forgejo/test_repo.git",
+		OriginalURL:   server.URL + "/forgejo/test_repo",
+		DefaultBranch: "master",
+		Website:       "https://codeberg.org/forgejo/forgejo/",
+	}
+
+	// Create Repo
+	require.NoError(t, uploader.CreateRepo(repoMock, opts))
+
+	repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerID: user.ID, Name: repoName})
+
+	// Create and Test Issues Uploading
+	issueA := &base.Issue{
+		Title:        "First issue",
+		Number:       0,
+		PosterID:     37243484,
+		PosterName:   "PatDyn",
+		PosterEmail:  "",
+		Content:      "Mock Content",
+		Milestone:    "Mock Milestone",
+		State:        "open",
+		Created:      time.Date(2025, 8, 7, 12, 44, 7, 0, time.UTC),
+		Updated:      time.Date(2025, 8, 7, 12, 44, 47, 0, time.UTC),
+		Labels:       nil,
+		Reactions:    nil,
+		Closed:       nil,
+		IsLocked:     false,
+		Assignees:    nil,
+		ForeignIndex: 0,
+	}
+
+	issueB := &base.Issue{
+		Title:        "Second Issue",
+		Number:       1,
+		PosterID:     37243484,
+		PosterName:   "PatDyn",
+		PosterEmail:  "",
+		Content:      "Mock Content",
+		Milestone:    "Mock Milestone",
+		State:        "open",
+		Created:      time.Date(2025, 8, 7, 12, 45, 44, 0, time.UTC),
+		Updated:      time.Date(2025, 8, 7, 13, 7, 25, 0, time.UTC),
+		Labels:       nil,
+		Reactions:    nil,
+		Closed:       nil,
+		IsLocked:     false,
+		Assignees:    nil,
+		ForeignIndex: 1,
+	}
+
+	err := uploader.CreateIssues(issueA, issueB)
+	require.NoError(t, err)
+
+	issues, err := issues_model.Issues(db.DefaultContext, &issues_model.IssuesOptions{
+		RepoIDs:  []int64{repo.ID},
+		IsPull:   optional.Some(false),
+		SortType: "newest",
+	})
+	require.NoError(t, err)
+	assert.Len(t, issues, 2)
+
+	// Create and Test Comment Uploading
+	issueAComment := &base.Comment{
+		IssueIndex:  0,
+		Index:       0,
+		CommentType: "comment",
+		PosterID:    37243484,
+		PosterName:  "PatDyn",
+		PosterEmail: "",
+		Created:     time.Date(2025, 8, 7, 12, 44, 24, 0, time.UTC),
+		Updated:     time.Date(2025, 8, 7, 12, 44, 24, 0, time.UTC),
+		Content:     "First Mock Comment",
+		Reactions:   nil,
+		Meta:        nil,
+	}
+	issueBComment := &base.Comment{
+		IssueIndex:  1,
+		Index:       1,
+		CommentType: "comment",
+		PosterID:    37243484,
+		PosterName:  "PatDyn",
+		PosterEmail: "",
+		Created:     time.Date(2025, 8, 7, 13, 7, 25, 0, time.UTC),
+		Updated:     time.Date(2025, 8, 7, 13, 7, 25, 0, time.UTC),
+		Content:     "Second Mock Comment",
+		Reactions:   nil,
+		Meta:        nil,
+	}
+	require.NoError(t, uploader.CreateComments(issueBComment, issueAComment))
+
+	issues, err = issues_model.Issues(db.DefaultContext, &issues_model.IssuesOptions{
+		RepoIDs:  []int64{repo.ID},
+		IsPull:   optional.Some(false),
+		SortType: "newest",
+	})
+	require.NoError(t, err)
+	assert.Len(t, issues, 2)
+	require.NoError(t, issues[0].LoadDiscussComments(db.DefaultContext))
+	require.NoError(t, issues[1].LoadDiscussComments(db.DefaultContext))
+	assert.Len(t, issues[0].Comments, 1)
+	assert.Len(t, issues[1].Comments, 1)
+}
+
 func TestGiteaUploadRepo(t *testing.T) {
 	// FIXME: Since no accesskey or user/password will trigger rate limit of github, just skip
 	t.Skip()
@ -40,9 +163,9 @@ func TestGiteaUploadRepo(t *testing.T) {

 	var (
 		ctx        = t.Context()
-		downloader = NewGithubDownloaderV3(ctx, "https://github.com", "", "", "", "go-xorm", "builder")
+		downloader = NewGithubDownloaderV3(ctx, "https://github.com", true, true, "", "", "", "go-xorm", "builder")
 		repoName   = "builder-" + time.Now().Format("2006-01-02-15-04-05")
-		uploader   = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
+		uploader   = NewGiteaLocalUploader(t.Context(), user, user.Name, repoName)
 	)

 	err := migrateRepository(db.DefaultContext, user, downloader, uploader, base.MigrateOptions{