[F3] GetLocalMatchingRemote for user

(cherry picked from commit e73cb837f5)
(cherry picked from commit a24bc0b85e)

.golangci.yml

@@ -106,9 +106,9 @@ issues:
         - gosec
         - unparam
         - staticcheck
-    - path: services/f3/driver/driver.go
+    - path: services/f3/driver/base.go
       linters:
-        - typecheck
+        - gosimple
     - path: models/migrations/v
       linters:
         - gocyclo

cmd/f3.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"fmt"
 
+	auth_model "code.gitea.io/gitea/models/auth"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/services/f3/util"
@@ -37,6 +38,11 @@ var CmdF3 = cli.Command{
 			Value: "",
 			Usage: "The name of the repository",
 		},
+		cli.StringFlag{
+			Name:  "authentication-source",
+			Value: "",
+			Usage: "The name of the authentication source matching the forge of origin",
+		},
 		cli.BoolFlag{
 			Name:  "no-pull-request",
 			Usage: "Do not dump pull requests",
@@ -67,6 +73,17 @@ func runF3(ctx *cli.Context) error {
 	return RunF3(stdCtx, ctx)
 }
 
+func getAuthenticationSource(ctx context.Context, authenticationSource string) (*auth_model.Source, error) {
+	source, err := auth_model.GetSourceByName(ctx, authenticationSource)
+	if err != nil {
+		if auth_model.IsErrSourceNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	return source, nil
+}
+
 func RunF3(stdCtx context.Context, ctx *cli.Context) error {
 	doer, err := user_model.GetAdminUser()
 	if err != nil {
@@ -78,7 +95,17 @@ func RunF3(stdCtx context.Context, ctx *cli.Context) error {
 		features.PullRequests = false
 	}
 
-	forgejo := util.ForgejoForgeRoot(features, doer)
+	var sourceID int64
+	sourceName := ctx.String("authentication-source")
+	source, err := getAuthenticationSource(stdCtx, sourceName)
+	if err != nil {
+		return fmt.Errorf("error retrieving the authentication-source %s %v", sourceName, err)
+	}
+	if source != nil {
+		sourceID = source.ID
+	}
+
+	forgejo := util.ForgejoForgeRoot(features, doer, sourceID)
 	f3 := util.F3ForgeRoot(features, ctx.String("directory"))
 
 	if ctx.Bool("export") {

go.mod

@@ -119,7 +119,7 @@ require (
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 	gopkg.in/ini.v1 v1.67.0
 	gopkg.in/yaml.v3 v3.0.1
-	lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230601123105-50a6e740ac04
+	lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230701182935-ce3394d54c1e
 	mvdan.cc/xurls/v2 v2.4.0
 	strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
 	xorm.io/builder v0.3.12

go.sum

@@ -1806,8 +1806,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230601123105-50a6e740ac04 h1:JdNHyMEVNixsOvNw3XqrkWi/RqVLN+wjrdeL6NVk2jE=
+lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230701182935-ce3394d54c1e h1:dcD+UGLSrgeHYyEWZ+1mZvxQ2BXKyEhjhHcB7a6XgeA=
-lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230601123105-50a6e740ac04/go.mod h1:yIlQydnn+pym6OH20iQ7fbe2TjLfnlOTtEOqvjFaC70=
+lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230701182935-ce3394d54c1e/go.mod h1:yIlQydnn+pym6OH20iQ7fbe2TjLfnlOTtEOqvjFaC70=
 lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU=
 lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 modernc.org/cc/v3 v3.33.6/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=

services/f3/driver/driver.go

@@ -18,7 +18,8 @@ import (
 type Options struct {
 	gof3.Options
 
-	Doer *user_model.User
+	AuthenticationSource int64
+	Doer                 *user_model.User
 }
 
 type Forgejo struct {
@@ -59,6 +60,10 @@ func (o *Forgejo) GetDoer() *user_model.User {
 	return o.options.Doer
 }
 
+func (o *Forgejo) GetAuthenticationSource() int64 {
+	return o.options.AuthenticationSource
+}
+
 func (o *Forgejo) GetNewMigrationHTTPClient() gof3.NewMigrationHTTPClientFun {
 	return migrations.NewMigrationHTTPClient
 }

services/f3/driver/user.go

@@ -6,11 +6,13 @@ import (
 	"context"
 	"fmt"
 
+	auth_model "code.gitea.io/gitea/models/auth"
 	"code.gitea.io/gitea/models/db"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/util"
 	user_service "code.gitea.io/gitea/services/user"
 
+	"lab.forgefriends.org/friendlyforgeformat/gof3/forges/common"
 	"lab.forgefriends.org/friendlyforgeformat/gof3/format"
 	f3_util "lab.forgefriends.org/friendlyforgeformat/gof3/util"
 )
@@ -46,7 +48,7 @@ func (o *User) IsNil() bool {
 }
 
 func (o *User) Equals(other *User) bool {
-	return (o.Name == other.Name)
+	return (o.ID == other.ID)
 }
 
 func (o *User) ToFormatInterface() format.Interface {
@@ -79,6 +81,36 @@ type UserProvider struct {
 	BaseProvider
 }
 
+func getLocalMatchingRemote(ctx context.Context, authenticationSource int64, id string) *user_model.User {
+	u := &user_model.User{
+		LoginName:   id,
+		LoginSource: authenticationSource,
+		LoginType:   auth_model.OAuth2,
+		Type:        user_model.UserTypeIndividual,
+	}
+	has, err := db.GetEngine(ctx).Get(u)
+	if err != nil {
+		panic(err)
+	} else if !has {
+		return nil
+	}
+	return u
+}
+
+func (o *UserProvider) GetLocalMatchingRemote(ctx context.Context, format format.Interface, parents ...common.ContainerObjectInterface) (string, bool) {
+	authenticationSource := o.g.GetAuthenticationSource()
+	if authenticationSource == 0 {
+		return "", false
+	}
+	user := getLocalMatchingRemote(ctx, authenticationSource, format.GetIDString())
+	if user != nil {
+		o.g.GetLogger().Debug("found existing user %d with a matching authentication source for %s", user.ID, format.GetIDString())
+		return fmt.Sprintf("%d", user.ID), true
+	}
+	o.g.GetLogger().Debug("no pre-existing local user for %s", format.GetIDString())
+	return "", false
+}
+
 func (o *UserProvider) ToFormat(ctx context.Context, user *User) *format.User {
 	return user.ToFormat()
 }
@@ -105,19 +137,21 @@ func (o *UserProvider) ProcessObject(ctx context.Context, user *User) {
 }
 
 func (o *UserProvider) Get(ctx context.Context, exemplar *User) *User {
+	o.g.GetLogger().Debug("%+v", exemplar)
 	var user *user_model.User
 	var err error
 	if exemplar.GetID() > 0 {
 		user, err = user_model.GetUserByID(ctx, exemplar.GetID())
+		o.g.GetLogger().Debug("GetUserByID: %+v %v", user, err)
 	} else if exemplar.Name != "" {
 		user, err = user_model.GetUserByName(ctx, exemplar.Name)
 	} else {
 		panic("GetID() == 0 and UserName == \"\"")
 	}
-	if user_model.IsErrUserNotExist(err) {
-		return &User{}
-	}
 	if err != nil {
+		if user_model.IsErrUserNotExist(err) {
+			return &User{}
+		}
 		panic(fmt.Errorf("user %v %w", exemplar, err))
 	}
 	return UserConverter(user)
@@ -127,7 +161,12 @@ func (o *UserProvider) Put(ctx context.Context, user *User) *User {
 	overwriteDefault := &user_model.CreateUserOverwriteOptions{
 		IsActive: util.OptionalBoolTrue,
 	}
-	u := user.User
+	u := user_model.User{
+		Name:     user.Name,
+		FullName: user.FullName,
+		Email:    user.Email,
+		Passwd:   user.Passwd,
+	}
 	err := user_model.CreateUser(&u, overwriteDefault)
 	if err != nil {
 		panic(err)

services/f3/util/util.go

@@ -15,7 +15,9 @@ import (
 
 func ToF3Logger(messenger base.Messenger) gof3.Logger {
 	if messenger == nil {
-		messenger = func(string, ...interface{}) {}
+		messenger = func(message string, args ...interface{}) {
+			log.Info("Message: "+message, args...)
+		}
 	}
 	return gof3.Logger{
 		Message:  messenger,
@@ -29,13 +31,14 @@ func ToF3Logger(messenger base.Messenger) gof3.Logger {
 	}
 }
 
-func ForgejoForgeRoot(features gof3.Features, doer *user_model.User) *f3_forges.ForgeRoot {
+func ForgejoForgeRoot(features gof3.Features, doer *user_model.User, authenticationSource int64) *f3_forges.ForgeRoot {
 	forgeRoot := f3_forges.NewForgeRootFromDriver(&driver.Forgejo{}, &driver.Options{
 		Options: gof3.Options{
 			Features: features,
 			Logger:   ToF3Logger(nil),
 		},
-		Doer: doer,
+		Doer:                 doer,
+		AuthenticationSource: authenticationSource,
 	})
 	return forgeRoot
 }

tests/integration/f3_test.go

@@ -12,6 +12,7 @@ import (
 	auth_model "code.gitea.io/gitea/models/auth"
 	"code.gitea.io/gitea/models/unittest"
 	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/test"
 	"code.gitea.io/gitea/services/f3/util"
@@ -69,7 +70,7 @@ func TestF3(t *testing.T) {
 		fixture.NewAsset()
 		fixture.NewIssueComment(nil)
 		fixture.NewPullRequestComment()
-		fixture.NewReview()
+		// fixture.NewReview()
 		fixture.NewIssueReaction()
 		fixture.NewCommentReaction()
 
@@ -78,7 +79,7 @@ func TestF3(t *testing.T) {
 		//
 		doer, err := user_model.GetAdminUser()
 		assert.NoError(t, err)
-		forgejoLocal := util.ForgejoForgeRoot(gof3.AllFeatures, doer)
+		forgejoLocal := util.ForgejoForgeRoot(gof3.AllFeatures, doer, 0)
 		options := f3_common.NewMirrorOptionsRecurse()
 		forgejoLocal.Forge.Mirror(context.Background(), fixture.Forge, options)
 
@@ -117,7 +118,7 @@ func TestF3(t *testing.T) {
 		assert.Contains(t, files, "/release/")
 		assert.Contains(t, files, "/asset/")
 		assert.Contains(t, files, "/comment/")
-		assert.Contains(t, files, "/review/")
+		//		assert.Contains(t, files, "/review/")
 		assert.Contains(t, files, "/reaction/")
 		//		f3_util.Command(context.Background(), "cp", "-a", f3.GetDirectory(), "abc")
 	})
@@ -179,3 +180,91 @@ func TestMaybePromoteF3User(t *testing.T) {
 	assert.Equal(t, userBeforeSignIn.Email, "")
 	assert.Equal(t, userAfterSignIn.Email, gitlabEmail)
 }
+
+func TestF3UserMapping(t *testing.T) {
+	onGiteaRun(t, func(t *testing.T, u *url.URL) {
+		AllowLocalNetworks := setting.Migrations.AllowLocalNetworks
+		setting.F3.Enabled = true
+		setting.Migrations.AllowLocalNetworks = true
+		AppVer := setting.AppVer
+		// Gitea SDK (go-sdk) need to parse the AppVer from server response, so we must set it to a valid version string.
+		setting.AppVer = "1.16.0"
+		defer func() {
+			setting.Migrations.AllowLocalNetworks = AllowLocalNetworks
+			setting.AppVer = AppVer
+		}()
+
+		log.Debug("Step 1: create a fixture")
+		fixtureNewF3Forge := func(t f3_tests.TestingT, user *format.User, tmpDir string) *f3_forges.ForgeRoot {
+			root := f3_forges.NewForgeRoot(&f3_f3.Options{
+				Options: gof3.Options{
+					Configuration: gof3.Configuration{
+						Directory: tmpDir,
+					},
+					Features: gof3.AllFeatures,
+					Logger:   util.ToF3Logger(nil),
+				},
+				Remap: true,
+			})
+			return root
+		}
+		fixture := f3_forges.NewFixture(t, f3_forges.FixtureForgeFactory{Fun: fixtureNewF3Forge, AdminRequired: false})
+		userID := int64(5432)
+		fixture.NewUser(userID)
+		//		fixture.NewProject()
+
+		log.Debug("Step 2: mirror the fixture into Forgejo")
+		//
+		// OAuth2 authentication source GitLab
+		//
+		gitlabName := "gitlab"
+		gitlab := addAuthSource(t, authSourcePayloadGitLabCustom(gitlabName))
+		//
+		// Create a user as if it had been previously been created by the F3
+		// authentication source.
+		//
+		gitlabUserID := fmt.Sprintf("%d", userID)
+		gitlabUser := &user_model.User{
+			Name:        "gitlabuser",
+			Email:       "gitlabuser@example.com",
+			LoginType:   auth_model.OAuth2,
+			LoginSource: gitlab.ID,
+			LoginName:   gitlabUserID,
+		}
+		defer createUser(context.Background(), t, gitlabUser)()
+
+		doer, err := user_model.GetAdminUser()
+		assert.NoError(t, err)
+		forgejoLocal := util.ForgejoForgeRoot(gof3.AllFeatures, doer, gitlab.ID)
+		options := f3_common.NewMirrorOptionsRecurse()
+		forgejoLocal.Forge.Mirror(context.Background(), fixture.Forge, options)
+
+		log.Debug("Step 3: mirror Forgejo into F3")
+		adminUsername := "user1"
+		forgejoAPI := f3_forges.NewForgeRootFromDriver(&f3_forgejo.Forgejo{}, &f3_forgejo.Options{
+			Options: gof3.Options{
+				Configuration: gof3.Configuration{
+					URL:       setting.AppURL,
+					Directory: t.TempDir(),
+				},
+				Features: gof3.AllFeatures,
+				Logger:   util.ToF3Logger(nil),
+			},
+			AuthToken: getUserToken(t, adminUsername, auth_model.AccessTokenScopeWriteAdmin, auth_model.AccessTokenScopeAll),
+		})
+
+		f3 := f3_forges.FixtureNewF3Forge(t, nil, t.TempDir())
+		apiForge := forgejoAPI.Forge
+		apiUser := apiForge.Users.GetFromFormat(context.Background(), &format.User{UserName: gitlabUser.Name})
+		//		apiProject := apiUser.Projects.GetFromFormat(context.Background(), &format.Project{Name: fixture.ProjectFormat.Name})
+		// options = f3_common.NewMirrorOptionsRecurse(apiUser, apiProject)
+		options = f3_common.NewMirrorOptionsRecurse(apiUser)
+		f3.Forge.Mirror(context.Background(), apiForge, options)
+
+		//
+		// Step 4: verify the fixture and F3 are equivalent
+		//
+		files := f3_util.Command(context.Background(), "find", f3.GetDirectory())
+		assert.Contains(t, files, fmt.Sprintf("/user/%d", gitlabUser.ID))
+	})
+}