diff --git a/modules/git/attribute/attribute.go b/modules/git/attribute/attribute.go index adf323ef41c05..ee4ac7802e54f 100644 --- a/modules/git/attribute/attribute.go +++ b/modules/git/attribute/attribute.go @@ -88,6 +88,10 @@ func (attrs *Attributes) GetLinguistLanguage() optional.Option[string] { return attrs.Get(LinguistLanguage).ToString() } +func (attrs *Attributes) MatchLFS() bool { + return attrs.Get(Filter).ToString().Value() == "lfs" +} + func (attrs *Attributes) GetGitlabLanguage() optional.Option[string] { attrStr := attrs.Get(GitlabLanguage).ToString() if attrStr.Has() { diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index f40d39a2517f6..6083b88a78e22 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -485,26 +485,18 @@ func ChangeFiles(ctx *context.APIContext) { Message: apiOpts.Message, OldBranch: apiOpts.BranchName, NewBranch: apiOpts.NewBranchName, - Committer: &files_service.IdentityOptions{ - GitUserName: apiOpts.Committer.Name, - GitUserEmail: apiOpts.Committer.Email, + Committer: &git.Signature{ + Name: apiOpts.Committer.Name, + Email: apiOpts.Committer.Email, + When: apiOpts.Dates.Committer, }, - Author: &files_service.IdentityOptions{ - GitUserName: apiOpts.Author.Name, - GitUserEmail: apiOpts.Author.Email, - }, - Dates: &files_service.CommitDateOptions{ - Author: apiOpts.Dates.Author, - Committer: apiOpts.Dates.Committer, + Author: &git.Signature{ + Name: apiOpts.Author.Name, + Email: apiOpts.Author.Email, + When: apiOpts.Dates.Author, }, Signoff: apiOpts.Signoff, } - if opts.Dates.Author.IsZero() { - opts.Dates.Author = time.Now() - } - if opts.Dates.Committer.IsZero() { - opts.Dates.Committer = time.Now() - } if opts.Message == "" { opts.Message = changeFilesCommitMessage(ctx, files) @@ -582,26 +574,18 @@ func CreateFile(ctx *context.APIContext) { Message: apiOpts.Message, OldBranch: apiOpts.BranchName, NewBranch: apiOpts.NewBranchName, - Committer: &files_service.IdentityOptions{ - GitUserName: apiOpts.Committer.Name, - GitUserEmail: apiOpts.Committer.Email, + Committer: &git.Signature{ + Name: apiOpts.Committer.Name, + Email: apiOpts.Committer.Email, + When: apiOpts.Dates.Committer, }, - Author: &files_service.IdentityOptions{ - GitUserName: apiOpts.Author.Name, - GitUserEmail: apiOpts.Author.Email, - }, - Dates: &files_service.CommitDateOptions{ - Author: apiOpts.Dates.Author, - Committer: apiOpts.Dates.Committer, + Author: &git.Signature{ + Name: apiOpts.Author.Name, + Email: apiOpts.Author.Email, + When: apiOpts.Dates.Author, }, Signoff: apiOpts.Signoff, } - if opts.Dates.Author.IsZero() { - opts.Dates.Author = time.Now() - } - if opts.Dates.Committer.IsZero() { - opts.Dates.Committer = time.Now() - } if opts.Message == "" { opts.Message = changeFilesCommitMessage(ctx, opts.Files) @@ -685,26 +669,18 @@ func UpdateFile(ctx *context.APIContext) { Message: apiOpts.Message, OldBranch: apiOpts.BranchName, NewBranch: apiOpts.NewBranchName, - Committer: &files_service.IdentityOptions{ - GitUserName: apiOpts.Committer.Name, - GitUserEmail: apiOpts.Committer.Email, + Committer: &git.Signature{ + Name: apiOpts.Committer.Name, + Email: apiOpts.Committer.Email, + When: apiOpts.Dates.Committer, }, - Author: &files_service.IdentityOptions{ - GitUserName: apiOpts.Author.Name, - GitUserEmail: apiOpts.Author.Email, - }, - Dates: &files_service.CommitDateOptions{ - Author: apiOpts.Dates.Author, - Committer: apiOpts.Dates.Committer, + Author: &git.Signature{ + Name: apiOpts.Author.Name, + Email: apiOpts.Author.Email, + When: apiOpts.Dates.Author, }, Signoff: apiOpts.Signoff, } - if opts.Dates.Author.IsZero() { - opts.Dates.Author = time.Now() - } - if opts.Dates.Committer.IsZero() { - opts.Dates.Committer = time.Now() - } if opts.Message == "" { opts.Message = changeFilesCommitMessage(ctx, opts.Files) @@ -844,26 +820,18 @@ func DeleteFile(ctx *context.APIContext) { Message: apiOpts.Message, OldBranch: apiOpts.BranchName, NewBranch: apiOpts.NewBranchName, - Committer: &files_service.IdentityOptions{ - GitUserName: apiOpts.Committer.Name, - GitUserEmail: apiOpts.Committer.Email, + Committer: &git.Signature{ + Name: apiOpts.Committer.Name, + Email: apiOpts.Committer.Email, + When: apiOpts.Dates.Committer, }, - Author: &files_service.IdentityOptions{ - GitUserName: apiOpts.Author.Name, - GitUserEmail: apiOpts.Author.Email, - }, - Dates: &files_service.CommitDateOptions{ - Author: apiOpts.Dates.Author, - Committer: apiOpts.Dates.Committer, + Author: &git.Signature{ + Name: apiOpts.Author.Name, + Email: apiOpts.Author.Email, + When: apiOpts.Dates.Author, }, Signoff: apiOpts.Signoff, } - if opts.Dates.Author.IsZero() { - opts.Dates.Author = time.Now() - } - if opts.Dates.Committer.IsZero() { - opts.Dates.Committer = time.Now() - } if opts.Message == "" { opts.Message = changeFilesCommitMessage(ctx, opts.Files) diff --git a/routers/web/repo/editor.go b/routers/web/repo/editor.go index c925b6115147a..d76e44d3f54a1 100644 --- a/routers/web/repo/editor.go +++ b/routers/web/repo/editor.go @@ -296,9 +296,15 @@ func editFilePost(ctx *context.Context, form forms.EditRepoFileForm, isNewFile b ContentReader: strings.NewReader(strings.ReplaceAll(form.Content, "\r", "")), }, }, - Signoff: form.Signoff, - Author: gitCommitter, - Committer: gitCommitter, + Signoff: form.Signoff, + Author: &git.Signature{ + Name: gitCommitter.GitUserName, + Email: gitCommitter.GitUserEmail, + }, + Committer: &git.Signature{ + Name: gitCommitter.GitUserName, + Email: gitCommitter.GitUserEmail, + }, }); err != nil { // This is where we handle all the errors thrown by files_service.ChangeRepoFiles if git.IsErrNotExist(err) { @@ -512,10 +518,16 @@ func DeleteFilePost(ctx *context.Context) { TreePath: ctx.Repo.TreePath, }, }, - Message: message, - Signoff: form.Signoff, - Author: gitCommitter, - Committer: gitCommitter, + Message: message, + Signoff: form.Signoff, + Author: &git.Signature{ + Name: gitCommitter.GitUserName, + Email: gitCommitter.GitUserEmail, + }, + Committer: &git.Signature{ + Name: gitCommitter.GitUserName, + Email: gitCommitter.GitUserEmail, + }, }); err != nil { // This is where we handle all the errors thrown by repofiles.DeleteRepoFile if git.IsErrNotExist(err) || files_service.IsErrRepoFileDoesNotExist(err) { diff --git a/services/repository/files/content_test.go b/services/repository/files/content_test.go index eb10f5c9b1188..f4361b3246611 100644 --- a/services/repository/files/content_test.go +++ b/services/repository/files/content_test.go @@ -20,10 +20,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestMain(m *testing.M) { - unittest.MainTest(m) -} - func getExpectedReadmeContentsResponse() *api.ContentsResponse { treePath := "README.md" sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" diff --git a/services/repository/files/fast_import.go b/services/repository/files/fast_import.go new file mode 100644 index 0000000000000..b60d9e95064d5 --- /dev/null +++ b/services/repository/files/fast_import.go @@ -0,0 +1,346 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "strings" + "time" + + git_model "code.gitea.io/gitea/models/git" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/git/attribute" + "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/tempdir" + "code.gitea.io/gitea/modules/util" +) + +// IdentityOptions for a person's identity like an author or committer +type IdentityOptions struct { + GitUserName string // to match "git config user.name" + GitUserEmail string // to match "git config user.email" +} + +// CommitDateOptions store dates for GIT_AUTHOR_DATE and GIT_COMMITTER_DATE +type CommitDateOptions struct { + Author time.Time + Committer time.Time +} + +type ChangeRepoFile struct { + Operation string // "create", "update", or "delete" + TreePath string + FromTreePath string + ContentReader io.ReadSeeker + SHA string + Options *RepoFileOptions +} + +// ChangeRepoFilesOptions holds the repository files update options +type ChangeRepoFilesOptions struct { + LastCommitID string + OldBranch string + NewBranch string + Message string + Files []*ChangeRepoFile + Author *git.Signature + Committer *git.Signature + Signer *git.Signature + SignKey string + Signoff bool +} + +type RepoFileOptions struct { + treePath string + fromTreePath string + executable bool +} + +// UpdateRepoBranchWithLFS updates the specified branch in the given repository with the provided file changes. +func UpdateRepoBranchWithLFS(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, gitRepo *git.Repository, opts ChangeRepoFilesOptions) error { + trustCommitter := repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel + repoPath := repo.RepoPath() + if !setting.LFS.StartServer { + return UpdateRepoBranch(ctx, doer, repoPath, trustCommitter, opts) + } + + // handle lfs files + fileNames := make([]string, 0, len(opts.Files)) + for _, file := range opts.Files { + if file.Operation == "create" || file.Operation == "update" { + fileNames = append(fileNames, file.TreePath) + } + } + if len(fileNames) == 0 { + return UpdateRepoBranch(ctx, doer, repoPath, trustCommitter, opts) + } + + attributesMap, err := attribute.CheckAttributes(ctx, gitRepo, "", attribute.CheckAttributeOpts{ + Attributes: []string{attribute.Filter}, + Filenames: fileNames, + }) + if err != nil { + return err + } + + contentStore := lfs.NewContentStore() + + // Upload the files to LFS Store and replace the content reader with the pointer + for _, file := range opts.Files { + if attributesMap[file.TreePath] == nil || !attributesMap[file.TreePath].MatchLFS() { + continue + } + + pointer, err := lfs.GeneratePointer(file.ContentReader) + if err != nil { + return err + } + file.ContentReader.Seek(0, io.SeekStart) + + // upload the file to LFS Store + exist, err := contentStore.Exists(pointer) + if err != nil { + return err + } + if !exist { + // FIXME: Put regenerates the hash and copies the file over. + // I guess this strictly ensures the soundness of the store but this is inefficient. + if err := contentStore.Put(pointer, file.ContentReader); err != nil { + // OK Now we need to cleanup + // Can't clean up the store, once uploaded there they're there. + return err + } + + // add the meta object to the database + lfsMetaObject, err := git_model.NewLFSMetaObject(ctx, repo.ID, pointer) + if err != nil { + // OK Now we need to cleanup + return err + } + defer func() { + if err != nil { + if _, err := git_model.RemoveLFSMetaObjectByOid(ctx, repo.ID, lfsMetaObject.Oid); err != nil { + log.Error("Unable to delete LFS meta object: %v", err) + } + } + }() + } + + // TODO: should the content reader be closed? + file.ContentReader = bytes.NewReader([]byte(pointer.StringContent())) + } + + return UpdateRepoBranch(ctx, doer, repo.RepoPath(), trustCommitter, opts) +} + +// UpdateRepoBranch updates the specified branch in the given repository with the provided file changes. +// It uses the fast-import command to perform the update efficiently. So that we can avoid to clone the whole repo. +func UpdateRepoBranch(ctx context.Context, doer *user_model.User, repoPath string, trustCommitter bool, opts ChangeRepoFilesOptions) error { + fPath, cancel, err := generateFastImportFile(ctx, doer, repoPath, trustCommitter, opts) + if err != nil { + return err + } + defer cancel() + + f, err := os.Open(fPath) + if err != nil { + return err + } + defer f.Close() + + _, _, err = git.NewCommand("fast-import"). + RunStdString(ctx, &git.RunOpts{ + Stdin: f, + Dir: repoPath, + }) + return err +} + +func getZoneOffsetStr(t time.Time) string { + // Get the timezone offset in hours and minutes + _, offset := t.Zone() + return fmt.Sprintf("%+03d%02d", offset/3600, (offset%3600)/60) +} + +func writeCommit(ctx context.Context, f io.Writer, _ string, _ bool, doer *user_model.User, opts *ChangeRepoFilesOptions) error { + _, err := fmt.Fprintf(f, "commit refs/heads/%s\n", util.Iif(opts.NewBranch != "", opts.NewBranch, opts.OldBranch)) + return err +} + +func writeAuthor(ctx context.Context, f io.Writer, _ string, _ bool, doer *user_model.User, opts *ChangeRepoFilesOptions) error { + _, err := fmt.Fprintf(f, "author %s <%s> %d %s\n", + opts.Author.Name, + opts.Author.Email, + opts.Author.When.Unix(), + getZoneOffsetStr(opts.Author.When)) + return err +} + +func writeCommitter(ctx context.Context, f io.Writer, _ string, _ bool, doer *user_model.User, opts *ChangeRepoFilesOptions) error { + _, err := fmt.Fprintf(f, "committer %s <%s> %d %s\n", + opts.Committer.Name, + opts.Committer.Email, + opts.Committer.When.Unix(), + getZoneOffsetStr(opts.Committer.When)) + return err +} + +func writeMessage(ctx context.Context, f io.Writer, repoPath string, trustCommitter bool, doer *user_model.User, opts *ChangeRepoFilesOptions) error { + messageBytes := new(bytes.Buffer) + if _, err := messageBytes.WriteString(opts.Message); err != nil { + return err + } + + committerSig := opts.Committer + + if opts.Signer != nil { + if trustCommitter { + if opts.Committer.Name != opts.Author.Name || opts.Committer.Email != opts.Author.Email { + // Add trailers + _, _ = messageBytes.WriteString("\n") + _, _ = messageBytes.WriteString("Co-authored-by: ") + _, _ = messageBytes.WriteString(opts.Author.String()) + _, _ = messageBytes.WriteString("\n") + _, _ = messageBytes.WriteString("Co-committed-by: ") + _, _ = messageBytes.WriteString(opts.Committer.String()) + _, _ = messageBytes.WriteString("\n") + } + } + committerSig = opts.Signer + } + + if opts.Signoff { + // Signed-off-by + _, _ = messageBytes.WriteString("\n") + _, _ = messageBytes.WriteString("Signed-off-by: ") + _, _ = messageBytes.WriteString(committerSig.String()) + } + _, err := fmt.Fprintf(f, "data %d\n%s\n", messageBytes.Len()+1, messageBytes.String()) + return err +} + +func writeFrom(ctx context.Context, f io.Writer, _ string, _ bool, doer *user_model.User, opts *ChangeRepoFilesOptions) error { + var fromStatement string + if opts.LastCommitID != "" && opts.LastCommitID != "HEAD" { + fromStatement = fmt.Sprintf("from %s\n", opts.LastCommitID) + } else if opts.OldBranch != "" { + fromStatement = fmt.Sprintf("from refs/heads/%s^0\n", opts.OldBranch) + } // if this is a new branch, so we cannot add from refs/heads/newbranch^0 + + if len(fromStatement) == 0 { + return nil + } + _, err := fmt.Fprint(f, fromStatement) + return err +} + +func writeGPGSign(ctx context.Context, f io.Writer, _ string, _ bool, doer *user_model.User, opts *ChangeRepoFilesOptions) error { + if opts.SignKey == "" { + return nil + } + + // write the GPG signature + if _, err := fmt.Fprintf(f, "gpgsig %s\n", opts.SignKey); err != nil { + return err + } + return nil +} + +// hashObject writes the provided content to the object db and returns its hash +func hashObject(ctx context.Context, repoPath string, content io.Reader) (string, error) { + stdOut := new(bytes.Buffer) + stdErr := new(bytes.Buffer) + + if err := git.NewCommand("hash-object", "-w", "--stdin"). + Run(ctx, &git.RunOpts{ + Dir: repoPath, + Stdin: content, + Stdout: stdOut, + Stderr: stdErr, + }); err != nil { + log.Error("Unable to hash-object to temporary repo: %s Error: %v\nstdout: %s\nstderr: %s", repoPath, err, stdOut.String(), stdErr.String()) + return "", fmt.Errorf("unable to hash-object to temporary repo: %s Error: %w\nstdout: %s\nstderr: %s", repoPath, err, stdOut.String(), stdErr.String()) + } + + return strings.TrimSpace(stdOut.String()), nil +} + +// generateFastImportFile generates a fast-import file based on the provided options. +func generateFastImportFile(ctx context.Context, doer *user_model.User, repoPath string, trustCommitter bool, opts ChangeRepoFilesOptions) (fPath string, cancel func(), err error) { + if opts.OldBranch == "" && opts.NewBranch == "" { + return "", nil, fmt.Errorf("both old and new branches are empty") + } + if opts.OldBranch == opts.NewBranch { + opts.NewBranch = "" + } + + writeFuncs := []func(context.Context, io.Writer, string, bool, *user_model.User, *ChangeRepoFilesOptions) error{ + writeCommit, + writeAuthor, + writeCommitter, + writeGPGSign, + writeMessage, + writeFrom, + } + + f, cancel, err := tempdir.OsTempDir("gitea-fast-import-").CreateTempFileRandom("fast-import-*.txt") + if err != nil { + return "", nil, err + } + defer func() { + if err != nil { + cancel() + } + }() + + for _, writeFunc := range writeFuncs { + if err := writeFunc(ctx, f, repoPath, trustCommitter, doer, &opts); err != nil { + return "", nil, err + } + } + + // Write the file changes to the fast-import file + for _, file := range opts.Files { + switch file.Operation { + case "create", "update": + // delete the old file if it exists + if file.FromTreePath != file.TreePath && file.FromTreePath != "" { + if _, err := fmt.Fprintf(f, "D %s\n", file.FromTreePath); err != nil { + return "", nil, err + } + } + + fileMask := util.Iif(file.Options != nil && file.Options.executable, "100755", "100644") + + // Write the file to objects + objectHash, err := hashObject(ctx, repoPath, file.ContentReader) + if err != nil { + return "", nil, err + } + + if _, err := fmt.Fprintf(f, "M %s %s %s\n", fileMask, objectHash, file.TreePath); err != nil { + return "", nil, err + } + case "delete": + if file.FromTreePath == "" { + return "", nil, fmt.Errorf("delete operation requires FromTreePath") + } + if _, err := fmt.Fprintf(f, "D %s\n", file.FromTreePath); err != nil { + return "", nil, err + } + default: + return "", nil, fmt.Errorf("unknown operation: %s", file.Operation) + } + } + + return f.Name(), cancel, nil +} diff --git a/services/repository/files/fast_import_test.go b/services/repository/files/fast_import_test.go new file mode 100644 index 0000000000000..8243fcc042330 --- /dev/null +++ b/services/repository/files/fast_import_test.go @@ -0,0 +1,310 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "bytes" + "fmt" + "io" + "os" + "slices" + "testing" + "time" + + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + + "github.com/stretchr/testify/assert" +) + +func TestFastImport(t *testing.T) { + unittest.PrepareTestEnv(t) + + // Initialize the repository + repoPath, err := os.MkdirTemp("", "test-repo-*.git") + assert.NoError(t, err) + defer os.RemoveAll(repoPath) + + _, _, err = git.NewCommand("init", "--bare").RunStdString(t.Context(), + &git.RunOpts{ + Dir: repoPath, + }) + assert.NoError(t, err) + + // Create a temporary directory for the test + tempDir, err := os.MkdirTemp("", "fast_import_test") + assert.NoError(t, err) + defer os.RemoveAll(tempDir) + + // Create a test file + testFilePath := fmt.Sprintf("%s/testfile.txt", tempDir) + err = os.WriteFile(testFilePath, []byte("Hello, World!"), 0o644) + assert.NoError(t, err) + + f, err := os.Open(testFilePath) + assert.NoError(t, err) + defer f.Close() + + doer, err := user_model.GetUserByID(t.Context(), 1) + assert.NoError(t, err) + + // 1 - Create the commit file + t.Run("Create commit file", func(t *testing.T) { + // Prepare the ChangeRepoFilesOptions + options := ChangeRepoFilesOptions{ + LastCommitID: "HEAD", + NewBranch: "test-branch", + Message: "Test commit", + Files: []*ChangeRepoFile{ + { + Operation: "create", + TreePath: "testfile.txt", + ContentReader: f, + }, + }, + Author: &git.Signature{ + Name: "Test User", + Email: "testuser@gitea.com", + When: time.Now(), + }, + Committer: &git.Signature{ + Name: "Test Committer", + Email: "testuser@gitea.com", + When: time.Now(), + }, + } + + err = UpdateRepoBranch(t.Context(), doer, repoPath, true, options) + assert.NoError(t, err) + + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + assert.NoError(t, err) + defer gitRepo.Close() + + branches, total, err := gitRepo.GetBranchNames(0, 0) + assert.NoError(t, err) + assert.Equal(t, 1, total) + assert.Equal(t, 1, len(branches)) + assert.Equal(t, "test-branch", branches[0]) + + commit, err := gitRepo.GetBranchCommit("test-branch") + assert.NoError(t, err) + assert.Equal(t, "Test commit\n", commit.Message()) + entries, err := commit.Tree.ListEntries() + assert.NoError(t, err) + assert.Equal(t, 1, len(entries)) + assert.Equal(t, "testfile.txt", entries[0].Name()) + content, err := entries[0].Blob().GetBlobContent(entries[0].Blob().Size()) + assert.NoError(t, err) + assert.Equal(t, "Hello, World!", content) + }) + + // 2 - Add a new file and update the existing one in a new branch + t.Run("Add and update files", func(t *testing.T) { + f2 := bytes.NewReader([]byte("Hello, World! 1")) + _, err = f.Seek(0, io.SeekStart) + assert.NoError(t, err) + options := ChangeRepoFilesOptions{ + LastCommitID: "HEAD", + OldBranch: "test-branch", + NewBranch: "test-branch-2", + Message: "Test commit-2", + Files: []*ChangeRepoFile{ + { + Operation: "create", + TreePath: "testfile2.txt", + ContentReader: f, + }, + { + Operation: "update", + TreePath: "testfile.txt", + ContentReader: f2, + }, + }, + Author: &git.Signature{ + Name: "Test User", + Email: "testuser@gitea.com", + When: time.Now(), + }, + Committer: &git.Signature{ + Name: "Test Committer", + Email: "testuser@gitea.com", + When: time.Now(), + }, + } + err = UpdateRepoBranch(t.Context(), doer, repoPath, true, options) + assert.NoError(t, err) + + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + assert.NoError(t, err) + defer gitRepo.Close() + + branches, total, err := gitRepo.GetBranchNames(0, 0) + assert.NoError(t, err) + assert.Equal(t, 2, total) + assert.Equal(t, 2, len(branches)) + assert.True(t, slices.Equal(branches, []string{"test-branch", "test-branch-2"})) + + commit, err := gitRepo.GetBranchCommit("test-branch-2") + assert.NoError(t, err) + assert.Equal(t, "Test commit-2\n", commit.Message()) + entries, err := commit.Tree.ListEntries() + assert.NoError(t, err) + assert.Equal(t, 2, len(entries)) + assert.Equal(t, "testfile.txt", entries[0].Name()) + assert.Equal(t, "testfile2.txt", entries[1].Name()) + content, err := entries[0].Blob().GetBlobContent(entries[0].Blob().Size()) + assert.NoError(t, err) + assert.Equal(t, "Hello, World! 1", content) + + content, err = entries[1].Blob().GetBlobContent(entries[1].Blob().Size()) + assert.NoError(t, err) + assert.Equal(t, "Hello, World!", content) + }) + + // 3 - Delete the file + t.Run("Delete file in the same branch", func(t *testing.T) { + options := ChangeRepoFilesOptions{ + OldBranch: "test-branch-2", + Message: "Test commit-3", + Files: []*ChangeRepoFile{ + { + Operation: "delete", + FromTreePath: "testfile.txt", + }, + }, + Author: &git.Signature{ + Name: "Test User", + Email: "testuser@gitea.com", + When: time.Now(), + }, + Committer: &git.Signature{ + Name: "Test Committer", + Email: "testuser@gitea.com", + When: time.Now(), + }, + } + err = UpdateRepoBranch(t.Context(), doer, repoPath, true, options) + assert.NoError(t, err) + + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + assert.NoError(t, err) + defer gitRepo.Close() + + branches, total, err := gitRepo.GetBranchNames(0, 0) + assert.NoError(t, err) + assert.Equal(t, 2, total) + assert.Equal(t, 2, len(branches)) + assert.True(t, slices.Equal(branches, []string{"test-branch", "test-branch-2"})) + + commit, err := gitRepo.GetBranchCommit("test-branch-2") + assert.NoError(t, err) + assert.Equal(t, "Test commit-3\n", commit.Message()) + entries, err := commit.Tree.ListEntries() + assert.NoError(t, err) + assert.Equal(t, 1, len(entries)) + assert.Equal(t, "testfile2.txt", entries[0].Name()) + content, err := entries[0].Blob().GetBlobContent(entries[0].Blob().Size()) + assert.NoError(t, err) + assert.Equal(t, "Hello, World!", content) + }) + + // 4 - Delete the file in a new branch + t.Run("Delete file in a new branch", func(t *testing.T) { + options := ChangeRepoFilesOptions{ + OldBranch: "test-branch-2", + NewBranch: "test-branch-3", + Message: "Test commit-4", + Files: []*ChangeRepoFile{ + { + Operation: "delete", + FromTreePath: "testfile2.txt", + }, + }, + Author: &git.Signature{ + Name: "Test User", + Email: "testuser@gitea.com", + When: time.Now(), + }, + Committer: &git.Signature{ + Name: "Test Committer", + Email: "testuser@gitea.com", + When: time.Now(), + }, + } + err = UpdateRepoBranch(t.Context(), doer, repoPath, true, options) + assert.NoError(t, err) + + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + assert.NoError(t, err) + defer gitRepo.Close() + + branches, total, err := gitRepo.GetBranchNames(0, 0) + assert.NoError(t, err) + assert.Equal(t, 3, total) + assert.Equal(t, 3, len(branches)) + assert.True(t, slices.Equal(branches, []string{"test-branch", "test-branch-2", "test-branch-3"})) + + commit, err := gitRepo.GetBranchCommit("test-branch-3") + assert.NoError(t, err) + assert.Equal(t, "Test commit-4\n", commit.Message()) + entries, err := commit.Tree.ListEntries() + assert.NoError(t, err) + assert.Equal(t, 0, len(entries)) + }) + + // 5 - add/delete the file in a new branch from test-branch-2 + t.Run("Add/Delete file in a new branch", func(t *testing.T) { + options := ChangeRepoFilesOptions{ + OldBranch: "test-branch-2", + NewBranch: "test-branch-4", + Message: "Test commit-5", + Files: []*ChangeRepoFile{ + { + Operation: "create", + TreePath: "testfile3.txt", + ContentReader: bytes.NewReader([]byte("Hello, World! 3")), + }, + { + Operation: "delete", + FromTreePath: "testfile2.txt", + }, + }, + Author: &git.Signature{ + Name: "Test User", + Email: "testuser@gitea.com", + When: time.Now(), + }, + Committer: &git.Signature{ + Name: "Test Committer", + Email: "testuser@gitea.com", + When: time.Now(), + }, + } + err = UpdateRepoBranch(t.Context(), doer, repoPath, true, options) + assert.NoError(t, err) + + gitRepo, err := git.OpenRepository(t.Context(), repoPath) + assert.NoError(t, err) + defer gitRepo.Close() + + branches, total, err := gitRepo.GetBranchNames(0, 0) + assert.NoError(t, err) + assert.Equal(t, 4, total) + assert.Equal(t, 4, len(branches)) + assert.True(t, slices.Equal(branches, []string{"test-branch", "test-branch-2", "test-branch-3", "test-branch-4"})) + + commit, err := gitRepo.GetBranchCommit("test-branch-4") + assert.NoError(t, err) + assert.Equal(t, "Test commit-5\n", commit.Message()) + entries, err := commit.Tree.ListEntries() + assert.NoError(t, err) + assert.Equal(t, 1, len(entries)) + assert.Equal(t, "testfile3.txt", entries[0].Name()) + content, err := entries[0].Blob().GetBlobContent(entries[0].Blob().Size()) + assert.NoError(t, err) + assert.Equal(t, "Hello, World! 3", content) + }) +} diff --git a/services/repository/files/main_test.go b/services/repository/files/main_test.go new file mode 100644 index 0000000000000..6b07e102a241f --- /dev/null +++ b/services/repository/files/main_test.go @@ -0,0 +1,15 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package files + +import ( + "testing" + + "code.gitea.io/gitea/models/unittest" + _ "code.gitea.io/gitea/models/user" +) + +func TestMain(m *testing.M) { + unittest.MainTest(m) +} diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go index 493ff9998d0e7..f1eb5e3c0567a 100644 --- a/services/repository/files/temp_repo.go +++ b/services/repository/files/temp_repo.go @@ -166,21 +166,7 @@ func (t *TemporaryUploadRepository) RemoveFilesFromIndex(ctx context.Context, fi // HashObject writes the provided content to the object db and returns its hash func (t *TemporaryUploadRepository) HashObject(ctx context.Context, content io.Reader) (string, error) { - stdOut := new(bytes.Buffer) - stdErr := new(bytes.Buffer) - - if err := git.NewCommand("hash-object", "-w", "--stdin"). - Run(ctx, &git.RunOpts{ - Dir: t.basePath, - Stdin: content, - Stdout: stdOut, - Stderr: stdErr, - }); err != nil { - log.Error("Unable to hash-object to temporary repo: %s (%s) Error: %v\nstdout: %s\nstderr: %s", t.repo.FullName(), t.basePath, err, stdOut.String(), stdErr.String()) - return "", fmt.Errorf("Unable to hash-object to temporary repo: %s Error: %w\nstdout: %s\nstderr: %s", t.repo.FullName(), err, stdOut.String(), stdErr.String()) - } - - return strings.TrimSpace(stdOut.String()), nil + return hashObject(ctx, t.basePath, content) } // AddObjectToIndex adds the provided object hash to the index with the provided mode and path diff --git a/services/repository/files/update.go b/services/repository/files/update.go index fbf59c40edb97..b40ec6c1c0423 100644 --- a/services/repository/files/update.go +++ b/services/repository/files/update.go @@ -9,7 +9,6 @@ import ( "io" "path" "strings" - "time" git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" @@ -27,46 +26,6 @@ import ( pull_service "code.gitea.io/gitea/services/pull" ) -// IdentityOptions for a person's identity like an author or committer -type IdentityOptions struct { - GitUserName string // to match "git config user.name" - GitUserEmail string // to match "git config user.email" -} - -// CommitDateOptions store dates for GIT_AUTHOR_DATE and GIT_COMMITTER_DATE -type CommitDateOptions struct { - Author time.Time - Committer time.Time -} - -type ChangeRepoFile struct { - Operation string - TreePath string - FromTreePath string - ContentReader io.ReadSeeker - SHA string - Options *RepoFileOptions -} - -// ChangeRepoFilesOptions holds the repository files update options -type ChangeRepoFilesOptions struct { - LastCommitID string - OldBranch string - NewBranch string - Message string - Files []*ChangeRepoFile - Author *IdentityOptions - Committer *IdentityOptions - Dates *CommitDateOptions - Signoff bool -} - -type RepoFileOptions struct { - treePath string - fromTreePath string - executable bool -} - // ErrRepoFileDoesNotExist represents a "RepoFileDoesNotExist" kind of error. type ErrRepoFileDoesNotExist struct { Path string @@ -268,18 +227,17 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use // Now commit the tree commitOpts := &CommitTreeUserOptions{ - ParentCommitID: opts.LastCommitID, - TreeHash: treeHash, - CommitMessage: message, - SignOff: opts.Signoff, - DoerUser: doer, - AuthorIdentity: opts.Author, - AuthorTime: nil, - CommitterIdentity: opts.Committer, - CommitterTime: nil, - } - if opts.Dates != nil { - commitOpts.AuthorTime, commitOpts.CommitterTime = &opts.Dates.Author, &opts.Dates.Committer + ParentCommitID: opts.LastCommitID, + TreeHash: treeHash, + CommitMessage: message, + SignOff: opts.Signoff, + DoerUser: doer, + // FIXME: + // AuthorIdentity: opts.Author, + AuthorTime: &opts.Author.When, + // FIXME: + // CommitterIdentity: opts.Committer, + CommitterTime: &opts.Committer.When, } commitHash, err := t.CommitTree(ctx, commitOpts) if err != nil { diff --git a/services/repository/files/upload.go b/services/repository/files/upload.go index f348cb68ab543..58409e08725fb 100644 --- a/services/repository/files/upload.go +++ b/services/repository/files/upload.go @@ -184,7 +184,7 @@ func copyUploadedLFSFileIntoRepository(ctx context.Context, info *uploadInfo, at defer file.Close() var objectHash string - if setting.LFS.StartServer && attributesMap[info.upload.Name] != nil && attributesMap[info.upload.Name].Get(attribute.Filter).ToString().Value() == "lfs" { + if setting.LFS.StartServer && attributesMap[info.upload.Name] != nil && attributesMap[info.upload.Name].MatchLFS() { // Handle LFS // FIXME: Inefficient! this should probably happen in models.Upload pointer, err := lfs.GeneratePointer(file) diff --git a/services/wiki/wiki.go b/services/wiki/wiki.go index 45a08dc5d686d..774476ed4e61e 100644 --- a/services/wiki/wiki.go +++ b/services/wiki/wiki.go @@ -5,6 +5,7 @@ package wiki import ( + "bytes" "context" "errors" "fmt" @@ -20,10 +21,10 @@ import ( "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/globallock" "code.gitea.io/gitea/modules/log" - repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" repo_service "code.gitea.io/gitea/services/repository" + files_service "code.gitea.io/gitea/services/repository/files" ) const DefaultRemote = "origin" @@ -102,46 +103,22 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model hasDefaultBranch := gitrepo.IsBranchExist(ctx, repo.WikiStorageRepo(), repo.DefaultWikiBranch) - basePath, cleanup, err := repo_module.CreateTemporaryPath("update-wiki") + gitRepo, err := gitrepo.OpenRepository(ctx, repo.WikiStorageRepo()) if err != nil { - return err - } - defer cleanup() - - cloneOpts := git.CloneRepoOptions{ - Bare: true, - Shared: true, - } - - if hasDefaultBranch { - cloneOpts.Branch = repo.DefaultWikiBranch - } - - if err := git.Clone(ctx, repo.WikiPath(), basePath, cloneOpts); err != nil { - log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err) - return fmt.Errorf("failed to clone repository: %s (%w)", repo.FullName(), err) - } - - gitRepo, err := git.OpenRepository(ctx, basePath) - if err != nil { - log.Error("Unable to open temporary repository: %s (%v)", basePath, err) - return fmt.Errorf("failed to open new temporary repository in: %s %w", basePath, err) + log.Error("Unable to open temporary repository: %s (%v)", repo.WikiPath(), err) + return fmt.Errorf("failed to open new temporary repository in: %s %w", repo.WikiPath(), err) } defer gitRepo.Close() - if hasDefaultBranch { - if err := gitRepo.ReadTreeToIndex("HEAD"); err != nil { - log.Error("Unable to read HEAD tree to index in: %s %v", basePath, err) - return fmt.Errorf("fnable to read HEAD tree to index in: %s %w", basePath, err) - } - } - isWikiExist, newWikiPath, err := prepareGitPath(gitRepo, repo.DefaultWikiBranch, newWikiName) if err != nil { return err } + operation := "update" + var oldWikiPath string if isNew { + operation = "create" if isWikiExist { return repo_model.ErrWikiAlreadyExist{ Title: newWikiPath, @@ -149,87 +126,40 @@ func updateWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model } } else { // avoid check existence again if wiki name is not changed since gitRepo.LsFiles(...) is not free. - isOldWikiExist := true - oldWikiPath := newWikiPath + oldWikiPath = newWikiPath if oldWikiName != newWikiName { - isOldWikiExist, oldWikiPath, err = prepareGitPath(gitRepo, repo.DefaultWikiBranch, oldWikiName) + _, oldWikiPath, err = prepareGitPath(gitRepo, repo.DefaultWikiBranch, oldWikiName) if err != nil { return err } } - - if isOldWikiExist { - err := gitRepo.RemoveFilesFromIndex(oldWikiPath) - if err != nil { - log.Error("RemoveFilesFromIndex failed: %v", err) - return err - } - } } // FIXME: The wiki doesn't have lfs support at present - if this changes need to check attributes here - - objectHash, err := gitRepo.HashObject(strings.NewReader(content)) - if err != nil { - log.Error("HashObject failed: %v", err) - return err - } - - if err := gitRepo.AddObjectToIndex("100644", objectHash, newWikiPath); err != nil { - log.Error("AddObjectToIndex failed: %v", err) - return err - } - - tree, err := gitRepo.WriteTree() - if err != nil { - log.Error("WriteTree failed: %v", err) - return err - } - - commitTreeOpts := git.CommitTreeOpts{ - Message: message, - } - + author := doer.NewGitSig() committer := doer.NewGitSig() - sign, signingKey, signer, _ := asymkey_service.SignWikiCommit(ctx, repo, doer) - if sign { - commitTreeOpts.KeyID = signingKey - if repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { - committer = signer - } - } else { - commitTreeOpts.NoGPGSign = true - } - if hasDefaultBranch { - commitTreeOpts.Parents = []string{"HEAD"} - } - - commitHash, err := gitRepo.CommitTree(doer.NewGitSig(), committer, tree, commitTreeOpts) - if err != nil { - log.Error("CommitTree failed: %v", err) - return err - } - - if err := git.Push(gitRepo.Ctx, basePath, git.PushOptions{ - Remote: DefaultRemote, - Branch: fmt.Sprintf("%s:%s%s", commitHash.String(), git.BranchPrefix, repo.DefaultWikiBranch), - Env: repo_module.FullPushingEnvironment( - doer, - doer, - repo, - repo.Name+".wiki", - 0, - ), - }); err != nil { - log.Error("Push failed: %v", err) - if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) { - return err - } - return fmt.Errorf("failed to push: %w", err) - } - - return nil + _, signingKey, signer, _ := asymkey_service.SignWikiCommit(ctx, repo, doer) + + trustCommitter := repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel + + return files_service.UpdateRepoBranch(ctx, doer, repo.WikiPath(), trustCommitter, files_service.ChangeRepoFilesOptions{ + OldBranch: util.Iif(hasDefaultBranch, repo.DefaultWikiBranch, ""), + NewBranch: util.Iif(hasDefaultBranch, "", repo.DefaultWikiBranch), + Files: []*files_service.ChangeRepoFile{ + { + Operation: operation, + FromTreePath: oldWikiPath, + TreePath: newWikiPath, + ContentReader: bytes.NewReader([]byte(content)), + }, + }, + Message: message, + Author: author, + Committer: committer, + Signer: signer, + SignKey: signingKey, + }) } // AddWikiPage adds a new wiki page with a given wikiPath. @@ -260,93 +190,44 @@ func DeleteWikiPage(ctx context.Context, doer *user_model.User, repo *repo_model return fmt.Errorf("InitWiki: %w", err) } - basePath, cleanup, err := repo_module.CreateTemporaryPath("update-wiki") + gitRepo, err := git.OpenRepository(ctx, repo.WikiPath()) if err != nil { - return err - } - defer cleanup() - - if err := git.Clone(ctx, repo.WikiPath(), basePath, git.CloneRepoOptions{ - Bare: true, - Shared: true, - Branch: repo.DefaultWikiBranch, - }); err != nil { - log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err) - return fmt.Errorf("failed to clone repository: %s (%w)", repo.FullName(), err) - } - - gitRepo, err := git.OpenRepository(ctx, basePath) - if err != nil { - log.Error("Unable to open temporary repository: %s (%v)", basePath, err) - return fmt.Errorf("failed to open new temporary repository in: %s %w", basePath, err) + log.Error("Unable to open temporary repository: %s (%v)", repo.WikiPath(), err) + return fmt.Errorf("failed to open new temporary repository in: %s %w", repo.WikiPath(), err) } defer gitRepo.Close() - if err := gitRepo.ReadTreeToIndex("HEAD"); err != nil { - log.Error("Unable to read HEAD tree to index in: %s %v", basePath, err) - return fmt.Errorf("unable to read HEAD tree to index in: %s %w", basePath, err) - } - found, wikiPath, err := prepareGitPath(gitRepo, repo.DefaultWikiBranch, wikiName) if err != nil { return err } - if found { - err := gitRepo.RemoveFilesFromIndex(wikiPath) - if err != nil { - return err - } - } else { + if !found { return os.ErrNotExist } // FIXME: The wiki doesn't have lfs support at present - if this changes need to check attributes here - - tree, err := gitRepo.WriteTree() - if err != nil { - return err - } message := fmt.Sprintf("Delete page %q", wikiName) - commitTreeOpts := git.CommitTreeOpts{ - Message: message, - Parents: []string{"HEAD"}, - } - + author := doer.NewGitSig() committer := doer.NewGitSig() - sign, signingKey, signer, _ := asymkey_service.SignWikiCommit(ctx, repo, doer) - if sign { - commitTreeOpts.KeyID = signingKey - if repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { - committer = signer - } - } else { - commitTreeOpts.NoGPGSign = true - } - - commitHash, err := gitRepo.CommitTree(doer.NewGitSig(), committer, tree, commitTreeOpts) - if err != nil { - return err - } - - if err := git.Push(gitRepo.Ctx, basePath, git.PushOptions{ - Remote: DefaultRemote, - Branch: fmt.Sprintf("%s:%s%s", commitHash.String(), git.BranchPrefix, repo.DefaultWikiBranch), - Env: repo_module.FullPushingEnvironment( - doer, - doer, - repo, - repo.Name+".wiki", - 0, - ), - }); err != nil { - if git.IsErrPushOutOfDate(err) || git.IsErrPushRejected(err) { - return err - } - return fmt.Errorf("Push: %w", err) - } - - return nil + _, signingKey, signer, _ := asymkey_service.SignWikiCommit(ctx, repo, doer) + + trustCommitter := repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel + + return files_service.UpdateRepoBranch(ctx, doer, repo.WikiPath(), trustCommitter, files_service.ChangeRepoFilesOptions{ + OldBranch: repo.DefaultWikiBranch, + Files: []*files_service.ChangeRepoFile{ + { + Operation: "delete", + FromTreePath: wikiPath, + }, + }, + Message: message, + Author: author, + Committer: committer, + Signer: signer, + SignKey: signingKey, + }) } // DeleteWiki removes the actual and local copy of repository wiki.