From 8fc5265d6f144fc5af22609b50b62f6561184dbe Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Sat, 30 Jul 2022 21:45:03 -0400 Subject: [PATCH 01/13] git-annex support [git-annex](https://git-annex.branchable.com/) is a more complicated cousin to git-lfs, storing large files in an optional-download side content. Unlike lfs, it allows mixing and matching storage remotes, so the content remote(s) doesn't need to be on the same server as the git remote, making it feasible to scatter a collection across cloud storage, old harddrives, or anywhere else storage can be scavenged. Since this can get complicated, fast, it has a content-tracking database (`git annex whereis`) to help find everything later. The use-case we imagine for including it in Gitea is just the simple case, where we're primarily emulating git-lfs: each repo has its large content at the same URL. Our motivation is so we can self-host https://www.datalad.org/ datasets, which currently are only hostable by fragilely scrounging together cloud storage -- and having to manage all the credentials associated with all the pieces -- or at https://openneuro.org which is fragile in its own ways. Supporting git-annex also allows multiple Gitea instance to be annex remotes for each other, mirroring the content or otherwise collaborating the split up the hosting costs. Enabling -------- TODO HTTP ---- TODO Permission Checking ------------------- This tweaks the API in routers/private/serv.go to expose the calling user's computed permission, instead of just returning HTTP 403. This doesn't fit in super well. It's the opposite from how the git-lfs support is done, where there's a complete list of possible subcommands and their matching permission levels, and then the API compares the requested with the actual level and returns HTTP 403 if the check fails. But it's necessary. The main git-annex verbs, 'git-annex-shell configlist' and 'git-annex-shell p2pstdio' are both either read-only or read-write operations, depending on the state on disk on either end of the connection and what the user asked it to ask for, with no way to know before git-annex examines the situation. So tell the level via GIT_ANNEX_READONLY and trust it to handle itself. In the older Gogs version, the permission was directly read in cmd/serv.go: ``` mode, err = db.UserAccessMode(user.ID, repo) ``` - https://github.com/G-Node/gogs/blob/966e925cf320beff768b192276774d9265706df5/internal/cmd/serv.go#L334 but in Gitea permission enforcement has been centralized in the API layer. (perhaps so the cmd layer can avoid making direct DB connections?) Deletion -------- git-annex has this "lockdown" feature where it tries really quite very hard to prevent you deleting its data, to the point that even an rm -rf won't do it: each file in annex/objects/ is nested inside a folder with read-only permissions. The recommended workaround is to run chmod -R +w when you're sure you actually want to delete a repo. See https://git-annex.branchable.com/internals/lockdown So we edit util.RemoveAll() to do just that, so now it's `chmod -R +w && rm -rf` instead of just `rm -rf`. --- cmd/serv.go | 73 ++++++++++++++++++++++++++++++++++++++--- modules/private/serv.go | 1 + modules/util/remove.go | 34 ++++++++++++++++++- routers/private/serv.go | 12 +++++-- 4 files changed, 111 insertions(+), 9 deletions(-) diff --git a/cmd/serv.go b/cmd/serv.go index 26fc91a3b7a30..20ca4484c386d 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -37,6 +37,7 @@ import ( const ( lfsAuthenticateVerb = "git-lfs-authenticate" + gitAnnexShellVerb = "git-annex-shell" ) // CmdServ represents the available serv sub-command. @@ -89,6 +90,7 @@ var ( "git-upload-archive": perm.AccessModeRead, "git-receive-pack": perm.AccessModeWrite, lfsAuthenticateVerb: perm.AccessModeNone, + gitAnnexShellVerb: perm.AccessModeNone, // annex permissions are enforced by GIT_ANNEX_SHELL_READONLY, rather than the Gitea API } alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`) ) @@ -201,6 +203,7 @@ func runServ(c *cli.Context) error { verb := words[0] repoPath := words[1] + if repoPath[0] == '/' { repoPath = repoPath[1:] } @@ -216,9 +219,44 @@ func runServ(c *cli.Context) error { } } + if verb == gitAnnexShellVerb { + // if !setting.Annex.Enabled { // TODO: https://github.com/neuropoly/gitea/issues/8 + if false { + return fail(ctx, "Unknown git command", "git-annex request over SSH denied, git-annex support is disabled") + } + + if len(words) < 3 { + return fail(ctx, "Too few arguments", "Too few arguments in cmd: %s", cmd) + } + + // git-annex always puts the repo in words[2], unlike most other + // git subcommands; and it sometimes names repos like /~/, as if + // $HOME should get expanded while also being rooted. e.g.: + // git-annex-shell 'configlist' '/~/user/repo' + // git-annex-shell 'sendkey' '/user/repo 'key' + repoPath = words[2] + repoPath = strings.TrimPrefix(repoPath, "/") + repoPath = strings.TrimPrefix(repoPath, "~/") + } + // LowerCase and trim the repoPath as that's how they are stored. repoPath = strings.ToLower(strings.TrimSpace(repoPath)) + // prevent directory traversal attacks + repoPath = filepath.Clean("/" + repoPath)[1:] + + // put the sanitized repoPath back into the argument list for later + if verb == gitAnnexShellVerb { + // git-annex-shell demands an absolute path + absRepoPath, err := filepath.Abs(filepath.Join(setting.RepoRootPath, repoPath)) + if err != nil { + return fail(ctx, "Error locating repoPath", "%v", err) + } + words[2] = absRepoPath + } else { + words[1] = repoPath + } + rr := strings.SplitN(repoPath, "/", 2) if len(rr) != 2 { return fail(ctx, "Invalid repository path", "Invalid repository path: %v", repoPath) @@ -305,21 +343,46 @@ func runServ(c *cli.Context) error { return nil } - var gitcmd *exec.Cmd gitBinPath := filepath.Dir(git.GitExecutable) // e.g. /usr/bin gitBinVerb := filepath.Join(gitBinPath, verb) // e.g. /usr/bin/git-upload-pack if _, err := os.Stat(gitBinVerb); err != nil { // if the command "git-upload-pack" doesn't exist, try to split "git-upload-pack" to use the sub-command with git // ps: Windows only has "git.exe" in the bin path, so Windows always uses this way + // ps: git-annex-shell and other extensions may not necessarily be in gitBinPath, + // but '{gitBinPath}/git annex-shell' should be able to find them on $PATH. verbFields := strings.SplitN(verb, "-", 2) if len(verbFields) == 2 { // use git binary with the sub-command part: "C:\...\bin\git.exe", "upload-pack", ... - gitcmd = exec.CommandContext(ctx, git.GitExecutable, verbFields[1], repoPath) + gitBinVerb = git.GitExecutable + words = append([]string{verbFields[1]}, words...) } } - if gitcmd == nil { - // by default, use the verb (it has been checked above by allowedCommands) - gitcmd = exec.CommandContext(ctx, gitBinVerb, repoPath) + + // by default, use the verb (it has been checked above by allowedCommands) + gitcmd := exec.CommandContext(ctx, gitBinVerb, words[1:]...) + + if verb == gitAnnexShellVerb { + // This doesn't get its own isolated section like LFS does, because LFS + // is handled by internal Gitea routines, but git-annex has to be shelled out + // to like other git subcommands, so we need to build up gitcmd. + + // TODO: does this work on Windows? + gitcmd.Env = append(gitcmd.Env, + // "If set, disallows running git-shell to handle unknown commands." + // - git-annex-shell(1) + "GIT_ANNEX_SHELL_LIMITED=True", + // "If set, git-annex-shell will refuse to run commands + // that do not operate on the specified directory." + // - git-annex-shell(1) + fmt.Sprintf("GIT_ANNEX_SHELL_DIRECTORY=%s", words[2]), + ) + if results.UserMode < perm.AccessModeWrite { + // "If set, disallows any action that could modify the git-annex repository." + // - git-annex-shell(1) + // We set this when the backend API has told us that we don't have write permission to this repo. + log.Debug("Setting GIT_ANNEX_SHELL_READONLY=True") + gitcmd.Env = append(gitcmd.Env, "GIT_ANNEX_SHELL_READONLY=True") + } } process.SetSysProcAttribute(gitcmd) diff --git a/modules/private/serv.go b/modules/private/serv.go index 480a44695496d..6c7c753cf09ca 100644 --- a/modules/private/serv.go +++ b/modules/private/serv.go @@ -40,6 +40,7 @@ type ServCommandResults struct { UserName string UserEmail string UserID int64 + UserMode perm.AccessMode OwnerName string RepoName string RepoID int64 diff --git a/modules/util/remove.go b/modules/util/remove.go index d1e38faf5f1fb..0f41471fcc374 100644 --- a/modules/util/remove.go +++ b/modules/util/remove.go @@ -4,7 +4,9 @@ package util import ( + "io/fs" "os" + "path/filepath" "runtime" "syscall" "time" @@ -41,10 +43,40 @@ func Remove(name string) error { return err } -// RemoveAll removes the named file or (empty) directory with at most 5 attempts. +// RemoveAll removes the named file or directory with at most 5 attempts. func RemoveAll(name string) error { var err error + for i := 0; i < 5; i++ { + // Do chmod -R +w to help ensure the removal succeeds. + // In particular, in the git-annex case, this handles + // https://git-annex.branchable.com/internals/lockdown/ : + // + // > (The only bad consequence of this is that rm -rf .git + // > doesn't work unless you first run chmod -R +w .git) + + err = filepath.WalkDir(name, func(path string, d fs.DirEntry, err error) error { + // NB: this is called WalkDir but it works on a single file too + if err == nil { + info, err := d.Info() + if err != nil { + return err + } + + // 0200 == u+w, in octal unix permission notation + err = os.Chmod(path, info.Mode()|0o200) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + // try again + <-time.After(100 * time.Millisecond) + continue + } + err = os.RemoveAll(name) if err == nil { break diff --git a/routers/private/serv.go b/routers/private/serv.go index 00731947a55a8..ad33d86e1462a 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -81,12 +81,14 @@ func ServCommand(ctx *context.PrivateContext) { ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") mode := perm.AccessMode(ctx.FormInt("mode")) + verbs := ctx.FormStrings("verb") // Set the basic parts of the results to return results := private.ServCommandResults{ RepoName: repoName, OwnerName: ownerName, KeyID: keyID, + UserMode: perm.AccessModeNone, } // Now because we're not translating things properly let's just default some English strings here @@ -287,8 +289,10 @@ func ServCommand(ctx *context.PrivateContext) { repo.IsPrivate || owner.Visibility.IsPrivate() || (user != nil && user.IsRestricted) || // user will be nil if the key is a deploykey + ( /*setting.Annex.Enabled && */ len(verbs) > 0 && verbs[0] == "git-annex-shell") || // git-annex has its own permission enforcement, for which we expose results.UserMode setting.Service.RequireSignInView) { if key.Type == asymkey_model.KeyTypeDeploy { + results.UserMode = deployKey.Mode if deployKey.Mode < mode { ctx.JSON(http.StatusUnauthorized, private.Response{ UserMsg: fmt.Sprintf("Deploy Key: %d:%s is not authorized to %s %s/%s.", key.ID, key.Name, modeString, results.OwnerName, results.RepoName), @@ -310,9 +314,9 @@ func ServCommand(ctx *context.PrivateContext) { return } - userMode := perm.UnitAccessMode(unitType) + results.UserMode = perm.UnitAccessMode(unitType) - if userMode < mode { + if results.UserMode < mode { log.Warn("Failed authentication attempt for %s with key %s (not authorized to %s %s/%s) from %s", user.Name, key.Name, modeString, ownerName, repoName, ctx.RemoteAddr()) ctx.JSON(http.StatusUnauthorized, private.Response{ UserMsg: fmt.Sprintf("User: %d:%s with Key: %d:%s is not authorized to %s %s/%s.", user.ID, user.Name, key.ID, key.Name, modeString, ownerName, repoName), @@ -353,6 +357,7 @@ func ServCommand(ctx *context.PrivateContext) { }) return } + results.UserMode = perm.AccessModeWrite results.RepoID = repo.ID } @@ -381,13 +386,14 @@ func ServCommand(ctx *context.PrivateContext) { return } } - log.Debug("Serv Results:\nIsWiki: %t\nDeployKeyID: %d\nKeyID: %d\tKeyName: %s\nUserName: %s\nUserID: %d\nOwnerName: %s\nRepoName: %s\nRepoID: %d", + log.Debug("Serv Results:\nIsWiki: %t\nDeployKeyID: %d\nKeyID: %d\tKeyName: %s\nUserName: %s\nUserID: %d\nUserMode: %d\nOwnerName: %s\nRepoName: %s\nRepoID: %d", results.IsWiki, results.DeployKeyID, results.KeyID, results.KeyName, results.UserName, results.UserID, + results.UserMode, results.OwnerName, results.RepoName, results.RepoID) From 900a9673e21e39f291259b1cdb1f1c8063a11820 Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 19 Aug 2022 02:49:18 -0400 Subject: [PATCH 02/13] git-annex tests (#13) Fixes https://github.com/neuropoly/gitea/issues/11 Tests: * `git annex init` * `git annex copy --from origin` * `git annex copy --to origin` over: * ssh for: * the owner * a collaborator * a read-only collaborator * a stranger in a * public repo * private repo And then confirms: * Deletion of the remote repo (to ensure lockdown isn't messing with us: https://git-annex.branchable.com/internals/lockdown/#comment-0cc5225dc5abe8eddeb843bfd2fdc382) ------ To support all this: * Add util.FileCmp() * Patch withKeyFile() so it can be nested in other copies of itself ------- Many thanks to Mathieu for giving style tips and catching several bugs, including a subtle one in util.filecmp() which neutered it. Co-authored-by: Mathieu Guay-Paquet --- .github/workflows/pull-db-tests.yml | 4 + Makefile | 2 +- modules/util/filecmp.go | 87 ++ .../api_helper_for_declarative_test.go | 25 + tests/integration/git_annex_test.go | 759 ++++++++++++++++++ .../git_helper_for_declarative_test.go | 22 + 6 files changed, 898 insertions(+), 1 deletion(-) create mode 100644 modules/util/filecmp.go create mode 100644 tests/integration/git_annex_test.go diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 97446e6cd3b2f..c44e82f82eadd 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -46,6 +46,7 @@ jobs: - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 pgsql ldap minio" | sudo tee -a /etc/hosts' - run: make deps-backend + - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex - run: make backend env: TAGS: bindata @@ -69,6 +70,7 @@ jobs: go-version-file: go.mod check-latest: true - run: make deps-backend + - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex - run: make backend env: TAGS: bindata gogit sqlite sqlite_unlock_notify @@ -172,6 +174,7 @@ jobs: - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mysql elasticsearch smtpimap" | sudo tee -a /etc/hosts' - run: make deps-backend + - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex - run: make backend env: TAGS: bindata @@ -205,6 +208,7 @@ jobs: - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mssql" | sudo tee -a /etc/hosts' - run: make deps-backend + - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex - run: make backend env: TAGS: bindata diff --git a/Makefile b/Makefile index 068dda5f52b17..f8838b601beca 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ self := $(location) @tmpdir=`mktemp --tmpdir -d` ; \ echo Using temporary directory $$tmpdir for test repositories ; \ USE_REPO_TEST_DIR= $(MAKE) -f $(self) --no-print-directory REPO_TEST_DIR=$$tmpdir/ $@ ; \ - STATUS=$$? ; rm -r "$$tmpdir" ; exit $$STATUS + STATUS=$$? ; chmod -R +w "$$tmpdir" && rm -r "$$tmpdir" ; exit $$STATUS else diff --git a/modules/util/filecmp.go b/modules/util/filecmp.go new file mode 100644 index 0000000000000..76e7705cc1b56 --- /dev/null +++ b/modules/util/filecmp.go @@ -0,0 +1,87 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "bytes" + "io" + "os" +) + +// Decide if two files have the same contents or not. +// chunkSize is the size of the blocks to scan by; pass 0 to get a sensible default. +// *Follows* symlinks. +// +// May return an error if something else goes wrong; in this case, you should ignore the value of 'same'. +// +// derived from https://stackoverflow.com/a/30038571 +// under CC-BY-SA-4.0 by several contributors +func FileCmp(file1, file2 string, chunkSize int) (same bool, err error) { + if chunkSize == 0 { + chunkSize = 4 * 1024 + } + + // shortcuts: check file metadata + stat1, err := os.Stat(file1) + if err != nil { + return false, err + } + + stat2, err := os.Stat(file2) + if err != nil { + return false, err + } + + // are inputs are literally the same file? + if os.SameFile(stat1, stat2) { + return true, nil + } + + // do inputs at least have the same size? + if stat1.Size() != stat2.Size() { + return false, nil + } + + // long way: compare contents + f1, err := os.Open(file1) + if err != nil { + return false, err + } + defer f1.Close() + + f2, err := os.Open(file2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, chunkSize) + b2 := make([]byte, chunkSize) + for { + n1, err1 := io.ReadFull(f1, b1) + n2, err2 := io.ReadFull(f2, b2) + + // https://pkg.go.dev/io#Reader + // > Callers should always process the n > 0 bytes returned + // > before considering the error err. Doing so correctly + // > handles I/O errors that happen after reading some bytes + // > and also both of the allowed EOF behaviors. + + if !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + + if (err1 == io.EOF && err2 == io.EOF) || (err1 == io.ErrUnexpectedEOF && err2 == io.ErrUnexpectedEOF) { + return true, nil + } + + // some other error, like a dropped network connection or a bad transfer + if err1 != nil { + return false, err1 + } + if err2 != nil { + return false, err2 + } + } +} diff --git a/tests/integration/api_helper_for_declarative_test.go b/tests/integration/api_helper_for_declarative_test.go index 3524ce9834add..58f4d63f30bac 100644 --- a/tests/integration/api_helper_for_declarative_test.go +++ b/tests/integration/api_helper_for_declarative_test.go @@ -21,6 +21,7 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/services/forms" + "github.com/google/uuid" "github.com/stretchr/testify/assert" ) @@ -462,3 +463,27 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r ctx.Session.MakeRequest(t, req, http.StatusNoContent) } } + +// generate and activate an ssh key for the user attached to the APITestContext +// TODO: pick a better name; golang doesn't do method overriding. +func withCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { + // we need to have write:public_key to do this step + // the easiest way is to create a throwaway ctx that is identical but only has that permission + ctxKeyWriter := ctx + ctxKeyWriter.Token = getTokenForLoggedInUser(t, ctx.Session, auth.AccessTokenScopeWriteUser) + + keyName := "One of " + ctx.Username + "'s keys: #" + uuid.New().String() + withKeyFile(t, keyName, func(keyFile string) { + var key api.PublicKey + + doAPICreateUserKey(ctxKeyWriter, keyName, keyFile, + func(t *testing.T, _key api.PublicKey) { + // save the key ID so we can delete it at the end + key = _key + })(t) + + defer doAPIDeleteUserKey(ctxKeyWriter, key.ID)(t) + + callback() + }) +} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go new file mode 100644 index 0000000000000..ee7d4c17205b6 --- /dev/null +++ b/tests/integration/git_annex_test.go @@ -0,0 +1,759 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integration + +import ( + "errors" + "fmt" + "math/rand" + "net/url" + "os" + "path" + "regexp" + "strings" + "testing" + + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/perm" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/require" +) + +// Some guidelines: +// +// * a APITestContext is an awkward union of session credential + username + target repo +// which is assumed to be owned by that username; if you want to target a different +// repo, you need to edit its .Reponame or just ignore it and write "username/reponame.git" + +func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, private bool) (err error) { + // creating a repo counts as editing the user's profile (is done by POSTing + // to /api/v1/user/repos/) -- which means it needs a User-scoped token and + // both that and editing need a Repo-scoped token because they edit repositories. + rescopedCtx := ctx + rescopedCtx.Token = getTokenForLoggedInUser(t, ctx.Session, auth_model.AccessTokenScopeWriteUser, auth_model.AccessTokenScopeWriteRepository) + doAPICreateRepository(rescopedCtx, false)(t) + doAPIEditRepository(rescopedCtx, &api.EditRepoOption{Private: &private})(t) + + repoURL := createSSHUrl(ctx.GitPath(), u) + + // Fill in fixture data + withAnnexCtxKeyFile(t, ctx, func() { + err = doInitRemoteAnnexRepository(t, repoURL) + }) + if err != nil { + return fmt.Errorf("Unable to initialize remote repo with git-annex fixture: %w", err) + } + return nil +} + +/* +Test that permissions are enforced on git-annex-shell commands. + + Along the way, test that uploading, downloading, and deleting all work. +*/ +func TestGitAnnexPermissions(t *testing.T) { + /* + // TODO: look into how LFS did this + if !setting.Annex.Enabled { + t.Skip() + } + */ + + // Each case below is split so that 'clone' is done as + // the repo owner, but 'copy' as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + t.Run("Public", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + ownerCtx := NewAPITestContext(t, "user2", "annex-public", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, false)) + + // double-check it's public + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.False(t, repo.IsPrivate) + + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Writer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Reader", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + }) + }) + + t.Run("Private", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + ownerCtx := NewAPITestContext(t, "user2", "annex-private", auth_model.AccessTokenScopeWriteRepository) + + // create a private repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, true)) + + // double-check it's private + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.True(t, repo.IsPrivate) + + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + // Note: there's also full anonymous access, which is only available for public HTTP repos; + // it should behave the same as 'outsider' but we (will) test it separately below anyway + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Writer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Reader", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + }) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + }) + }) + }) +} + +/* +Test that 'git annex init' works. + + precondition: repoPath contains a pre-cloned repo set up by doInitAnnexRepository(). +*/ +func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { + _, _, err = git.NewCommand(git.DefaultContext, "annex", "init", "cloned-repo").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't `git annex init`: %w", err) + } + + // - method 0: 'git config remote.origin.annex-uuid'. + // Demonstrates that 'git annex init' successfully contacted + // the remote git-annex and was able to learn its ID number. + readAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't read remote `git config remote.origin.annex-uuid`: %w", err) + } + readAnnexUUID = strings.TrimSpace(readAnnexUUID) + + match := regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$").MatchString(readAnnexUUID) + if !match { + return fmt.Errorf("'git config remote.origin.annex-uuid' should have been able to download the remote's uuid; but instead read '%s'", readAnnexUUID) + } + + remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) + if err != nil { + return fmt.Errorf("Couldn't read local `git config annex.uuid`: %w", err) + } + + remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) + match = regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$").MatchString(remoteAnnexUUID) + if !match { + return fmt.Errorf("'git annex init' should have been able to download the remote's uuid; but instead read '%s'", remoteAnnexUUID) + } + + if readAnnexUUID != remoteAnnexUUID { + return fmt.Errorf("'git annex init' should have read the expected annex UUID '%s', but instead got '%s'", remoteAnnexUUID, readAnnexUUID) + } + + // - method 1: 'git annex whereis'. + // Demonstrates that git-annex understands the annexed file can be found in the remote annex. + annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't `git annex whereis large.bin`: %w", err) + } + // Note: this regex is unanchored because 'whereis' outputs multiple lines containing + // headers and 1+ remotes and we just want to find one of them. + match = regexp.MustCompile(regexp.QuoteMeta(remoteAnnexUUID) + " -- .* \\[origin\\]\n").MatchString(annexWhereis) + if !match { + return errors.New("'git annex whereis' should report large.bin is known to be in [origin]") + } + + return nil +} + +func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { + // NB: this test does something slightly different if run separately from "doAnnexInitTest()": + // "git annex copy" will notice and run "git annex init", silently. + // This shouldn't change any results, but be aware in case it does. + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // verify the file was downloaded + localObjectPath, err := annexObjectPath(repoPath, "large.bin") + if err != nil { + return err + } + // localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file + + remoteObjectPath, err := annexObjectPath(remoteRepoPath, "large.bin") + if err != nil { + return err + } + + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil +} + +func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { + // NB: this test does something slightly different if run separately from "Init": + // it first runs "git annex init" silently in the background. + // This shouldn't change any results, but be aware in case it does. + + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + if err != nil { + return err + } + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex another file"}) + if err != nil { + return err + } + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // verify the file was uploaded + localObjectPath, err := annexObjectPath(repoPath, "contribution.bin") + if err != nil { + return err + } + // localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file + + remoteObjectPath, err := annexObjectPath(remoteRepoPath, "contribution.bin") + if err != nil { + return err + } + + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil +} + +// ---- Helpers ---- + +func generateRandomFile(size int, path string) (err error) { + // Generate random file + + // XXX TODO: maybe this should not be random, but instead a predictable pattern, so that the test is deterministic + bufSize := 4 * 1024 + if bufSize > size { + bufSize = size + } + + buffer := make([]byte, bufSize) + + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + written := 0 + for written < size { + n := size - written + if n > bufSize { + n = bufSize + } + _, err := rand.Read(buffer[:n]) + if err != nil { + return err + } + n, err = f.Write(buffer[:n]) + if err != nil { + return err + } + written += n + } + if err != nil { + return err + } + + return nil +} + +// ---- Annex-specific helpers ---- + +/* +Initialize a repo with some baseline annexed and non-annexed files. + + TODO: perhaps this generator could be replaced with a fixture (see + integrations/gitea-repositories-meta/ and models/fixtures/repository.yml). + However we reuse this template for -different- repos, so maybe not. +*/ +func doInitAnnexRepository(repoPath string) error { + // set up what files should be annexed + // in this case, all *.bin files will be annexed + // without this, git-annex's default config annexes every file larger than some number of megabytes + f, err := os.Create(path.Join(repoPath, ".gitattributes")) + if err != nil { + return err + } + defer f.Close() + + // set up git-annex to store certain filetypes via *annex* pointers + // (https://git-annex.branchable.com/internals/pointer_file/). + // but only when run via 'git add' (see git-annex-smudge(1)) + _, err = f.WriteString("* annex.largefiles=anything\n") + if err != nil { + return err + } + _, err = f.WriteString("*.bin filter=annex\n") + if err != nil { + return err + } + f.Close() + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Configure git-annex settings"}) + if err != nil { + return err + } + + // 'git annex init' + err = git.NewCommand(git.DefaultContext, "annex", "init", "test-repo").Run(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // add a file to the annex + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin")) + if err != nil { + return err + } + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"}) + if err != nil { + return err + } + + return nil +} + +/* +Initialize a remote repo with some baseline annexed and non-annexed files. +*/ +func doInitRemoteAnnexRepository(t *testing.T, repoURL *url.URL) error { + repoPath := path.Join(t.TempDir(), path.Base(repoURL.Path)) + // This clone is immediately thrown away, which + // helps force the tests to be end-to-end. + defer util.RemoveAll(repoPath) + + doGitClone(repoPath, repoURL)(t) // TODO: this call is the only reason for the testing.T; can it be removed? + + err := doInitAnnexRepository(repoPath) + if err != nil { + return err + } + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + return nil +} + +/* +Find the path in .git/annex/objects/ of the contents for a given annexed file. + + repoPath: the git repository to examine + file: the path (in the repo's current HEAD) of the annex pointer + + TODO: pass a parameter to allow examining non-HEAD branches +*/ +func annexObjectPath(repoPath, file string) (string, error) { + // NB: `git annex lookupkey` is more reliable, but doesn't work in bare repos. + annexKey, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "show").AddDynamicArguments("HEAD:" + file).RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return "", fmt.Errorf("in %s: %w", repoPath, err) // the error from git prints the filename but not repo + } + + // There are two formats an annexed file pointer might be: + // * a symlink to .git/annex/objects/$HASHDIR/$ANNEX_KEY/$ANNEX_KEY - used by files created with 'git annex add' + // * a text file containing /annex/objects/$ANNEX_KEY - used by files for which 'git add' was configured to run git-annex-smudge + // This recovers $ANNEX_KEY from either case: + annexKey = path.Base(strings.TrimSpace(annexKey)) + + contentPath, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(annexKey).RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return "", fmt.Errorf("in %s: %s does not seem to be annexed: %w", repoPath, file, err) + } + contentPath = strings.TrimSpace(contentPath) + + return path.Join(repoPath, contentPath), nil +} + +/* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ +func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { + _gitAnnexUseGitSSH, gitAnnexUseGitSSHExists := os.LookupEnv("GIT_ANNEX_USE_GIT_SSH") + defer func() { + // reset + if gitAnnexUseGitSSHExists { + os.Setenv("GIT_ANNEX_USE_GIT_SSH", _gitAnnexUseGitSSH) + } + }() + + os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set + + withCtxKeyFile(t, ctx, callback) +} diff --git a/tests/integration/git_helper_for_declarative_test.go b/tests/integration/git_helper_for_declarative_test.go index 10cf79b9fd8b2..e959e2e06cfa2 100644 --- a/tests/integration/git_helper_for_declarative_test.go +++ b/tests/integration/git_helper_for_declarative_test.go @@ -39,6 +39,28 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { "ssh -o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\" -o \"IdentitiesOnly=yes\" -i \""+keyFile+"\" \"$@\""), 0o700) assert.NoError(t, err) + // reset ssh wrapper afterwards + _gitSSH, gitSSHExists := os.LookupEnv("GIT_SSH") + defer func() { + if gitSSHExists { + os.Setenv("GIT_SSH", _gitSSH) + } + }() + + _gitSSHCommand, gitSSHCommandExists := os.LookupEnv("GIT_SSH_COMMAND") + defer func() { + if gitSSHCommandExists { + os.Setenv("GIT_SSH_COMMAND", _gitSSHCommand) + } + }() + + _gitSSHVariant, gitSSHVariantExists := os.LookupEnv("GIT_SSH_VARIANT") + defer func() { + if gitSSHVariantExists { + os.Setenv("GIT_SSH_VARIANT", _gitSSHVariant) + } + }() + // Setup ssh wrapper os.Setenv("GIT_SSH", path.Join(tmpDir, "ssh")) os.Setenv("GIT_SSH_COMMAND", From 39e65a00c804d1e0f5593a3a0f0de73049e61070 Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 19 Sep 2022 17:22:42 -0400 Subject: [PATCH 03/13] git-annex: add configuration setting [annex].ENABLED (#18) Fixes https://github.com/neuropoly/gitea/issues/8 Co-authored-by: Mathieu Guay-Paquet --- cmd/serv.go | 3 +-- cmd/web.go | 4 ++++ custom/conf/app.example.ini | 9 +++++++++ modules/setting/annex.go | 20 ++++++++++++++++++++ modules/setting/setting.go | 1 + routers/private/serv.go | 2 +- tests/integration/git_annex_test.go | 9 +++------ tests/mssql.ini.tmpl | 3 +++ tests/mysql.ini.tmpl | 3 +++ tests/pgsql.ini.tmpl | 3 +++ tests/sqlite.ini.tmpl | 3 +++ 11 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 modules/setting/annex.go diff --git a/cmd/serv.go b/cmd/serv.go index 20ca4484c386d..48489501e13b2 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -220,8 +220,7 @@ func runServ(c *cli.Context) error { } if verb == gitAnnexShellVerb { - // if !setting.Annex.Enabled { // TODO: https://github.com/neuropoly/gitea/issues/8 - if false { + if !setting.Annex.Enabled { return fail(ctx, "Unknown git command", "git-annex request over SSH denied, git-annex support is disabled") } diff --git a/cmd/web.go b/cmd/web.go index 01386251becfa..ce22d708acccb 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -311,6 +311,10 @@ func listen(m http.Handler, handleRedirector bool) error { log.Info("LFS server enabled") } + if setting.Annex.Enabled { + log.Info("git-annex enabled") + } + var err error switch setting.Protocol { case setting.HTTP: diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index a4e777fa12894..f8a7505b051b3 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2520,6 +2520,15 @@ LEVEL = Info ;; override the minio base path if storage type is minio ;MINIO_BASE_PATH = lfs/ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;[annex] +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Whether git-annex is enabled; defaults to false +;ENABLED = false + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; settings for packages, will override storage setting diff --git a/modules/setting/annex.go b/modules/setting/annex.go new file mode 100644 index 0000000000000..a0eeac9bb8a1c --- /dev/null +++ b/modules/setting/annex.go @@ -0,0 +1,20 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "code.gitea.io/gitea/modules/log" +) + +// Annex represents the configuration for git-annex +var Annex = struct { + Enabled bool `ini:"ENABLED"` +}{} + +func loadAnnexFrom(rootCfg ConfigProvider) { + sec := rootCfg.Section("annex") + if err := sec.MapTo(&Annex); err != nil { + log.Fatal("Failed to map Annex settings: %v", err) + } +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index d444d9a0175c6..2b201bb51c01b 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -149,6 +149,7 @@ func loadCommonSettingsFrom(cfg ConfigProvider) error { loadCamoFrom(cfg) loadI18nFrom(cfg) loadGitFrom(cfg) + loadAnnexFrom(cfg) loadMirrorFrom(cfg) loadMarkupFrom(cfg) loadOtherFrom(cfg) diff --git a/routers/private/serv.go b/routers/private/serv.go index ad33d86e1462a..df56553eeac63 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -289,7 +289,7 @@ func ServCommand(ctx *context.PrivateContext) { repo.IsPrivate || owner.Visibility.IsPrivate() || (user != nil && user.IsRestricted) || // user will be nil if the key is a deploykey - ( /*setting.Annex.Enabled && */ len(verbs) > 0 && verbs[0] == "git-annex-shell") || // git-annex has its own permission enforcement, for which we expose results.UserMode + (setting.Annex.Enabled && len(verbs) > 0 && verbs[0] == "git-annex-shell") || // git-annex has its own permission enforcement, for which we expose results.UserMode setting.Service.RequireSignInView) { if key.Type == asymkey_model.KeyTypeDeploy { results.UserMode = deployKey.Mode diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index ee7d4c17205b6..33ababb0f6e3f 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -61,12 +61,9 @@ Test that permissions are enforced on git-annex-shell commands. Along the way, test that uploading, downloading, and deleting all work. */ func TestGitAnnexPermissions(t *testing.T) { - /* - // TODO: look into how LFS did this - if !setting.Annex.Enabled { - t.Skip() - } - */ + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } // Each case below is split so that 'clone' is done as // the repo owner, but 'copy' as the user under test. diff --git a/tests/mssql.ini.tmpl b/tests/mssql.ini.tmpl index 3cd64ec5cb8ca..5933e865aba7f 100644 --- a/tests/mssql.ini.tmpl +++ b/tests/mssql.ini.tmpl @@ -104,6 +104,9 @@ INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.h [lfs] PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mssql/data/lfs +[annex] +ENABLED = true + [packages] ENABLED = true diff --git a/tests/mysql.ini.tmpl b/tests/mysql.ini.tmpl index 2f890e67eb926..b9b83cc4f654a 100644 --- a/tests/mysql.ini.tmpl +++ b/tests/mysql.ini.tmpl @@ -102,6 +102,9 @@ INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.h [lfs] PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mysql/data/lfs +[annex] +ENABLED = true + [packages] ENABLED = true diff --git a/tests/pgsql.ini.tmpl b/tests/pgsql.ini.tmpl index a1679cad6a6e9..ad4c7b646c61c 100644 --- a/tests/pgsql.ini.tmpl +++ b/tests/pgsql.ini.tmpl @@ -105,6 +105,9 @@ INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.h [lfs] MINIO_BASE_PATH = lfs/ +[annex] +ENABLED = true + [attachment] MINIO_BASE_PATH = attachments/ diff --git a/tests/sqlite.ini.tmpl b/tests/sqlite.ini.tmpl index 74e1957113150..3f7a40f72c7dc 100644 --- a/tests/sqlite.ini.tmpl +++ b/tests/sqlite.ini.tmpl @@ -104,6 +104,9 @@ JWT_SECRET = KZb_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko [lfs] PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-sqlite/data/lfs +[annex] +ENABLED = true + [packages] ENABLED = true From 60f0d4a3f494a209a5263fc03b5d78bbdcb644cb Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 20 Sep 2022 16:17:56 -0400 Subject: [PATCH 04/13] git-annex: support downloading over HTTP (#6) This makes HTTP symmetric with SSH clone URLs. This gives us the fancy feature of _anonymous_ downloads, so people can access datasets without having to set up an account or manage ssh keys. Previously, to access "open access" data shared this way, users would need to: 1. Create an account on gitea.example.com 2. Create ssh keys 3. Upload ssh keys (and make sure to find and upload the correct file) 4. `git clone git@gitea.example.com:user/dataset.git` 5. `cd dataset` 6. `git annex get` This cuts that down to just the last three steps: 1. `git clone https://gitea.example.com/user/dataset.git` 2. `cd dataset` 3. `git annex get` This is significantly simpler for downstream users, especially for those unfamiliar with the command line. Unfortunately there's no uploading. While git-annex supports uploading over HTTP to S3 and some other special remotes, it seems to fail on a _plain_ HTTP remote. See https://github.com/neuropoly/gitea/issues/7 and https://git-annex.branchable.com/forum/HTTP_uploads/#comment-ce28adc128fdefe4c4c49628174d9b92. This is not a major loss since no one wants uploading to be anonymous anyway. To support private repos, I had to hunt down and patch a secret extra security corner that Gitea only applies to HTTP for some reason (services/auth/basic.go). This was guided by https://git-annex.branchable.com/tips/setup_a_public_repository_on_a_web_site/ Fixes https://github.com/neuropoly/gitea/issues/3 Co-authored-by: Mathieu Guay-Paquet --- modules/git/command.go | 3 +- routers/web/repo/githttp.go | 31 ++ routers/web/web.go | 13 + services/auth/auth.go | 11 + services/auth/basic.go | 4 +- tests/integration/git_annex_test.go | 360 +++++++++++++++++- .../git_helper_for_declarative_test.go | 7 + 7 files changed, 412 insertions(+), 17 deletions(-) diff --git a/modules/git/command.go b/modules/git/command.go index f095bb18bef75..8550f7759451a 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -439,12 +439,13 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS } // AllowLFSFiltersArgs return globalCommandArgs with lfs filter, it should only be used for tests +// It also re-enables git-credential(1), which is used to test git-annex's HTTP support func AllowLFSFiltersArgs() TrustedCmdArgs { // Now here we should explicitly allow lfs filters to run filteredLFSGlobalArgs := make(TrustedCmdArgs, len(globalCommandArgs)) j := 0 for _, arg := range globalCommandArgs { - if strings.Contains(string(arg), "lfs") { + if strings.Contains(string(arg), "lfs") || strings.Contains(string(arg), "credential") { j-- } else { filteredLFSGlobalArgs[j] = arg diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index 6ff385f989050..41dcc6f6b95e9 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -611,3 +611,34 @@ func GetIdxFile(ctx *context.Context) { h.sendFile("application/x-git-packed-objects-toc", "objects/pack/pack-"+ctx.Params("file")+".idx") } } + +// GetAnnexObject implements git-annex dumb HTTP +func GetAnnexObject(ctx *context.Context) { + h := httpBase(ctx) + if h != nil { + // git-annex objects are stored in .git/annex/objects/{hash1}/{hash2}/{key}/{key} + // where key is a string containing the size and (usually SHA256) checksum of the file, + // and hash1+hash2 are the first few bits of the md5sum of key itself. + // ({hash1}/{hash2}/ is just there to avoid putting too many files in one directory) + // ref: https://git-annex.branchable.com/internals/hashing/ + + // keyDir should = key, but we don't enforce that + object := path.Join(ctx.Params("hash1"), ctx.Params("hash2"), ctx.Params("keyDir"), ctx.Params("key")) + + // Sanitize the input against directory traversals. + // + // This works because at the filesystem root, "/.." = "/"; + // So if a path starts rooted ("/"), path.Clean(), which + // path.Join() calls internally, removes all '..' prefixes. + // After, this unroots the path unconditionally ([1:]), which + // works because we know the input is never supposed to be rooted. + // + // The router code probably also disallows "..", so this + // should be redundant, but it's defensive to keep it + // whenever touching filesystem paths with user input. + object = path.Join("/", object)[1:] + + h.setHeaderCacheForever() + h.sendFile("application/octet-stream", "annex/objects/"+object) + } +} diff --git a/routers/web/web.go b/routers/web/web.go index f8b745fb10b55..d7bcdef290492 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -331,6 +331,13 @@ func registerRoutes(m *web.Route) { } } + annexEnabled := func(ctx *context.Context) { + if !setting.Annex.Enabled { + ctx.Error(http.StatusNotFound) + return + } + } + federationEnabled := func(ctx *context.Context) { if !setting.Federation.Enabled { ctx.Error(http.StatusNotFound) @@ -1514,6 +1521,12 @@ func registerRoutes(m *web.Route) { }) }, ignSignInAndCsrf, lfsServerEnabled) + m.Group("", func() { + // for git-annex + m.GetOptions("/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` + m.GetOptions("/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) + }, ignSignInAndCsrf, annexEnabled, context_service.UserAssignmentWeb()) + gitHTTPRouters(m) }) }) diff --git a/services/auth/auth.go b/services/auth/auth.go index 713463a3d47ed..b391486dd5033 100644 --- a/services/auth/auth.go +++ b/services/auth/auth.go @@ -54,6 +54,17 @@ func isGitRawOrAttachOrLFSPath(req *http.Request) bool { return false } +var annexPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/annex/`) + +func isAnnexPath(req *http.Request) bool { + if setting.Annex.Enabled { + // "/config" is git's config, not specifically git-annex's; but the only current + // user of it is when git-annex downloads the annex.uuid during 'git annex init'. + return strings.HasSuffix(req.URL.Path, "/config") || annexPathRe.MatchString(req.URL.Path) + } + return false +} + // handleSignIn clears existing session variables and stores new ones for the specified user object func handleSignIn(resp http.ResponseWriter, req *http.Request, sess SessionStore, user *user_model.User) { // We need to regenerate the session... diff --git a/services/auth/basic.go b/services/auth/basic.go index 1184d12d1c4b4..4aa7f3b8d20e3 100644 --- a/services/auth/basic.go +++ b/services/auth/basic.go @@ -42,8 +42,8 @@ func (b *Basic) Name() string { // name/token on successful validation. // Returns nil if header is empty or validation fails. func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) { - // Basic authentication should only fire on API, Download or on Git or LFSPaths - if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) { + // Basic authentication should only fire on API, Download or on Git or LFSPaths or Git-Annex paths + if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) && !isAnnexPath(req) { return nil, nil } diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 33ababb0f6e3f..634c3dfa0ecb8 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -58,7 +58,8 @@ func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, /* Test that permissions are enforced on git-annex-shell commands. - Along the way, test that uploading, downloading, and deleting all work. + Along the way, this also tests that uploading, downloading, and deleting all work, + so we haven't written separate tests for those. */ func TestGitAnnexPermissions(t *testing.T) { if !setting.Annex.Enabled { @@ -74,6 +75,16 @@ func TestGitAnnexPermissions(t *testing.T) { // 'annex copy' -- potentially leaving a security gap. onGiteaRun(t, func(t *testing.T, u *url.URL) { + // Tell git-annex to allow http://127.0.0.1, http://localhost and http://::1. Without + // this, all `git annex` commands will silently fail when run against http:// remotes + // without explaining what's wrong. + // + // Note: onGiteaRun() sets up an alternate HOME so this actually edits + // tests/integration/gitea-integration-*/data/home/.gitconfig and + // if you're debugging you need to remember to match that. + _, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddArguments("annex.security.allowed-ip-addresses", "all").RunStdString(&git.RunOpts{}) + require.NoError(t, err) + t.Run("Public", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -87,8 +98,6 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, err) require.False(t, repo.IsPrivate) - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost // Different sessions, so we can test different permissions. @@ -110,6 +119,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -137,6 +148,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -145,6 +181,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -172,6 +210,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -180,6 +243,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -207,6 +272,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Outsider", func(t *testing.T) { @@ -215,6 +305,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -242,6 +334,61 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Anonymous", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Only HTTP has an anonymous mode + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) }) t.Run("Delete", func(t *testing.T) { @@ -267,8 +414,6 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, err) require.True(t, repo.IsPrivate) - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost // Different sessions, so we can test different permissions. @@ -292,6 +437,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -319,6 +466,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -327,6 +499,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -354,6 +528,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -362,6 +561,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -389,6 +590,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Outsider", func(t *testing.T) { @@ -397,6 +623,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -424,6 +652,61 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Anonymous", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Only HTTP has an anonymous mode + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) }) t.Run("Delete", func(t *testing.T) { @@ -444,7 +727,7 @@ Test that 'git annex init' works. precondition: repoPath contains a pre-cloned repo set up by doInitAnnexRepository(). */ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { - _, _, err = git.NewCommand(git.DefaultContext, "annex", "init", "cloned-repo").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "init", "cloned-repo").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't `git annex init`: %w", err) } @@ -452,7 +735,7 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { // - method 0: 'git config remote.origin.annex-uuid'. // Demonstrates that 'git annex init' successfully contacted // the remote git-annex and was able to learn its ID number. - readAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + readAnnexUUID, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't read remote `git config remote.origin.annex-uuid`: %w", err) } @@ -463,7 +746,7 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { return fmt.Errorf("'git config remote.origin.annex-uuid' should have been able to download the remote's uuid; but instead read '%s'", readAnnexUUID) } - remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) + remoteAnnexUUID, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) if err != nil { return fmt.Errorf("Couldn't read local `git config annex.uuid`: %w", err) } @@ -480,7 +763,7 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { // - method 1: 'git annex whereis'. // Demonstrates that git-annex understands the annexed file can be found in the remote annex. - annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't `git annex whereis large.bin`: %w", err) } @@ -499,7 +782,7 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { // "git annex copy" will notice and run "git annex init", silently. // This shouldn't change any results, but be aware in case it does. - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -547,12 +830,12 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -664,7 +947,7 @@ func doInitAnnexRepository(repoPath string) error { } // 'git annex init' - err = git.NewCommand(git.DefaultContext, "annex", "init", "test-repo").Run(&git.RunOpts{Dir: repoPath}) + err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "init", "test-repo").Run(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -702,7 +985,7 @@ func doInitRemoteAnnexRepository(t *testing.T, repoURL *url.URL) error { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -754,3 +1037,52 @@ func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { withCtxKeyFile(t, ctx, callback) } + +/* +Like withKeyFile(), but sets HTTP credentials instead of SSH credentials. + + It does this by temporarily arranging through `git config --global` + to use git-credential-store(1) with the password written to a tempfile. + + This is the only reliable way to pass HTTP credentials non-interactively + to git-annex. See https://git-annex.branchable.com/bugs/http_remotes_ignore_annex.web-options_--netrc/#comment-b5a299e9826b322f2d85c96d4929a430 + for joeyh's proclamation on the subject. + + This **is only effective** when used around git.NewCommandContextNoGlobals() calls. + git.NewCommand() disables credential.helper as a precaution (see modules/git/git.go). + + In contrast, the tests in git_test.go put the password in the remote's URL like + `git config remote.origin.url http://user2:password@localhost:3003/user2/repo-name.git`, + writing the password in repoPath+"/.git/config". That would be equally good, except + that git-annex ignores it! +*/ +func withAnnexCtxHTTPPassword(t *testing.T, u *url.URL, ctx APITestContext, callback func()) { + credentialedURL := *u + credentialedURL.User = url.UserPassword(ctx.Username, userPassword) // NB: all test users use the same password + + creds := path.Join(t.TempDir(), "creds") + require.NoError(t, os.WriteFile(creds, []byte(credentialedURL.String()), 0o600)) + + originalCredentialHelper, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper").RunStdString(&git.RunOpts{}) + if err != nil && !err.IsExitCode(1) { + // ignore the 'error' thrown when credential.helper is unset (when git config returns 1) + // but catch all others + require.NoError(t, err) + } + hasOriginalCredentialHelper := (err == nil) + + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper", fmt.Sprintf("store --file=%s", creds)).RunStdString(&git.RunOpts{}) + require.NoError(t, err) + + defer (func() { + // reset + if hasOriginalCredentialHelper { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddArguments("credential.helper").AddDynamicArguments(originalCredentialHelper).RunStdString(&git.RunOpts{}) + } else { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddOptionValues("--unset").AddArguments("credential.helper").RunStdString(&git.RunOpts{}) + } + require.NoError(t, err) + })() + + callback() +} diff --git a/tests/integration/git_helper_for_declarative_test.go b/tests/integration/git_helper_for_declarative_test.go index e959e2e06cfa2..4d91c4d78b0c2 100644 --- a/tests/integration/git_helper_for_declarative_test.go +++ b/tests/integration/git_helper_for_declarative_test.go @@ -70,6 +70,13 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { callback(keyFile) } +func createHTTPUrl(gitPath string, u *url.URL) *url.URL { + // this assumes u contains the HTTP base URL that Gitea is running on + u2 := *u + u2.Path = gitPath + return &u2 +} + func createSSHUrl(gitPath string, u *url.URL) *url.URL { u2 := *u u2.Scheme = "ssh" From 77547cc97e63bfc8e8a045e0ed99000bb7cc9fc1 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 27 Nov 2022 00:28:55 -0500 Subject: [PATCH 05/13] git-annex: create modules/annex (#21) This moves the `annexObjectPath()` helper out of the tests and into a dedicated sub-package as `annex.ContentLocation()`, and expands it with `.Pointer()` (which validates using `git annex examinekey`), `.IsAnnexed()` and `.Content()` to make it a more useful module. The tests retain their own wrapper version of `ContentLocation()` because I tried to follow close to the API modules/lfs uses, which in terms of abstract `git.Blob` and `git.TreeEntry` objects, not in terms of `repoPath string`s which are more convenient for the tests. --- modules/annex/annex.go | 154 ++++++++++++++++++++++++++++ modules/git/blob.go | 4 + modules/git/blob_gogit.go | 3 +- modules/git/repo_blob_gogit.go | 1 + modules/git/tree_entry_gogit.go | 1 + tests/integration/git_annex_test.go | 39 ++++--- 6 files changed, 184 insertions(+), 18 deletions(-) create mode 100644 modules/annex/annex.go diff --git a/modules/annex/annex.go b/modules/annex/annex.go new file mode 100644 index 0000000000000..bb049d77ed686 --- /dev/null +++ b/modules/annex/annex.go @@ -0,0 +1,154 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +// Unlike modules/lfs, which operates mainly on git.Blobs, this operates on git.TreeEntrys. +// The motivation for this is that TreeEntrys have an easy pointer to the on-disk repo path, +// while blobs do not (in fact, if building with TAGS=gogit, blobs might exist only in a mock +// filesystem, living only in process RAM). We must have the on-disk path to do anything +// useful with git-annex because all of its interesting data is on-disk under .git/annex/. + +package annex + +import ( + "errors" + "fmt" + "os" + "path" + "strings" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +const ( + // > The maximum size of a pointer file is 32 kb. + // - https://git-annex.branchable.com/internals/pointer_file/ + // It's unclear if that's kilobytes or kibibytes; assuming kibibytes: + blobSizeCutoff = 32 * 1024 +) + +// ErrInvalidPointer occurs if the pointer's value doesn't parse +var ErrInvalidPointer = errors.New("Not a git-annex pointer") + +// Gets the content of the blob as raw text, up to n bytes. +// (the pre-existing blob.GetBlobContent() has a hardcoded 1024-byte limit) +func getBlobContent(b *git.Blob, n int) (string, error) { + dataRc, err := b.DataAsync() + if err != nil { + return "", err + } + defer dataRc.Close() + buf := make([]byte, n) + n, _ = util.ReadAtMost(dataRc, buf) + buf = buf[:n] + return string(buf), nil +} + +func Pointer(blob *git.Blob) (string, error) { + // git-annex doesn't seem fully spec what its pointer are, but + // the fullest description is here: + // https://git-annex.branchable.com/internals/pointer_file/ + + // a pointer can be: + // the original format, generated by `git annex add`: a symlink to '.git/annex/objects/$HASHDIR/$HASHDIR2/$KEY/$KEY' + // the newer, git-lfs influenced, format, generated by `git annex smudge`: a text file containing '/annex/objects/$KEY' + // + // in either case we can extract the $KEY the same way, and we need not actually know if it's a symlink or not because + // git.Blob.DataAsync() works like open() + readlink(), handling both cases in one. + + if blob.Size() > blobSizeCutoff { + // > The maximum size of a pointer file is 32 kb. If it is any longer, it is not considered to be a valid pointer file. + // https://git-annex.branchable.com/internals/pointer_file/ + + // It's unclear to me whether the same size limit applies to symlink-pointers, but it seems sensible to limit them too. + return "", ErrInvalidPointer + } + + pointer, err := getBlobContent(blob, blobSizeCutoff) + if err != nil { + return "", fmt.Errorf("error reading %s: %w", blob.Name(), err) + } + + // the spec says a pointer file can contain multiple lines each with a pointer in them + // but that makes no sense to me, so I'm just ignoring all but the first + lines := strings.Split(pointer, "\n") + if len(lines) < 1 { + return "", ErrInvalidPointer + } + pointer = lines[0] + + // in both the symlink and pointer-file formats, the pointer must have "/annex/" somewhere in it + if !strings.Contains(pointer, "/annex/") { + return "", ErrInvalidPointer + } + + // extract $KEY + pointer = path.Base(strings.TrimSpace(pointer)) + + // ask git-annex's opinion on $KEY + // XXX: this is probably a bit slow, especially if this operation gets run often + // and examinekey is not that strict: + // - it doesn't enforce that the "BACKEND" tag is one it knows, + // - it doesn't enforce that the fields and their format fit the "BACKEND" tag + // so maybe this is a wasteful step + _, examineStderr, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "examinekey").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) + if err != nil { + // TODO: make ErrInvalidPointer into a type capable of wrapping err + if strings.TrimSpace(examineStderr) == "git-annex: bad key" { + return "", ErrInvalidPointer + } + return "", err + } + + return pointer, nil +} + +// return the absolute path of the content pointed to by the annex pointer stored in the git object +// errors if the content is not found in this repo +func ContentLocation(blob *git.Blob) (string, error) { + pointer, err := Pointer(blob) + if err != nil { + return "", err + } + + contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) + if err != nil { + return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", blob.Repo().Path, pointer, err) + } + contentLocation = strings.TrimSpace(contentLocation) + contentLocation = path.Clean("/" + contentLocation)[1:] // prevent directory traversals + contentLocation = path.Join(blob.Repo().Path, contentLocation) + + return contentLocation, nil +} + +// returns a stream open to the annex content +func Content(blob *git.Blob) (*os.File, error) { + contentLocation, err := ContentLocation(blob) + if err != nil { + return nil, err + } + + return os.Open(contentLocation) +} + +// whether the object appears to be a valid annex pointer +// does *not* verify if the content is actually in this repo; +// for that, use ContentLocation() +func IsAnnexed(blob *git.Blob) (bool, error) { + if !setting.Annex.Enabled { + return false, nil + } + + // Pointer() is written to only return well-formed pointers + // so the test is just to see if it errors + _, err := Pointer(blob) + if err != nil { + if errors.Is(err, ErrInvalidPointer) { + return false, nil + } + return false, err + } + return true, nil +} diff --git a/modules/git/blob.go b/modules/git/blob.go index bcecb42e16ebb..34224f6c085b0 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -15,6 +15,10 @@ import ( // This file contains common functions between the gogit and !gogit variants for git Blobs +func (b *Blob) Repo() *Repository { + return b.repo +} + // Name returns name of the tree entry this blob object was created from (or empty string) func (b *Blob) Name() string { return b.name diff --git a/modules/git/blob_gogit.go b/modules/git/blob_gogit.go index aa206409d0b6f..f98a9d9084f99 100644 --- a/modules/git/blob_gogit.go +++ b/modules/git/blob_gogit.go @@ -14,7 +14,8 @@ import ( // Blob represents a Git object. type Blob struct { - ID SHA1 + ID SHA1 + repo *Repository gogitEncodedObj plumbing.EncodedObject name string diff --git a/modules/git/repo_blob_gogit.go b/modules/git/repo_blob_gogit.go index 7f0892f6f5e91..605c05072b771 100644 --- a/modules/git/repo_blob_gogit.go +++ b/modules/git/repo_blob_gogit.go @@ -17,6 +17,7 @@ func (repo *Repository) getBlob(id SHA1) (*Blob, error) { return &Blob{ ID: id, + repo: repo, gogitEncodedObj: encodedObj, }, nil } diff --git a/modules/git/tree_entry_gogit.go b/modules/git/tree_entry_gogit.go index 194dd12f7dbb1..0c08a766d8229 100644 --- a/modules/git/tree_entry_gogit.go +++ b/modules/git/tree_entry_gogit.go @@ -89,6 +89,7 @@ func (te *TreeEntry) Blob() *Blob { return &Blob{ ID: te.gogitTreeEntry.Hash, + repo: te.ptree.repo, gogitEncodedObj: encodedObj, name: te.Name(), } diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 634c3dfa0ecb8..2e1d0eef7df7f 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -788,13 +789,13 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { } // verify the file was downloaded - localObjectPath, err := annexObjectPath(repoPath, "large.bin") + localObjectPath, err := contentLocation(repoPath, "large.bin") if err != nil { return err } // localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file - remoteObjectPath, err := annexObjectPath(remoteRepoPath, "large.bin") + remoteObjectPath, err := contentLocation(remoteRepoPath, "large.bin") if err != nil { return err } @@ -841,13 +842,13 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { } // verify the file was uploaded - localObjectPath, err := annexObjectPath(repoPath, "contribution.bin") + localObjectPath, err := contentLocation(repoPath, "contribution.bin") if err != nil { return err } // localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file - remoteObjectPath, err := annexObjectPath(remoteRepoPath, "contribution.bin") + remoteObjectPath, err := contentLocation(remoteRepoPath, "contribution.bin") if err != nil { return err } @@ -1001,26 +1002,30 @@ Find the path in .git/annex/objects/ of the contents for a given annexed file. TODO: pass a parameter to allow examining non-HEAD branches */ -func annexObjectPath(repoPath, file string) (string, error) { - // NB: `git annex lookupkey` is more reliable, but doesn't work in bare repos. - annexKey, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "show").AddDynamicArguments("HEAD:" + file).RunStdString(&git.RunOpts{Dir: repoPath}) +func contentLocation(repoPath, file string) (path string, err error) { + path = "" + + repo, err := git.OpenRepository(git.DefaultContext, repoPath) + if err != nil { + return path, nil + } + + commitID, err := repo.GetRefCommitID("HEAD") // NB: to examine a *branch*, prefix with "refs/branch/", or call repo.GetBranchCommitID(); ditto for tags if err != nil { - return "", fmt.Errorf("in %s: %w", repoPath, err) // the error from git prints the filename but not repo + return path, nil } - // There are two formats an annexed file pointer might be: - // * a symlink to .git/annex/objects/$HASHDIR/$ANNEX_KEY/$ANNEX_KEY - used by files created with 'git annex add' - // * a text file containing /annex/objects/$ANNEX_KEY - used by files for which 'git add' was configured to run git-annex-smudge - // This recovers $ANNEX_KEY from either case: - annexKey = path.Base(strings.TrimSpace(annexKey)) + commit, err := repo.GetCommit(commitID) + if err != nil { + return path, nil + } - contentPath, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(annexKey).RunStdString(&git.RunOpts{Dir: repoPath}) + treeEntry, err := commit.GetTreeEntryByPath(file) if err != nil { - return "", fmt.Errorf("in %s: %s does not seem to be annexed: %w", repoPath, file, err) + return path, nil } - contentPath = strings.TrimSpace(contentPath) - return path.Join(repoPath, contentPath), nil + return annex.ContentLocation(treeEntry.Blob()) } /* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ From a782b886d1421ff1ec2de2acb4baadc616e29890 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 27 Nov 2022 00:40:06 -0500 Subject: [PATCH 06/13] git-annex: make /media/ download annexed content (#20) Previously, Gitea's LFS support allowed direct-downloads of LFS content, via http://$HOSTNAME:$PORT/$USER/$REPO/media/branch/$BRANCH/$FILE Expand that grace to git-annex too. Now /media should provide the relevant *content* from the .git/annex/objects/ folder. This adds tests too. And expands the tests to try symlink-based annexing, since /media implicitly supports both that and pointer-file-based annexing. --- routers/web/repo/download.go | 21 ++++ tests/integration/git_annex_test.go | 144 ++++++++++++++++++++++++---- 2 files changed, 147 insertions(+), 18 deletions(-) diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index a9e2e2b2fad77..45b975537a2f4 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -9,6 +9,7 @@ import ( "time" git_model "code.gitea.io/gitea/models/git" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/httpcache" @@ -79,6 +80,26 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified *time.Tim } closed = true + // check for git-annex files + // (this code is weirdly redundant because I'm trying not to delete any lines in order to make merges easier) + isAnnexed, err := annex.IsAnnexed(blob) + if err != nil { + ctx.ServerError("annex.IsAnnexed", err) + return err + } + if isAnnexed { + content, err := annex.Content(blob) + if err != nil { + // XXX are there any other possible failure cases here? + // there are, there could be unrelated io errors; those should be ctx.ServerError()s + ctx.NotFound("annex.Content", err) + return err + } + defer content.Close() + common.ServeContentByReadSeeker(ctx.Base, ctx.Repo.TreePath, lastModified, content) + return nil + } + return common.ServeBlob(ctx.Base, ctx.Repo.TreePath, blob, lastModified) } diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 2e1d0eef7df7f..c5376f07f0ed8 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -7,7 +7,9 @@ package integration import ( "errors" "fmt" + "io" "math/rand" + "net/http" "net/url" "os" "path" @@ -56,6 +58,63 @@ func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, return nil } +func TestGitAnnexMedia(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + ctx := NewAPITestContext(t, "user2", "annex-media-test", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + + // the filenames here correspond to specific cases defined in doInitAnnexRepository() + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.bin") + }) + }) +} + +func doAnnexMediaTest(t *testing.T, ctx APITestContext, file string) { + // Make sure that downloading via /media on the website recognizes it should give the annexed content + + // TODO: + // - [ ] roll this into TestGitAnnexPermissions to ensure that permission enforcement works correctly even on /media? + + session := loginUser(t, ctx.Username) // logs in to the http:// site/API, storing a cookie; + // this is a different auth method than the git+ssh:// or git+http:// protocols TestGitAnnexPermissions uses! + + // compute server-side path of the annexed file + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + remoteObjectPath, err := contentLocation(remoteRepoPath, file) + require.NoError(t, err) + + // download annexed file + localObjectPath := path.Join(t.TempDir(), file) + fd, err := os.OpenFile(localObjectPath, os.O_CREATE|os.O_WRONLY, 0o777) + defer fd.Close() + require.NoError(t, err) + + mediaLink := path.Join("/", ctx.Username, ctx.Reponame, "/media/branch/master", file) + req := NewRequest(t, "GET", mediaLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + _, err = io.Copy(fd, resp.Body) + require.NoError(t, err) + fd.Close() + + // verify the download + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + require.NoError(t, err) + require.True(t, match, "Annexed files should be the same") +} + /* Test that permissions are enforced on git-annex-shell commands. @@ -763,16 +822,16 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { } // - method 1: 'git annex whereis'. - // Demonstrates that git-annex understands the annexed file can be found in the remote annex. - annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + // Demonstrates that git-annex understands annexed files can be found in the remote annex. + annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "annexed.bin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { - return fmt.Errorf("Couldn't `git annex whereis large.bin`: %w", err) + return fmt.Errorf("Couldn't `git annex whereis`: %w", err) } // Note: this regex is unanchored because 'whereis' outputs multiple lines containing // headers and 1+ remotes and we just want to find one of them. match = regexp.MustCompile(regexp.QuoteMeta(remoteAnnexUUID) + " -- .* \\[origin\\]\n").MatchString(annexWhereis) if !match { - return errors.New("'git annex whereis' should report large.bin is known to be in [origin]") + return errors.New("'git annex whereis' should report files are known to be in [origin]") } return nil @@ -788,27 +847,56 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { return err } - // verify the file was downloaded - localObjectPath, err := contentLocation(repoPath, "large.bin") - if err != nil { - return err + // verify the files downloaded + + cmp := func(filename string) error { + localObjectPath, err := contentLocation(repoPath, filename) + if err != nil { + return err + } + // localObjectPath := path.Join(repoPath, filename) // or, just compare against the checked-out file + + remoteObjectPath, err := contentLocation(remoteRepoPath, filename) + if err != nil { + return err + } + + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil } - // localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file - remoteObjectPath, err := contentLocation(remoteRepoPath, "large.bin") + // this is the annex-symlink file + stat, err := os.Lstat(path.Join(repoPath, "annexed.tiff")) if err != nil { + return fmt.Errorf("Lstat: %w", err) + } + if !((stat.Mode() & os.ModeSymlink) != 0) { + // this line is really just double-checking that the text fixture is set up correctly + return errors.New("*.tiff should be a symlink") + } + if err = cmp("annexed.tiff"); err != nil { return err } - match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + // this is the annex-pointer file + stat, err = os.Lstat(path.Join(repoPath, "annexed.bin")) if err != nil { - return err + return fmt.Errorf("Lstat: %w", err) } - if !match { - return errors.New("Annexed files should be the same") + if !((stat.Mode() & os.ModeSymlink) == 0) { + // this line is really just double-checking that the text fixture is set up correctly + return errors.New("*.bin should not be a symlink") } + err = cmp("annexed.bin") - return nil + return err } func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { @@ -953,16 +1041,36 @@ func doInitAnnexRepository(repoPath string) error { return err } - // add a file to the annex - err = generateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin")) + // add files to the annex, stored via annex symlinks + // // a binary file + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "annexed.tiff")) + if err != nil { + return err + } + + err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "add", ".").Run(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // add files to the annex, stored via git-annex-smudge + // // a binary file + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "annexed.bin")) + if err != nil { + return err + } + if err != nil { return err } + err = git.AddChanges(repoPath, false, ".") if err != nil { return err } - err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"}) + + // save everything + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex files"}) if err != nil { return err } From 1d65c323a4bd488b0394bc9cc3bef9c7e1a58cb2 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 27 Nov 2022 02:13:46 -0500 Subject: [PATCH 07/13] git-annex: views for annex files (#22) This updates the repo index/file view endpoints so annex files match the way LFS files are rendered, making annexed files accessible via the web instead of being black boxes only accessible by git clone. This mostly just duplicates the existing LFS logic. It doesn't try to combine itself with the existing logic, to make merging with upstream easier. If upstream ever decides to accept, I would like to try to merge the redundant logic. The one bit that doesn't directly copy LFS is my choice to hide annex-symlinks. LFS files are always _pointer files_ and therefore always render with the "file" icon and no special label, but annex files come in two flavours: symlinks or pointer files. I've conflated both kinds to try to give a consistent experience. The tests in here ensure the correct download link (/media, from the last PR) renders in both the toolbar and, if a binary file (like most annexed files will be), in the main pane, but it also adds quite a bit of code to make sure text files that happen to be annexed are dug out and rendered inline like LFS files are. --- modules/base/tool.go | 7 ++ options/locale/locale_cs-CZ.ini | 2 + options/locale/locale_de-DE.ini | 2 + options/locale/locale_el-GR.ini | 2 + options/locale/locale_en-US.ini | 2 + options/locale/locale_es-ES.ini | 2 + options/locale/locale_fa-IR.ini | 2 + options/locale/locale_fr-FR.ini | 2 + options/locale/locale_hu-HU.ini | 2 + options/locale/locale_id-ID.ini | 2 + options/locale/locale_is-IS.ini | 1 + options/locale/locale_it-IT.ini | 2 + options/locale/locale_ja-JP.ini | 2 + options/locale/locale_ko-KR.ini | 1 + options/locale/locale_lv-LV.ini | 2 + options/locale/locale_nl-NL.ini | 2 + options/locale/locale_pl-PL.ini | 2 + options/locale/locale_pt-BR.ini | 2 + options/locale/locale_pt-PT.ini | 2 + options/locale/locale_ru-RU.ini | 2 + options/locale/locale_si-LK.ini | 2 + options/locale/locale_sk-SK.ini | 1 + options/locale/locale_sv-SE.ini | 2 + options/locale/locale_tr-TR.ini | 2 + options/locale/locale_uk-UA.ini | 2 + options/locale/locale_zh-CN.ini | 2 + options/locale/locale_zh-HK.ini | 1 + options/locale/locale_zh-TW.ini | 2 + routers/web/repo/view.go | 66 +++++++++++--- templates/repo/file_info.tmpl | 1 + tests/integration/git_annex_test.go | 133 ++++++++++++++++++++++++++++ 31 files changed, 246 insertions(+), 11 deletions(-) diff --git a/modules/base/tool.go b/modules/base/tool.go index 71dcb83fb4850..da0df40757822 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -19,6 +19,7 @@ import ( "unicode" "unicode/utf8" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -204,6 +205,12 @@ func IsLetter(ch rune) bool { func EntryIcon(entry *git.TreeEntry) string { switch { case entry.IsLink(): + isAnnexed, _ := annex.IsAnnexed(entry.Blob()) + if isAnnexed { + // git-annex files are sometimes stored as symlinks; + // short-circuit that so like LFS they are displayed as regular files + return "file" + } te, err := entry.FollowLink() if err != nil { log.Debug(err.Error()) diff --git a/options/locale/locale_cs-CZ.ini b/options/locale/locale_cs-CZ.ini index 602fe0bce83f7..4115aac6e4410 100644 --- a/options/locale/locale_cs-CZ.ini +++ b/options/locale/locale_cs-CZ.ini @@ -1094,6 +1094,7 @@ view_git_blame=Zobrazit Git Blame video_not_supported_in_browser=Váš prohlížeč nepodporuje značku pro HTML5 video. audio_not_supported_in_browser=Váš prohlížeč nepodporuje značku pro HTML5 audio. stored_lfs=Uloženo pomocí Git LFS +stored_annex=Uloženo pomocí Git Annex symbolic_link=Symbolický odkaz commit_graph=Graf commitů commit_graph.select=Vybrat větve @@ -1113,6 +1114,7 @@ editor.upload_file=Nahrát soubor editor.edit_file=Upravit soubor editor.preview_changes=Náhled změn editor.cannot_edit_lfs_files=LFS soubory nemohou být upravovány přes webové rozhraní. +editor.cannot_edit_annex_files=Annex soubory nemohou být upravovány přes webové rozhraní. editor.cannot_edit_non_text_files=Binární soubory nemohou být upravovány přes webové rozhraní. editor.edit_this_file=Upravit soubor editor.this_file_locked=Soubor je uzamčen diff --git a/options/locale/locale_de-DE.ini b/options/locale/locale_de-DE.ini index f970fdb666846..cb76dddeaeaa1 100644 --- a/options/locale/locale_de-DE.ini +++ b/options/locale/locale_de-DE.ini @@ -1182,6 +1182,7 @@ view_git_blame=Git Blame ansehen video_not_supported_in_browser=Dein Browser unterstützt das HTML5 'video'-Tag nicht. audio_not_supported_in_browser=Dein Browser unterstützt den HTML5 'audio'-Tag nicht. stored_lfs=Gespeichert mit Git LFS +stored_annex=Gespeichert mit Git Annex symbolic_link=Softlink executable_file=Ausführbare Datei commit_graph=Commit graph @@ -1205,6 +1206,7 @@ editor.upload_file=Datei hochladen editor.edit_file=Datei bearbeiten editor.preview_changes=Vorschau der Änderungen editor.cannot_edit_lfs_files=LFS-Dateien können im Webinterface nicht bearbeitet werden. +editor.cannot_edit_annex_files=Annex-Dateien können im Webinterface nicht bearbeitet werden. editor.cannot_edit_non_text_files=Binärdateien können nicht im Webinterface bearbeitet werden. editor.edit_this_file=Datei bearbeiten editor.this_file_locked=Datei ist gesperrt diff --git a/options/locale/locale_el-GR.ini b/options/locale/locale_el-GR.ini index 2dc615a1b336f..ee8f303d00490 100644 --- a/options/locale/locale_el-GR.ini +++ b/options/locale/locale_el-GR.ini @@ -1111,6 +1111,7 @@ view_git_blame=Προβολή Git Blame video_not_supported_in_browser=Το πρόγραμμα περιήγησής σας δεν υποστηρίζει την ετικέτα HTML5 'video'. audio_not_supported_in_browser=Το πρόγραμμα περιήγησής σας δεν υποστηρίζει την ετικέτα HTML5 'audio'. stored_lfs=Αποθηκεύτηκε με το Git LFS +stored_annex=Αποθηκεύτηκε με το Git Annex symbolic_link=Symbolic link commit_graph=Γράφημα Υποβολών commit_graph.select=Επιλογή κλάδων @@ -1130,6 +1131,7 @@ editor.upload_file=Ανέβασμα Αρχείου editor.edit_file=Επεξεργασία Αρχείου editor.preview_changes=Προεπισκόπηση Αλλαγών editor.cannot_edit_lfs_files=Τα αρχεία LFS δεν μπορούν να επεξεργαστούν στη διεπαφή web. +editor.cannot_edit_annex_files=Τα αρχεία Annex δεν μπορούν να επεξεργαστούν στη διεπαφή web. editor.cannot_edit_non_text_files=Τα δυαδικά αρχεία δεν μπορούν να επεξεργαστούν στη διεπαφή web. editor.edit_this_file=Επεξεργασία Αρχείου editor.this_file_locked=Το αρχείο είναι κλειδωμένο diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index a7a7a4f4c50f9..6c344c745a451 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1179,6 +1179,7 @@ view_git_blame = View Git Blame video_not_supported_in_browser = Your browser does not support the HTML5 'video' tag. audio_not_supported_in_browser = Your browser does not support the HTML5 'audio' tag. stored_lfs = Stored with Git LFS +stored_annex = Stored with Git Annex symbolic_link = Symbolic link executable_file = Executable File commit_graph = Commit Graph @@ -1202,6 +1203,7 @@ editor.upload_file = Upload File editor.edit_file = Edit File editor.preview_changes = Preview Changes editor.cannot_edit_lfs_files = LFS files cannot be edited in the web interface. +editor.cannot_edit_annex_files = Annex files cannot be edited in the web interface. editor.cannot_edit_non_text_files = Binary files cannot be edited in the web interface. editor.edit_this_file = Edit File editor.this_file_locked = File is locked diff --git a/options/locale/locale_es-ES.ini b/options/locale/locale_es-ES.ini index 9eb9e856cef14..4f46f5671c160 100644 --- a/options/locale/locale_es-ES.ini +++ b/options/locale/locale_es-ES.ini @@ -1182,6 +1182,7 @@ view_git_blame=Ver la culpa de Git video_not_supported_in_browser=Su navegador no soporta el tag video de HTML5. audio_not_supported_in_browser=Su navegador no soporta el tag audio de HTML5. stored_lfs=Almacenados con Git LFS +stored_annex=Almacenados con Git Annex symbolic_link=Enlace simbólico executable_file=Archivo Ejecutable commit_graph=Gráfico de commits @@ -1205,6 +1206,7 @@ editor.upload_file=Subir archivo editor.edit_file=Editar Archivo editor.preview_changes=Vista previa de los cambios editor.cannot_edit_lfs_files=Los archivos LFS no se pueden editar en la interfaz web. +editor.cannot_edit_annex_files=Los archivos Annex no se pueden editar en la interfaz web. editor.cannot_edit_non_text_files=Los archivos binarios no se pueden editar en la interfaz web. editor.edit_this_file=Editar Archivo editor.this_file_locked=El archivo está bloqueado diff --git a/options/locale/locale_fa-IR.ini b/options/locale/locale_fa-IR.ini index b9194c48a55a8..b94436f594b88 100644 --- a/options/locale/locale_fa-IR.ini +++ b/options/locale/locale_fa-IR.ini @@ -923,6 +923,7 @@ file_copy_permalink=پرمالینک را کپی کنید video_not_supported_in_browser=مرورگر شما از تگ video که در HTML5 تعریف شده است، پشتیبانی نمی کند. audio_not_supported_in_browser=مرورگر شما از تگ audio که در HTML5 تعریف شده است، پشتیبانی نمی کند. stored_lfs=ذخیره شده با GIT LFS +stored_annex=ذخیره شده با GIT Annex symbolic_link=پیوند نمادین commit_graph=نمودار کامیت commit_graph.select=انتخاب برنچها @@ -940,6 +941,7 @@ editor.upload_file=بارگذاری پرونده editor.edit_file=ویرایش پرونده editor.preview_changes=پیش نمایش تغییرات editor.cannot_edit_lfs_files=پرونده های LFS در صحفه وب قابل تغییر نیست. +editor.cannot_edit_annex_files=پرونده های Annex در صحفه وب قابل تغییر نیست. editor.cannot_edit_non_text_files=پرونده‎های دودویی در صفحه وب قابل تغییر نیست. editor.edit_this_file=ویرایش پرونده editor.this_file_locked=پرونده قفل شده است diff --git a/options/locale/locale_fr-FR.ini b/options/locale/locale_fr-FR.ini index ebde7e5baeba7..de6bc5e787396 100644 --- a/options/locale/locale_fr-FR.ini +++ b/options/locale/locale_fr-FR.ini @@ -1182,6 +1182,7 @@ view_git_blame=Voir Git Blâme video_not_supported_in_browser=Votre navigateur ne supporte pas la balise « vidéo » HTML5. audio_not_supported_in_browser=Votre navigateur ne supporte pas la balise « audio » HTML5. stored_lfs=Stocké avec Git LFS +stored_annex=Stocké avec Git Annex symbolic_link=Lien symbolique executable_file=Fichiers exécutables commit_graph=Graphe des révisions @@ -1205,6 +1206,7 @@ editor.upload_file=Téléverser un fichier editor.edit_file=Modifier le fichier editor.preview_changes=Aperçu des modifications editor.cannot_edit_lfs_files=Les fichiers LFS ne peuvent pas être modifiés dans l'interface web. +editor.cannot_edit_annex_files=Les fichiers Annex ne peuvent pas être modifiés dans l'interface web. editor.cannot_edit_non_text_files=Les fichiers binaires ne peuvent pas être édités dans l'interface web. editor.edit_this_file=Modifier le fichier editor.this_file_locked=Le fichier est verrouillé diff --git a/options/locale/locale_hu-HU.ini b/options/locale/locale_hu-HU.ini index c22a049817671..e5a50face2188 100644 --- a/options/locale/locale_hu-HU.ini +++ b/options/locale/locale_hu-HU.ini @@ -692,6 +692,7 @@ file_too_large=Ez a fájl túl nagy ahhoz, hogy megjelenítsük. video_not_supported_in_browser=A böngésző nem támogatja a HTML5 video tag-et. audio_not_supported_in_browser=A böngésző nem támogatja a HTML5 audio tag-et. stored_lfs=Git LFS-el eltárolva +stored_annex=Git Annex-el eltárolva symbolic_link=Szimbolikus hivatkozás commit_graph=Commit gráf commit_graph.hide_pr_refs=Pull request-ek elrejtése @@ -704,6 +705,7 @@ editor.upload_file=Fájl feltöltése editor.edit_file=Fájl szerkesztése editor.preview_changes=Változások előnézete editor.cannot_edit_lfs_files=LFS fájlok nem szerkeszthetőek a webes felületen. +editor.cannot_edit_annex_files=Annex fájlok nem szerkeszthetőek a webes felületen. editor.cannot_edit_non_text_files=Bináris fájlok nem szerkeszthetőek a webes felületen. editor.edit_this_file=Fájl szerkesztése editor.this_file_locked=Zárolt állomány diff --git a/options/locale/locale_id-ID.ini b/options/locale/locale_id-ID.ini index a5efde6d07b82..60f2d5ce53f64 100644 --- a/options/locale/locale_id-ID.ini +++ b/options/locale/locale_id-ID.ini @@ -598,6 +598,7 @@ file_permalink=Permalink file_too_large=Berkas terlalu besar untuk ditampilkan. stored_lfs=Tersimpan dengan GIT LFS +stored_annex=Tersimpan dengan GIT Annex commit_graph=Grafik Komit blame=Salahkan normal_view=Pandangan Normal @@ -609,6 +610,7 @@ editor.upload_file=Unggah Berkas editor.edit_file=Sunting Berkas editor.preview_changes=Tinjau Perubahan editor.cannot_edit_lfs_files=Berkas LFS tidak dapat disunting dalam antarmuka web. +editor.cannot_edit_annex_files=Berkas Annex tidak dapat disunting dalam antarmuka web. editor.cannot_edit_non_text_files=Berkas biner tidak dapat disunting dalam antarmuka web. editor.edit_this_file=Sunting Berkas editor.this_file_locked=Berkas terkunci diff --git a/options/locale/locale_is-IS.ini b/options/locale/locale_is-IS.ini index 28ce5ae7f7ac7..57471c0b1432c 100644 --- a/options/locale/locale_is-IS.ini +++ b/options/locale/locale_is-IS.ini @@ -683,6 +683,7 @@ file_view_rendered=Skoða Unnið file_copy_permalink=Afrita Varanlega Slóð stored_lfs=Geymt með Git LFS +stored_annex=Geymt með Git Annex commit_graph.hide_pr_refs=Fela Sameiningarbeiðnir commit_graph.monochrome=Einlitað commit_graph.color=Litað diff --git a/options/locale/locale_it-IT.ini b/options/locale/locale_it-IT.ini index e73d063613707..1d575c5e63f93 100644 --- a/options/locale/locale_it-IT.ini +++ b/options/locale/locale_it-IT.ini @@ -994,6 +994,7 @@ view_git_blame=Visualizza Git Blame video_not_supported_in_browser=Il tuo browser non supporta i tag "video" di HTML5. audio_not_supported_in_browser=Il tuo browser non supporta il tag "video" di HTML5. stored_lfs=Memorizzati con Git LFS +stored_annex=Memorizzati con Git Annex symbolic_link=Link Simbolico commit_graph=Grafico dei commit commit_graph.select=Seleziona rami @@ -1012,6 +1013,7 @@ editor.upload_file=Carica File editor.edit_file=Modifica File editor.preview_changes=Anteprima modifiche editor.cannot_edit_lfs_files=I file LFS non possono essere modificati nell'interfaccia web. +editor.cannot_edit_annex_files=I file Annex non possono essere modificati nell'interfaccia web. editor.cannot_edit_non_text_files=I file binari non possono essere modificati tramite interfaccia web. editor.edit_this_file=Modifica file editor.this_file_locked=Il file è bloccato diff --git a/options/locale/locale_ja-JP.ini b/options/locale/locale_ja-JP.ini index df1cd0a540e75..7a28031c9b4c9 100644 --- a/options/locale/locale_ja-JP.ini +++ b/options/locale/locale_ja-JP.ini @@ -1182,6 +1182,7 @@ view_git_blame=Git Blameを表示 video_not_supported_in_browser=このブラウザはHTML5のvideoタグをサポートしていません。 audio_not_supported_in_browser=このブラウザーはHTML5のaudioタグをサポートしていません。 stored_lfs=Git LFSで保管されています +stored_annex=Git Annexで保管されています symbolic_link=シンボリック リンク executable_file=実行ファイル commit_graph=コミットグラフ @@ -1205,6 +1206,7 @@ editor.upload_file=ファイルをアップロード editor.edit_file=ファイルを編集 editor.preview_changes=変更をプレビュー editor.cannot_edit_lfs_files=LFSのファイルはWebインターフェースで編集できません。 +editor.cannot_edit_annex_files=AnnexのファイルはWebインターフェースで編集できません。 editor.cannot_edit_non_text_files=バイナリファイルはWebインターフェースで編集できません。 editor.edit_this_file=ファイルを編集 editor.this_file_locked=ファイルはロックされています diff --git a/options/locale/locale_ko-KR.ini b/options/locale/locale_ko-KR.ini index d33bf0f850bc3..52cbaf18c2d50 100644 --- a/options/locale/locale_ko-KR.ini +++ b/options/locale/locale_ko-KR.ini @@ -638,6 +638,7 @@ file_too_large=보여주기에는 파일이 너무 큽니다. video_not_supported_in_browser=당신의 브라우저가 HTML5 'video' 태그를 지원하지 않습니다. audio_not_supported_in_browser=당신의 브라우저가 HTML5 'audio' 태그를 지원하지 않습니다. stored_lfs=Git LFS에 저장되어 있습니다 +stored_annex=Git Annex에 저장되어 있습니다 commit_graph=커밋 그래프 editor.new_file=새 파일 diff --git a/options/locale/locale_lv-LV.ini b/options/locale/locale_lv-LV.ini index 1af1cc368564c..a7d4ee5164d52 100644 --- a/options/locale/locale_lv-LV.ini +++ b/options/locale/locale_lv-LV.ini @@ -1110,6 +1110,7 @@ view_git_blame=Aplūkot Git vainīgos video_not_supported_in_browser=Jūsu pārlūks neatbalsta HTML5 video. audio_not_supported_in_browser=Jūsu pārlūks neatbalsta HTML5 audio. stored_lfs=Saglabāts Git LFS +stored_annex=Saglabāts Git Annex symbolic_link=Simboliska saite commit_graph=Revīziju grafs commit_graph.select=Izvēlieties atzarus @@ -1129,6 +1130,7 @@ editor.upload_file=Augšupielādēt failu editor.edit_file=Labot failu editor.preview_changes=Priekšskatīt izmaiņas editor.cannot_edit_lfs_files=LFS failus nevar labot no tīmekļa saskarnes. +editor.cannot_edit_annex_files=Annex failus nevar labot no tīmekļa saskarnes. editor.cannot_edit_non_text_files=Nav iespējams labot bināros failus no pārlūka saskarnes. editor.edit_this_file=Labot failu editor.this_file_locked=Fails ir bloķēts diff --git a/options/locale/locale_nl-NL.ini b/options/locale/locale_nl-NL.ini index 2440002241354..4ef7787b4316f 100644 --- a/options/locale/locale_nl-NL.ini +++ b/options/locale/locale_nl-NL.ini @@ -992,6 +992,7 @@ view_git_blame=Bekijk Git Blame video_not_supported_in_browser=Je browser ondersteunt de HTML5 'video'-tag niet. audio_not_supported_in_browser=Je browser ondersteunt de HTML5 'audio'-tag niet. stored_lfs=Opgeslagen met Git LFS +stored_annex=Opgeslagen met Git Annex symbolic_link=Symbolic link commit_graph=Commit grafiek commit_graph.select=Selecteer branches @@ -1010,6 +1011,7 @@ editor.upload_file=Upload bestand editor.edit_file=Bewerk bestand editor.preview_changes=Voorbeeld tonen editor.cannot_edit_lfs_files=LFS-bestanden kunnen niet worden bewerkt in de webinterface. +editor.cannot_edit_annex_files=Annex-bestanden kunnen niet worden bewerkt in de webinterface. editor.cannot_edit_non_text_files=Binaire bestanden kunnen niet worden bewerkt in de webinterface. editor.edit_this_file=Bewerk bestand editor.this_file_locked=Bestand is vergrendeld diff --git a/options/locale/locale_pl-PL.ini b/options/locale/locale_pl-PL.ini index 8811b218927a6..a2e4f12996110 100644 --- a/options/locale/locale_pl-PL.ini +++ b/options/locale/locale_pl-PL.ini @@ -927,6 +927,7 @@ file_copy_permalink=Kopiuj bezpośredni odnośnik video_not_supported_in_browser=Twoja przeglądarka nie obsługuje znacznika HTML5 "video". audio_not_supported_in_browser=Twoja przeglądarka nie obsługuje znacznika HTML5 "audio". stored_lfs=Przechowane za pomocą Git LFS +stored_annex=Przechowane za pomocą Git Annex symbolic_link=Dowiązanie symboliczne commit_graph=Wykres commitów commit_graph.select=Wybierz gałęzie @@ -944,6 +945,7 @@ editor.upload_file=Wyślij plik editor.edit_file=Edytuj plik editor.preview_changes=Podgląd zmian editor.cannot_edit_lfs_files=Pliki LFS nie mogą być edytowane poprzez interfejs przeglądarkowy. +editor.cannot_edit_annex_files=Pliki Annex nie mogą być edytowane poprzez interfejs przeglądarkowy. editor.cannot_edit_non_text_files=Pliki binarne nie mogą być edytowane poprzez interfejs przeglądarkowy. editor.edit_this_file=Edytuj plik editor.this_file_locked=Plik jest zablokowany diff --git a/options/locale/locale_pt-BR.ini b/options/locale/locale_pt-BR.ini index 302ff934667f4..62cc4e46549fd 100644 --- a/options/locale/locale_pt-BR.ini +++ b/options/locale/locale_pt-BR.ini @@ -1171,6 +1171,7 @@ view_git_blame=Ver Git Blame video_not_supported_in_browser=Seu navegador não suporta a tag 'video' do HTML5. audio_not_supported_in_browser=Seu navegador não suporta a tag 'audio' do HTML5. stored_lfs=Armazenado com Git LFS +stored_annex=Armazenado com Git Annex symbolic_link=Link simbólico executable_file=Arquivo executável commit_graph=Gráfico de commits @@ -1194,6 +1195,7 @@ editor.upload_file=Enviar arquivo editor.edit_file=Editar arquivo editor.preview_changes=Visualizar alterações editor.cannot_edit_lfs_files=Arquivos LFS não podem ser editados na interface web. +editor.cannot_edit_annex_files=Arquivos Annex não podem ser editados na interface web. editor.cannot_edit_non_text_files=Arquivos binários não podem ser editados na interface web. editor.edit_this_file=Editar arquivo editor.this_file_locked=Arquivo está bloqueado diff --git a/options/locale/locale_pt-PT.ini b/options/locale/locale_pt-PT.ini index 9d9c5eada1c07..776e4d04c061e 100644 --- a/options/locale/locale_pt-PT.ini +++ b/options/locale/locale_pt-PT.ini @@ -1182,6 +1182,7 @@ view_git_blame=Ver Git Blame video_not_supported_in_browser=O seu navegador não suporta a etiqueta 'video' do HTML5. audio_not_supported_in_browser=O seu navegador não suporta a etiqueta 'audio' do HTML5. stored_lfs=Armazenado com Git LFS +stored_annex=Armazenado com Git Annex symbolic_link=Ligação simbólica executable_file=Ficheiro executável commit_graph=Gráfico de cometimentos @@ -1205,6 +1206,7 @@ editor.upload_file=Carregar ficheiro editor.edit_file=Editar ficheiro editor.preview_changes=Pré-visualizar modificações editor.cannot_edit_lfs_files=Ficheiros LFS não podem ser editados na interface web. +editor.cannot_edit_annex_files=Ficheiros Annex não podem ser editados na interface web. editor.cannot_edit_non_text_files=Ficheiros binários não podem ser editados na interface da web. editor.edit_this_file=Editar ficheiro editor.this_file_locked=Ficheiro bloqueado diff --git a/options/locale/locale_ru-RU.ini b/options/locale/locale_ru-RU.ini index 3a560eba1d19d..e6c4cad266c0b 100644 --- a/options/locale/locale_ru-RU.ini +++ b/options/locale/locale_ru-RU.ini @@ -1156,6 +1156,7 @@ view_git_blame=Показать git blame video_not_supported_in_browser=Ваш браузер не поддерживает HTML5 'video' тэг. audio_not_supported_in_browser=Ваш браузер не поддерживает HTML5 'audio' тэг. stored_lfs=Хранится Git LFS +stored_annex=Хранится Git Annex symbolic_link=Символическая ссылка executable_file=Исполняемый файл commit_graph=Граф коммитов @@ -1179,6 +1180,7 @@ editor.upload_file=Загрузить файл editor.edit_file=Редактировать файл editor.preview_changes=Просмотр изменений editor.cannot_edit_lfs_files=LFS файлы невозможно редактировать в веб-интерфейсе. +editor.cannot_edit_annex_files=Annex файлы невозможно редактировать в веб-интерфейсе. editor.cannot_edit_non_text_files=Двоичные файлы нельзя редактировать в веб-интерфейсе. editor.edit_this_file=Редактировать файл editor.this_file_locked=Файл заблокирован diff --git a/options/locale/locale_si-LK.ini b/options/locale/locale_si-LK.ini index 6fdcb3422856b..603619653d436 100644 --- a/options/locale/locale_si-LK.ini +++ b/options/locale/locale_si-LK.ini @@ -892,6 +892,7 @@ file_copy_permalink=පිටපත් මාමලින්ක් video_not_supported_in_browser=ඔබගේ බ්රව්සරය HTML5 'වීඩියෝ' ටැගය සඳහා සහය නොදක්වයි. audio_not_supported_in_browser=ඔබගේ බ්රව්සරය HTML5 'ශ්රව්ය' ටැගය සඳහා සහය නොදක්වයි. stored_lfs=Git LFS සමඟ ගබඩා +stored_annex=Git Annex සමඟ ගබඩා symbolic_link=සංකේතාත්මක සබැඳිය commit_graph=ප්රස්තාරය කැප commit_graph.select=ශාඛා තෝරන්න @@ -909,6 +910,7 @@ editor.upload_file=ගොනුව උඩුගත කරන්න editor.edit_file=ගොනුව සංස්කරණය editor.preview_changes=වෙනස්කම් පෙරදසුන editor.cannot_edit_lfs_files=LFS ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. +editor.cannot_edit_annex_files=Annex ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. editor.cannot_edit_non_text_files=ද්විමය ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. editor.edit_this_file=ගොනුව සංස්කරණය editor.this_file_locked=ගොනුවට අගුළු ලා ඇත diff --git a/options/locale/locale_sk-SK.ini b/options/locale/locale_sk-SK.ini index d137a6d2beda6..0ba495cda4c84 100644 --- a/options/locale/locale_sk-SK.ini +++ b/options/locale/locale_sk-SK.ini @@ -914,6 +914,7 @@ view_git_blame=Zobraziť Git Blame video_not_supported_in_browser=Váš prehliadač nepodporuje HTML5 tag 'video'. audio_not_supported_in_browser=Váš prehliadač nepodporuje HTML5 tag 'audio'. stored_lfs=Uložené pomocou Git LFS +stored_annex=Uložené pomocou Git Annex symbolic_link=Symbolický odkaz commit_graph=Graf commitov diff --git a/options/locale/locale_sv-SE.ini b/options/locale/locale_sv-SE.ini index ce3f7d5d9bec4..d988e3b8c39b2 100644 --- a/options/locale/locale_sv-SE.ini +++ b/options/locale/locale_sv-SE.ini @@ -754,6 +754,7 @@ file_too_large=Filen är för stor för att visas. video_not_supported_in_browser=Din webbläsare stödjer ej HTML5-taggen 'video'. audio_not_supported_in_browser=Din webbläsare stöder inte taggen 'audio' i HTML5. stored_lfs=Sparad med Git LFS +stored_annex=Sparad med Git Annex symbolic_link=Symbolisk länk commit_graph=Commit-Graf commit_graph.monochrome=Mono @@ -767,6 +768,7 @@ editor.upload_file=Ladda Upp Fil editor.edit_file=Redigera Fil editor.preview_changes=Förhandsgranska ändringar editor.cannot_edit_lfs_files=LFS-filer kan inte redigeras i webbgränssnittet. +editor.cannot_edit_annex_files=Annex-filer kan inte redigeras i webbgränssnittet. editor.cannot_edit_non_text_files=Binära filer kan inte redigeras genom webbgränssnittet. editor.edit_this_file=Redigera Fil editor.this_file_locked=Filen är låst diff --git a/options/locale/locale_tr-TR.ini b/options/locale/locale_tr-TR.ini index 4c0fc95c0a26b..229dd4336105f 100644 --- a/options/locale/locale_tr-TR.ini +++ b/options/locale/locale_tr-TR.ini @@ -1169,6 +1169,7 @@ view_git_blame=Git Suç Görüntüle video_not_supported_in_browser=Tarayıcınız HTML5 'video' etiketini desteklemiyor. audio_not_supported_in_browser=Tarayıcınız HTML5 'audio' etiketini desteklemiyor. stored_lfs=Git LFS ile depolandı +stored_annex=Git Annex ile depolandı symbolic_link=Sembolik Bağlantı executable_file=Çalıştırılabilir Dosya commit_graph=İşleme Grafiği @@ -1192,6 +1193,7 @@ editor.upload_file=Dosya Yükle editor.edit_file=Dosyayı Düzenle editor.preview_changes=Değişiklikleri Önizle editor.cannot_edit_lfs_files=LFS dosyaları web arayüzünde düzenlenemez. +editor.cannot_edit_annex_files=Annex dosyaları web arayüzünde düzenlenemez. editor.cannot_edit_non_text_files=Bu tür dosyalar web arayüzünden düzenlenemez. editor.edit_this_file=Dosyayı Düzenle editor.this_file_locked=Dosya kilitlendi diff --git a/options/locale/locale_uk-UA.ini b/options/locale/locale_uk-UA.ini index 85adc61d33911..9d4671cb4d1c3 100644 --- a/options/locale/locale_uk-UA.ini +++ b/options/locale/locale_uk-UA.ini @@ -931,6 +931,7 @@ file_copy_permalink=Копіювати постійне посилання video_not_supported_in_browser=Ваш браузер не підтримує тег 'video' HTML5. audio_not_supported_in_browser=Ваш браузер не підтримує тег HTML5 'audio'. stored_lfs=Збережено з Git LFS +stored_annex=Збережено з Git Annex symbolic_link=Символічне посилання commit_graph=Графік комітів commit_graph.select=Виберіть гілки @@ -948,6 +949,7 @@ editor.upload_file=Завантажити файл editor.edit_file=Редагування файлу editor.preview_changes=Попередній перегляд змін editor.cannot_edit_lfs_files=Файли LFS не можна редагувати в веб-інтерфейсі. +editor.cannot_edit_annex_files=Файли Annex не можна редагувати в веб-інтерфейсі. editor.cannot_edit_non_text_files=Бінарні файли не можливо редагувати у веб-інтерфейсі. editor.edit_this_file=Редагувати файл editor.this_file_locked=Файл заблоковано diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8ebf6c509c3f9..4c565ac1c8688 100644 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1182,6 +1182,7 @@ view_git_blame=查看 Git Blame video_not_supported_in_browser=您的浏览器不支持使用 HTML5 'video' 标签。 audio_not_supported_in_browser=您的浏览器不支持使用 HTML5 'video' 标签。 stored_lfs=存储到Git LFS +stored_annex=存储到Git Annex symbolic_link=符号链接 executable_file=可执行文件 commit_graph=提交图 @@ -1205,6 +1206,7 @@ editor.upload_file=上传文件 editor.edit_file=编辑文件 editor.preview_changes=预览变更 editor.cannot_edit_lfs_files=无法在 web 界面中编辑 lfs 文件。 +editor.cannot_edit_annex_files=无法在 web 界面中编辑 lfs 文件。 editor.cannot_edit_non_text_files=网页不能编辑二进制文件。 editor.edit_this_file=编辑文件 editor.this_file_locked=文件已锁定 diff --git a/options/locale/locale_zh-HK.ini b/options/locale/locale_zh-HK.ini index dfa048c5ef0b9..fe71ddd9c1ac7 100644 --- a/options/locale/locale_zh-HK.ini +++ b/options/locale/locale_zh-HK.ini @@ -365,6 +365,7 @@ file_view_raw=查看原始文件 file_permalink=永久連結 stored_lfs=儲存到到 Git LFS +stored_annex=儲存到到 Git Annex editor.preview_changes=預覽更改 editor.or=或 diff --git a/options/locale/locale_zh-TW.ini b/options/locale/locale_zh-TW.ini index e058ca71919fe..a2460857aa434 100644 --- a/options/locale/locale_zh-TW.ini +++ b/options/locale/locale_zh-TW.ini @@ -1081,6 +1081,7 @@ view_git_blame=檢視 Git Blame video_not_supported_in_browser=您的瀏覽器不支援使用 HTML5 播放影片。 audio_not_supported_in_browser=您的瀏覽器不支援 HTML5 的「audio」標籤 stored_lfs=已使用 Git LFS 儲存 +stored_annex=已使用 Git Annex 儲存 symbolic_link=符號連結 commit_graph=提交線圖 commit_graph.select=選擇分支 @@ -1100,6 +1101,7 @@ editor.upload_file=上傳檔案 editor.edit_file=編輯檔案 editor.preview_changes=預覽更改 editor.cannot_edit_lfs_files=無法在 web 介面中編輯 LFS 檔。 +editor.cannot_edit_annex_files=無法在 web 介面中編輯 Annex 檔。 editor.cannot_edit_non_text_files=網站介面不能編輯二進位檔案 editor.edit_this_file=編輯檔案 editor.this_file_locked=檔案已被鎖定 diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 89bb1839e1747..4bcc8e32c285d 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -32,6 +32,7 @@ import ( unit_model "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/actions" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/container" @@ -200,14 +201,47 @@ func localizedExtensions(ext, languageCode string) (localizedExts []string) { } type fileInfo struct { - isTextFile bool - isLFSFile bool - fileSize int64 - lfsMeta *lfs.Pointer - st typesniffer.SniffedType + isTextFile bool + isLFSFile bool + isAnnexFile bool + fileSize int64 + lfsMeta *lfs.Pointer + st typesniffer.SniffedType } func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, io.ReadCloser, *fileInfo, error) { + isAnnexed, err := annex.IsAnnexed(blob) + if err != nil { + return nil, nil, nil, err + } + if isAnnexed { + // TODO: this code could be merged with the LFS case, especially the redundant type sniffer, + // but it is *currently* written this way to make merging with the non-annex upstream easier: + // this way, the git-annex patch is (mostly) pure additions. + + annexContent, err := annex.Content(blob) + if err != nil { + // in the case where annex content is missing, what should happen? + // do we render the page with an error message? + // actually that's not a bad idea, there's some sort of error message situation + // TODO: display an error to the user explaining that their data is missing + return nil, nil, nil, err + } + + stat, err := annexContent.Stat() + if err != nil { + return nil, nil, nil, err + } + + buf := make([]byte, 1024) + n, _ := util.ReadAtMost(annexContent, buf) + buf = buf[:n] + + st := typesniffer.DetectContentType(buf) + + return buf, annexContent, &fileInfo{st.IsText(), false, true, stat.Size(), nil, st}, nil + } + dataRc, err := blob.DataAsync() if err != nil { return nil, nil, nil, err @@ -222,17 +256,17 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, // FIXME: what happens when README file is an image? if !isTextFile || !setting.LFS.StartServer { - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } pointer, _ := lfs.ReadPointerFromBuffer(buf) if !pointer.IsValid() { // fallback to plain file - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } meta, err := git_model.GetLFSMetaObjectByOid(ctx, repoID, pointer.Oid) if err != nil && err != git_model.ErrLFSObjectNotExist { // fallback to plain file - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } dataRc.Close() @@ -255,7 +289,7 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, st = typesniffer.DetectContentType(buf) - return buf, dataRc, &fileInfo{st.IsText(), true, meta.Size, &meta.Pointer, st}, nil + return buf, dataRc, &fileInfo{st.IsText(), true, false, meta.Size, &meta.Pointer, st}, nil } func renderReadmeFile(ctx *context.Context, subfolder string, readmeFile *git.TreeEntry, readmeTreelink string) { @@ -383,10 +417,17 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st isDisplayingSource := ctx.FormString("display") == "source" isDisplayingRendered := !isDisplayingSource - if fInfo.isLFSFile { + if fInfo.isLFSFile || fInfo.isAnnexFile { ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) } + if fInfo.isAnnexFile { + // pre-git-annex v7, all annexed files were represented in-repo as symlinks; + // but we pretend they aren't, since that's a distracting quirk of git-annex + // and not a meaningful choice on the user's part + ctx.Data["FileIsSymlink"] = false + } + isRepresentableAsText := fInfo.st.IsRepresentableAsText() if !isRepresentableAsText { // If we can't show plain text, always try to render. @@ -394,6 +435,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st isDisplayingRendered = true } ctx.Data["IsLFSFile"] = fInfo.isLFSFile + ctx.Data["IsAnnexFile"] = fInfo.isAnnexFile ctx.Data["FileSize"] = fInfo.fileSize ctx.Data["IsTextFile"] = fInfo.isTextFile ctx.Data["IsRepresentableAsText"] = isRepresentableAsText @@ -428,6 +470,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st // Assume file is not editable first. if fInfo.isLFSFile { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_lfs_files") + } else if fInfo.isAnnexFile { + ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_annex_files") } else if !isRepresentableAsText { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_non_text_files") } @@ -543,7 +587,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.Data["FileContent"] = fileContent ctx.Data["LineEscapeStatus"] = statuses } - if !fInfo.isLFSFile { + if !fInfo.isLFSFile && !fInfo.isAnnexFile { if ctx.Repo.CanEnableEditor(ctx, ctx.Doer) { if lfsLock != nil && lfsLock.OwnerID != ctx.Doer.ID { ctx.Data["CanEditFile"] = false diff --git a/templates/repo/file_info.tmpl b/templates/repo/file_info.tmpl index 3003fbbdb6df5..16d8dcb9e8f64 100644 --- a/templates/repo/file_info.tmpl +++ b/templates/repo/file_info.tmpl @@ -12,6 +12,7 @@ {{if .FileSize}}
{{FileSize .FileSize}}{{if .IsLFSFile}} ({{ctx.Locale.Tr "repo.stored_lfs"}}){{end}} + {{if .IsAnnexFile}} ({{ctx.Locale.Tr "repo.stored_annex"}}){{end}}
{{end}} {{if .LFSLock}} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index c5376f07f0ed8..6f91cbf955350 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -115,6 +115,111 @@ func doAnnexMediaTest(t *testing.T, ctx APITestContext, file string) { require.True(t, match, "Annexed files should be the same") } +func TestGitAnnexViews(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + ctx := NewAPITestContext(t, "user2", "annex-template-render-test", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + + session := loginUser(t, ctx.Username) + + t.Run("Index", func(t *testing.T) { + // test that annex symlinks renders with the _file icon_ on the main list + defer tests.PrintCurrentTest(t)() + + repoLink := path.Join("/", ctx.Username, ctx.Reponame) + req := NewRequest(t, "GET", repoLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + htmlDoc := NewHTMLParser(t, resp.Body) + isFileIcon := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file") + require.True(t, isFileIcon, "annexed files should render a plain file icon, even when stored via annex symlink") + }) + + t.Run("View", func(t *testing.T) { + // test how routers/web/repo/view.go + templates/repo/view_file.tmpl handle annexed files + defer tests.PrintCurrentTest(t)() + + doViewTest := func(file string) (htmlDoc *HTMLDoc, viewLink, mediaLink string) { + viewLink = path.Join("/", ctx.Username, ctx.Reponame, "/src/branch/master", file) + // rawLink := strings.Replace(viewLink, "/src/", "/raw/", 1) // TODO: do something with this? + mediaLink = strings.Replace(viewLink, "/src/", "/media/", 1) + + req := NewRequest(t, "GET", viewLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + htmlDoc = NewHTMLParser(t, resp.Body) + // the first button on the toolbar on the view template is the "Raw" button + // this CSS selector is the most precise I can think to use + buttonLink, exists := htmlDoc.Find(".file-header").Find("a[download]").Attr("href") + require.True(t, exists, "Download button should exist on the file header") + require.EqualValues(t, mediaLink, buttonLink, "Download link should use /media URL for annex files") + + return htmlDoc, viewLink, mediaLink + } + + t.Run("Binary", func(t *testing.T) { + // test that annexing a file renders the /media link in /src and NOT the /raw link + defer tests.PrintCurrentTest(t)() + + doBinaryViewTest := func(file string) { + htmlDoc, _, mediaLink := doViewTest(file) + + rawLink, exists := htmlDoc.Find("div.file-view > div.view-raw > a").Attr("href") + require.True(t, exists, "Download link should render instead of content because this is a binary file") + require.EqualValues(t, mediaLink, rawLink) + } + + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doBinaryViewTest("annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doBinaryViewTest("annexed.bin") + }) + }) + + t.Run("Text", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + doTextViewTest := func(file string) { + htmlDoc, _, _ := doViewTest(file) + require.True(t, htmlDoc.Find("div.file-view").Is(".code-view"), "should render as code") + } + + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doTextViewTest("annexed.txt") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.md") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doTextViewTest("annexed.rst") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.markdown") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) + }) + }) + }) + }) +} + /* Test that permissions are enforced on git-annex-shell commands. @@ -1024,6 +1129,14 @@ func doInitAnnexRepository(repoPath string) error { if err != nil { return err } + _, err = f.WriteString("*.rst filter=annex\n") + if err != nil { + return err + } + _, err = f.WriteString("*.markdown filter=annex\n") + if err != nil { + return err + } f.Close() err = git.AddChanges(repoPath, false, ".") @@ -1048,6 +1161,18 @@ func doInitAnnexRepository(repoPath string) error { return err } + // // a text file + err = os.WriteFile(path.Join(repoPath, "annexed.md"), []byte("Overview\n=====\n\n1. Profit\n2. ???\n3. Review Life Activations\n"), 0o777) + if err != nil { + return err + } + + // // a markdown file + err = os.WriteFile(path.Join(repoPath, "annexed.txt"), []byte("We're going to see the wizard\nThe wonderful\nMonkey of\nBoz\n"), 0o777) + if err != nil { + return err + } + err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "add", ".").Run(&git.RunOpts{Dir: repoPath}) if err != nil { return err @@ -1060,6 +1185,14 @@ func doInitAnnexRepository(repoPath string) error { return err } + // // a text file + err = os.WriteFile(path.Join(repoPath, "annexed.rst"), []byte("Title\n=====\n\n- this is to test annexing a text file\n- lists are fun\n"), 0o777) + if err != nil { + return err + } + + // // a markdown file + err = os.WriteFile(path.Join(repoPath, "annexed.markdown"), []byte("Overview\n=====\n\n1. Profit\n2. ???\n3. Review Life Activations\n"), 0o777) if err != nil { return err } From f78d0d8594d547d9a89f02d7cf73c2ad13daa161 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Mon, 8 May 2023 19:51:49 -0400 Subject: [PATCH 08/13] git-annex: Only run git-annex tests. Upstream can handle the full test suite; to avoid tedious waiting, we only test the code added in this fork. --- .github/workflows/pull-db-tests.yml | 8 ++++---- Makefile | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index c44e82f82eadd..f19f066b29e5a 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -50,7 +50,7 @@ jobs: - run: make backend env: TAGS: bindata - - run: make test-pgsql-migration test-pgsql + - run: make test-pgsql-migration test-pgsql#TestGitAnnex timeout-minutes: 50 env: TAGS: bindata gogit @@ -74,7 +74,7 @@ jobs: - run: make backend env: TAGS: bindata gogit sqlite sqlite_unlock_notify - - run: make test-sqlite-migration test-sqlite + - run: make test-sqlite-migration test-sqlite#TestGitAnnex timeout-minutes: 50 env: TAGS: bindata gogit sqlite sqlite_unlock_notify @@ -179,7 +179,7 @@ jobs: env: TAGS: bindata - name: run tests - run: make test-mysql-migration integration-test-coverage + run: make test-mysql-migration test-mysql#TestGitAnnex env: TAGS: bindata RACE_ENABLED: true @@ -212,7 +212,7 @@ jobs: - run: make backend env: TAGS: bindata - - run: make test-mssql-migration test-mssql + - run: make test-mssql-migration test-mssql#TestGitAnnex timeout-minutes: 50 env: TAGS: bindata diff --git a/Makefile b/Makefile index f8838b601beca..af404bc4c3ae6 100644 --- a/Makefile +++ b/Makefile @@ -114,8 +114,9 @@ LDFLAGS := $(LDFLAGS) -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(G LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64 GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/)) -GO_TEST_PACKAGES ?= $(filter-out $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/)) - +# Only test code modified in the git-annex feature branch; upstream can handle testing the full suite. +# This list was generated by `git diff --stat --name-only main.. -- '*.go' | xargs dirname | sort | uniq` +GO_TEST_PACKAGES ?= code.gitea.io/gitea/modules/annex code.gitea.io/gitea/modules/base code.gitea.io/gitea/modules/git code.gitea.io/gitea/modules/private code.gitea.io/gitea/modules/setting code.gitea.io/gitea/modules/util code.gitea.io/gitea/routers/private code.gitea.io/gitea/routers/web code.gitea.io/gitea/services/auth FOMANTIC_WORK_DIR := web_src/fomantic WEBPACK_SOURCES := $(shell find web_src/js web_src/css -type f) From a29a1396b0870bf2ae5f1c37676e2c57ae763abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 3 Nov 2023 15:32:57 +0100 Subject: [PATCH 09/13] git-annex: hide special branches from branch lists This hides the git-annex branch as well as the synced/* branches from many places in the UI where branches are otherwise shown. This includes e.g. the dropdown menu on the repo page and when comparing different branches, or the dedicated branches page of a repository. --- models/git/branch_list.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/models/git/branch_list.go b/models/git/branch_list.go index b5c1301a1dc00..2004abe8c7d02 100644 --- a/models/git/branch_list.go +++ b/models/git/branch_list.go @@ -79,6 +79,11 @@ func (opts *FindBranchOptions) Cond() builder.Cond { cond = cond.And(builder.Eq{"repo_id": opts.RepoID}) } + // Hide the git-annex branch if it exists + cond = cond.And(builder.Neq{"name": "git-annex"}) + // Hide synced/* branches that git-annex might have created + cond = cond.And(builder.Not{builder.Like{"name", "synced/%"}}) + if len(opts.ExcludeBranchNames) > 0 { cond = cond.And(builder.NotIn("name", opts.ExcludeBranchNames)) } From edb476ccb0e4176b5d660fba889b068cd77603d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 3 Nov 2023 15:34:01 +0100 Subject: [PATCH 10/13] git-annex: ignore actions on special branches This prevents actions that belong to the git-annex branch or synced/* branches from showing up in e.g. the users activity feed. --- models/activities/action.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/models/activities/action.go b/models/activities/action.go index 15bd9a52acc5c..9b4bbf9193efd 100644 --- a/models/activities/action.go +++ b/models/activities/action.go @@ -559,6 +559,11 @@ func activityQueryCondition(ctx context.Context, opts GetFeedsOptions) (builder. } } + // Hide actions related to the git-annex branch + cond = cond.And(builder.Neq{"ref_name": git.BranchPrefix + "git-annex"}) + // Hide actions related to git-annex's synced/* branches + cond = cond.And(builder.Not{builder.Like{"ref_name", git.BranchPrefix + "synced/%"}}) + return cond, nil } From e0bcd54f127a54550e73bed619e6d4ec3e81b74d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 3 Nov 2023 15:56:07 +0100 Subject: [PATCH 11/13] git-annex: hide PR banner for special branches This hides the banner you can see on a repositories page that proposes to open a PR for your branch if you recently pushed to said branch, but only if that branch is named git-annex or starts with synced/. --- models/git/branch.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/git/branch.go b/models/git/branch.go index 6d50fb9fb6a18..72edcc37ede68 100644 --- a/models/git/branch.go +++ b/models/git/branch.go @@ -394,6 +394,8 @@ func FindRecentlyPushedNewBranches(ctx context.Context, repoID, userID int64, ex err := db.GetEngine(ctx). Where("pusher_id=? AND is_deleted=?", userID, false). And("name <> ?", excludeBranchName). + And("name <> ?", "git-annex"). + And("NOT name LIKE ?", "synced/%"). And("repo_id = ?", repoID). And("commit_time >= ?", time.Now().Add(-time.Hour*6).Unix()). NotIn("name", subQuery). From 53d1ddcb23ffa4179fcbfc1bd454d3d43a3dfe9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 3 Nov 2023 16:31:10 +0100 Subject: [PATCH 12/13] git-annex: hide PR message for pushes This hides the message with a link to open a new PR that you normally get when pushing to a new branch if you push to a newly created git-annex or synced/ branch. --- cmd/hook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/hook.go b/cmd/hook.go index f38fd8831b8d5..77f8add8a5064 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -446,7 +446,7 @@ Gitea or set your environment appropriately.`, "") func hookPrintResults(results []private.HookPostReceiveBranchResult) { for _, res := range results { - if !res.Message { + if !res.Message || res.Branch == "git-annex" || strings.HasPrefix(res.Branch, "synced/") { continue } From a037b816fa72db56b6791eb4ddc7b63a634cc61a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Mon, 6 Nov 2023 16:32:53 +0100 Subject: [PATCH 13/13] git-annex: show info message for hidden branches --- modules/annex/annex.go | 6 ++++++ routers/web/repo/branch.go | 5 +++++ templates/repo/branch/list.tmpl | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index bb049d77ed686..af7fb7182dc97 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -152,3 +152,9 @@ func IsAnnexed(blob *git.Blob) (bool, error) { } return true, nil } + +// IsAnnexRepo determines if repo is a git-annex enabled repository +func IsAnnexRepo(repo *git.Repository) bool { + _, _, err := git.NewCommand(repo.Ctx, "annex", "info").RunStdString(&git.RunOpts{Dir: repo.Path}) + return err == nil +} diff --git a/routers/web/repo/branch.go b/routers/web/repo/branch.go index bc72d5f2eca17..7090f0364e596 100644 --- a/routers/web/repo/branch.go +++ b/routers/web/repo/branch.go @@ -15,6 +15,7 @@ import ( git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unit" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" @@ -45,6 +46,10 @@ func Branches(ctx *context.Context) { ctx.Data["PageIsViewCode"] = true ctx.Data["PageIsBranches"] = true + if annex.IsAnnexRepo(ctx.Repo.GitRepo) { + ctx.Flash.Info("This repository is a git-annex repository, the git-annex branch as well as all synced/* branches are hidden to avoid accidental changes to or from them.", true) + } + page := ctx.FormInt("page") if page <= 1 { page = 1 diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl index cec5b6fc3e62d..8314a58ad5c8c 100644 --- a/templates/repo/branch/list.tmpl +++ b/templates/repo/branch/list.tmpl @@ -1,5 +1,5 @@ {{template "base/head" .}} -
+
{{template "repo/header" .}}
{{template "base/alert" .}}