Add s3

conf.example.yml

@@ -267,7 +267,13 @@ destination:
       keep: 5 # only keeps x backups
       bare: true # clone the repositories as bare
       lfs: true # clone lfs repos, !! ATTENTION !! git and git-lfs must be installed on the system!
-
+  s3:
+   - endpoint: somewhere:9000 # whatever your s3 endpoint is
+     structured: true # checks repos out like hostersite/user|organization/repo
+     bucket: your-bucket-name
+     accesskey: your-access-key # can be an environment variable, just don't add a $ in front of it
+     secretkey: your-secret-key # can be an environment variable, just don't add a $ in front of it
+     usessl: true # wheter to use ssl or not
 cron: 0 22 * * * # optional - when cron is not provided, the program runs once and exits.
 # Otherwise, it runs according to the cron schedule.
 # See timezone commentary in docker-compose.yml for making sure this container runs

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/gookit/color v1.5.4
 	github.com/ktrysmt/go-bitbucket v0.9.80
 	github.com/melbahja/goph v1.4.0
+	github.com/minio/minio-go/v7 v7.0.75
 	github.com/prometheus/client_golang v1.19.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/zerolog v1.33.0
@@ -33,20 +34,27 @@ require (
 	github.com/cloudflare/circl v1.3.9 // indirect
 	github.com/cyphar/filepath-securejoin v0.3.1 // indirect
 	github.com/davidmz/go-pageant v1.0.2 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/go-fed/httpsig v1.1.0 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
@@ -55,13 +63,15 @@ require (
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/rs/xid v1.5.0 // indirect
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect
 	github.com/skeema/knownhosts v1.3.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	golang.org/x/net v0.27.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect

go.sum

@@ -37,6 +37,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
 github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
 github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
@@ -55,6 +57,10 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
 github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys=
 github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 h1:UjoPNDAQ5JPCjlxoJd6K8ALZqSDDhk2ymieAZOVaDg0=
 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85/go.mod h1:fR6z1Ie6rtF7kl/vBYMfgD5/G5B1blui7z426/sj2DU=
@@ -71,6 +77,8 @@ github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27u
 github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
 github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
@@ -89,6 +97,11 @@ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q
 github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
+github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -111,6 +124,10 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/melbahja/goph v1.4.0 h1:z0PgDbBFe66lRYl3v5dGb9aFgPy0kotuQ37QOwSQFqs=
 github.com/melbahja/goph v1.4.0/go.mod h1:uG+VfK2Dlhk+O32zFrRlc3kYKTlV6+BtvPWd/kK7U68=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.75 h1:0uLrB6u6teY2Jt+cJUVi9cTvDRuBKWSRzSAcznRkwlE=
+github.com/minio/minio-go/v7 v7.0.75/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -141,6 +158,7 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
 github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=

main.go

@@ -6,13 +6,15 @@ import (
 	"io"
 	"os"
 	"os/user"
+	"path"
 	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/cooperspencer/gickup/onedev"
+	"github.com/cooperspencer/gickup/s3"
 	"github.com/cooperspencer/gickup/sourcehut"
 	"github.com/go-git/go-git/v5"
 	"github.com/google/go-cmp/cmp"
@@ -183,6 +185,75 @@ func backup(repos []types.Repo, conf *types.Conf) {
 			prometheus.DestinationBackupsComplete.WithLabelValues("local").Inc()
 		}
 
+		for _, d := range conf.Destination.S3 {
+			repotime := time.Now()
+			status := 0
+
+			log.Info().
+				Str("stage", "s3").
+				Str("url", d.Endpoint).
+				Msgf("pushing %s to %s", types.Blue(r.Name), d.Bucket)
+
+			if !cli.Dry {
+				tempname := fmt.Sprintf("s3-%x", repotime)
+				tempdir, err := os.MkdirTemp(os.TempDir(), tempname)
+				if err != nil {
+					log.Error().
+						Str("stage", "tempclone").
+						Str("url", r.URL).
+						Msg(err.Error())
+					continue
+				}
+
+				if d.Structured {
+					r.Name = path.Join(r.Hoster, r.Owner, r.Name)
+				}
+
+				defer os.RemoveAll(tempdir)
+				_, err = local.TempClone(r, path.Join(tempdir, r.Name))
+				if err != nil {
+					if err == git.NoErrAlreadyUpToDate {
+						log.Info().
+							Str("stage", "s3").
+							Str("url", r.URL).
+							Msg(err.Error())
+					} else {
+						log.Error().
+							Str("stage", "tempclone").
+							Str("url", r.URL).
+							Str("git", "clone").
+							Msg(err.Error())
+						os.RemoveAll(tempdir)
+						continue
+					}
+				}
+
+				// Check if environment variables are used for accesskey and secretkey
+				d.AccessKey, err = d.GetKey(d.AccessKey)
+				if err != nil {
+					log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error())
+				}
+				d.SecretKey, err = d.GetKey(d.SecretKey)
+				if err != nil {
+					log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error())
+				}
+
+				err = s3.UploadDirToS3(tempdir, d)
+				if err != nil {
+					log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error())
+				}
+				err = s3.DeleteObjectsNotInRepo(tempdir, r.Name, d)
+				if err != nil {
+					log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error())
+				}
+				prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "s3", d.Endpoint).Set(time.Since(repotime).Seconds())
+				status = 1
+
+				prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "s3", d.Endpoint).Set(float64(status))
+				prometheus.DestinationBackupsComplete.WithLabelValues("s3").Inc()
+			}
+		}
+
 		for _, d := range conf.Destination.Gitea {
 			if d.MirrorInterval != "" {
 				log.Warn().

s3/s3.go

@@ -0,0 +1,104 @@
+package s3
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/cooperspencer/gickup/logger"
+	"github.com/cooperspencer/gickup/types"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/rs/zerolog"
+)
+
+var (
+	sub zerolog.Logger
+)
+
+// UploadDirToS3 uploads the contents of a directory to S3-compatible storage
+func UploadDirToS3(directory string, s3repo types.S3Repo) error {
+	// Initialize minio client object.
+	client, err := minio.New(s3repo.Endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, ""),
+		Secure: s3repo.UseSSL,
+		Region: s3repo.Region,
+	})
+	if err != nil {
+		return err
+	}
+	err = filepath.Walk(directory, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() || (info.Mode()&os.ModeSymlink != 0) {
+			return nil // Skip directories and symbolic links
+		}
+
+		// Open the file
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		// Get file info
+		stat, err := file.Stat()
+		if err != nil {
+			return err
+		}
+
+		// Upload the file to S3-compatible storage
+		objectName := filepath.ToSlash(path[len(directory)+1:]) // Object name in bucket
+
+		_, err = client.PutObject(context.Background(), s3repo.Bucket, objectName, file, stat.Size(), minio.PutObjectOptions{})
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// DeleteObjectsNotInRepo deletes objects from the bucket that are not present in the repository
+func DeleteObjectsNotInRepo(directory, bucketdir string, s3repo types.S3Repo) error {
+	sub = logger.CreateSubLogger("stage", "s3", "endpoint", s3repo.Endpoint, "bucket", s3repo.Bucket)
+	// Initialize minio client object.
+	client, err := minio.New(s3repo.Endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, ""),
+		Secure: s3repo.UseSSL,
+		Region: s3repo.Region,
+	})
+	if err != nil {
+		return err
+	}
+
+	// List objects in the bucket within the specified directory (prefix)
+	for object := range client.ListObjects(context.Background(), s3repo.Bucket, minio.ListObjectsOptions{
+		Prefix:    bucketdir + "/", // Only list objects within the specific bucket directory
+		Recursive: true,
+	}) {
+		if object.Err != nil {
+			return object.Err
+		}
+		objectPath := filepath.Join(directory, object.Key)
+		if _, err := os.Stat(objectPath); err != nil {
+			if os.IsNotExist(err) {
+				sub.Debug().Msgf("Removing %s from bucket %s", object.Key, s3repo.Bucket)
+				// File does not exist in the repository, delete it from the bucket
+				err := client.RemoveObject(context.Background(), s3repo.Bucket, object.Key, minio.RemoveObjectOptions{})
+				if err != nil {
+					return err
+				}
+			} else {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

types/types.go

@@ -29,6 +29,7 @@ type Destination struct {
 	Gogs      []GenRepo `yaml:"gogs"`
 	OneDev    []GenRepo `yaml:"onedev"`
 	Sourcehut []GenRepo `yaml:"sourcehut"`
+	S3        []S3Repo  `yaml:"s3"`
 }
 
 // Count TODO.
@@ -39,7 +40,8 @@ func (dest Destination) Count() int {
 		len(dest.Github) +
 		len(dest.Gitlab) +
 		len(dest.OneDev) +
-		len(dest.Sourcehut)
+		len(dest.Sourcehut) +
+		len(dest.S3)
 }
 
 // Local TODO.
@@ -529,3 +531,28 @@ func StatRemote(remoteURL, sshURL string, repo GenRepo) bool {
 
 	return err == nil
 }
+
+type S3Repo struct {
+	Bucket     string `yaml:"bucket"`
+	Endpoint   string `yaml:"endpoint"`
+	AccessKey  string `yaml:"accesskey"`
+	SecretKey  string `yaml:"secretkey"`
+	Region     string `yaml:"region"`
+	UseSSL     bool   `yaml:"usessl"`
+	Structured bool   `yaml:"structured"`
+}
+
+func (s3 S3Repo) GetKey(accessString string) (string, error) {
+	if accessString == "" {
+		return "", fmt.Errorf("accesskey or secretkey are empty")
+	}
+	if accessString != "" {
+		envstring := os.Getenv(accessString)
+		if envstring != "" {
+			return envstring, nil
+		}
+		return accessString, nil
+	}
+
+	return "", fmt.Errorf("accesskey or secretkey are empty")
+}