handle multiple configs (#114)

* handle multiple configs

* upgrade golangci_lint_version

* SnakeCase and replaces ioutil with os

.github/workflows/validate-lint.yml

@@ -7,7 +7,7 @@ on:
 
 env:
   GO_VERSION: 1.17
-  GOLANGCI_LINT_VERSION: v1.45.0
+  GOLANGCI_LINT_VERSION: v1.48.0
 
 jobs:
 

conf.example.yml

@@ -177,3 +177,11 @@ metrics:
     urls:
       - http(s)://url-to-make-request-to
       - http(s)://another-url-to-make-request-to
+
+---
+
+# you can define separate source and destination pairs,
+# like "mirror all repos from github to gitea but keep gitlab repos up-to-date in ~/backup"
+# if cron is defined in the first config, this cron interval will be used for all the other confgurations, except it has one of its own.
+# if cron is not enabled for the first config, cron will not run for any other configuration
+# metrics configuration is always used from the first configuration

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/zerolog v1.27.0
 	github.com/sergi/go-diff v1.2.0 // indirect
+	github.com/stretchr/testify v1.8.0 // indirect
 	github.com/xanzy/go-gitlab v0.69.0
 	github.com/xanzy/ssh-agent v0.3.1 // indirect
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa

go.sum

@@ -374,14 +374,17 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/xanzy/go-gitlab v0.69.0 h1:sPci9xHzlX+lcJvPqNu3y3BQpePuR2R694Bal4AeyB8=
 github.com/xanzy/go-gitlab v0.69.0/go.mod h1:o4yExCtdaqlM8YGdDJWuZoBmfxBsmA9TPEjs9mx1UO4=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=

main.go

@@ -1,15 +1,18 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
-	"github.com/cooperspencer/gickup/onedev"
-	"io/ioutil"
+	"io"
 	"os"
 	"os/user"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
+	"github.com/cooperspencer/gickup/onedev"
+
 	"github.com/alecthomas/kong"
 	"github.com/cooperspencer/gickup/bitbucket"
 	"github.com/cooperspencer/gickup/gitea"
@@ -29,36 +32,55 @@ import (
 )
 
 var cli struct {
-	Configfile string `arg name:"conf" help:"Path to the configfile." default:"conf.yml"`
-	Version    bool   `flag name:"version" help:"Show version."`
-	Dry        bool   `flag name:"dryrun" help:"Make a dry-run."`
-	Quiet      bool   `flag name:"quiet" help:"Output only warnings, errors, and fatal messages to stderr log output"`
-	Silent     bool   `flag name:"silent" help:"Suppress all stderr log output"`
+	Configfiles []string `arg name:"conf" help:"Path to the configfile." default:"conf.yml"`
+	Version     bool     `flag name:"version" help:"Show version."`
+	Dry         bool     `flag name:"dryrun" help:"Make a dry-run."`
+	Quiet       bool     `flag name:"quiet" help:"Output only warnings, errors, and fatal messages to stderr log output"`
+	Silent      bool     `flag name:"silent" help:"Suppress all stderr log output"`
 }
 
 var version = "unknown"
 
-func readConfigFile(configfile string) *types.Conf {
-	cfgdata, err := ioutil.ReadFile(configfile)
+func readConfigFile(configfile string) []*types.Conf {
+	conf := []*types.Conf{}
+	cfgdata, err := os.ReadFile(configfile)
 	if err != nil {
 		log.Fatal().
 			Str("stage", "readconfig").
 			Str("file", configfile).
 			Msgf("Cannot open config file from %s", types.Red(configfile))
 	}
 
-	t := types.Conf{}
+	//	t := types.Conf{}
 
-	err = yaml.Unmarshal(cfgdata, &t)
+	dec := yaml.NewDecoder(bytes.NewReader(cfgdata))
 
-	if err != nil {
-		log.Fatal().
-			Str("stage", "readconfig").
-			Str("file", configfile).
-			Msg("Cannot map yml config file to interface, possible syntax error")
+	//	err = yaml.Unmarshal(cfgdata, &t)
+
+	i := 0
+	for {
+		var c *types.Conf
+		err = dec.Decode(&c)
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			if len(conf) > 0 {
+				log.Fatal().
+					Str("stage", "readconfig").
+					Str("file", configfile).
+					Msgf("an error occured in the %d place of %s", i, configfile)
+			} else {
+				log.Fatal().
+					Str("stage", "readconfig").
+					Str("file", configfile).
+					Msg("Cannot map yml config file to interface, possible syntax error")
+			}
+		}
+		conf = append(conf, c)
+		i++
 	}
 
-	return &t
+	return conf
 }
 
 func getUserHome() (string, error) {
@@ -149,43 +171,45 @@ func backup(repos []types.Repo, conf *types.Conf) {
 	}
 }
 
-func runBackup(conf *types.Conf) {
+func runBackup(conf *types.Conf, num int) {
 	log.Info().Msg("Backup run starting")
 
+	numstring := strconv.Itoa(num)
+
 	startTime := time.Now()
 
 	prometheus.JobsStarted.Inc()
 
 	// Github
 	repos := github.Get(conf)
-	prometheus.CountReposDiscovered.WithLabelValues("github").Set(float64(len(repos)))
+	prometheus.CountReposDiscovered.WithLabelValues("github", numstring).Set(float64(len(repos)))
 	backup(repos, conf)
 
 	// Gitea
 	repos = gitea.Get(conf)
-	prometheus.CountReposDiscovered.WithLabelValues("gitea").Set(float64(len(repos)))
+	prometheus.CountReposDiscovered.WithLabelValues("gitea", numstring).Set(float64(len(repos)))
 	backup(repos, conf)
 
 	// Gogs
 	repos = gogs.Get(conf)
-	prometheus.CountReposDiscovered.WithLabelValues("gogs").Set(float64(len(repos)))
+	prometheus.CountReposDiscovered.WithLabelValues("gogs", numstring).Set(float64(len(repos)))
 	backup(repos, conf)
 
 	// Gitlab
 	repos = gitlab.Get(conf)
-	prometheus.CountReposDiscovered.WithLabelValues("gitlab").Set(float64(len(repos)))
+	prometheus.CountReposDiscovered.WithLabelValues("gitlab", numstring).Set(float64(len(repos)))
 	backup(repos, conf)
 
 	repos = bitbucket.Get(conf)
-	prometheus.CountReposDiscovered.WithLabelValues("bitbucket").Set(float64(len(repos)))
+	prometheus.CountReposDiscovered.WithLabelValues("bitbucket", numstring).Set(float64(len(repos)))
 	backup(repos, conf)
 
 	repos = whatever.Get(conf)
-	prometheus.CountReposDiscovered.WithLabelValues("whatever").Set(float64(len(repos)))
+	prometheus.CountReposDiscovered.WithLabelValues("whatever", numstring).Set(float64(len(repos)))
 	backup(repos, conf)
 
 	repos = onedev.Get(conf)
-	prometheus.CountReposDiscovered.WithLabelValues("onedev").Set(float64(len(repos)))
+	prometheus.CountReposDiscovered.WithLabelValues("onedev", numstring).Set(float64(len(repos)))
 	backup(repos, conf)
 
 	endTime := time.Now()
@@ -250,51 +274,74 @@ func main() {
 			Msgf("this is a %s", types.Blue("dry run"))
 	}
 
-	log.Info().Str("file", cli.Configfile).
-		Msgf("Reading %s", types.Green(cli.Configfile))
+	confs := []*types.Conf{}
+	for _, f := range cli.Configfiles {
+		log.Info().Str("file", f).
+			Msgf("Reading %s", types.Green(f))
 
-	conf := readConfigFile(cli.Configfile)
-	if conf.Log.Timeformat == "" {
-		conf.Log.Timeformat = timeformat
+		confs = append(confs, readConfigFile(f)...)
+	}
+	if confs[0].Log.Timeformat == "" {
+		confs[0].Log.Timeformat = timeformat
 	}
 
-	log.Logger = logger.CreateLogger(conf.Log)
+	log.Logger = logger.CreateLogger(confs[0].Log)
 
-	// one pair per source-destination
-	pairs := conf.Source.Count() * conf.Destination.Count()
-	log.Info().
-		Int("sources", conf.Source.Count()).
-		Int("destinations", conf.Destination.Count()).
-		Int("pairs", pairs).
-		Msg("Configuration loaded")
+	validcron := confs[0].HasValidCronSpec()
 
-	if conf.HasValidCronSpec() {
-		c := cron.New()
+	var c *cron.Cron
 
-		logNextRun(conf)
+	if validcron {
+		c = cron.New()
+		c.Start()
+	}
 
-		_, err := c.AddFunc(conf.Cron, func() {
-			runBackup(conf)
-		})
-		if err != nil {
-			log.Fatal().
-				Int("sources", conf.Source.Count()).
-				Int("destinations", conf.Destination.Count()).
-				Int("pairs", pairs).
-				Msg(err.Error())
+	sourcecount := 0
+	destinationcount := 0
+	// one pair per source-destination
+	for num, conf := range confs {
+		pairs := conf.Source.Count() * conf.Destination.Count()
+		sourcecount += conf.Source.Count()
+		destinationcount += conf.Destination.Count()
+		log.Info().
+			Int("sources", conf.Source.Count()).
+			Int("destinations", conf.Destination.Count()).
+			Int("pairs", pairs).
+			Msg("Configuration loaded")
+
+		if !conf.HasValidCronSpec() {
+			conf.Cron = confs[0].Cron
 		}
 
-		c.Start()
+		if conf.HasValidCronSpec() && validcron {
+			conf := conf // https://stackoverflow.com/questions/57095167/how-do-i-create-multiple-cron-function-by-looping-through-a-list
+			num := num
+
+			logNextRun(conf)
+
+			_, err := c.AddFunc(conf.Cron, func() {
+				runBackup(conf, num)
+			})
+			if err != nil {
+				log.Fatal().
+					Int("sources", conf.Source.Count()).
+					Int("destinations", conf.Destination.Count()).
+					Int("pairs", pairs).
+					Msg(err.Error())
+			}
+		} else {
+			runBackup(conf, num)
+		}
+	}
 
-		if conf.HasAllPrometheusConf() {
-			prometheus.CountSourcesConfigured.Add(float64(conf.Source.Count()))
-			prometheus.CountDestinationsConfigured.Add(float64(conf.Destination.Count()))
-			prometheus.Serve(conf.Metrics.Prometheus)
+	if validcron {
+		if confs[0].HasAllPrometheusConf() {
+			prometheus.CountSourcesConfigured.Add(float64(sourcecount))
+			prometheus.CountDestinationsConfigured.Add(float64(destinationcount))
+			prometheus.Serve(confs[0].Metrics.Prometheus)
 		} else {
 			playsForever()
 		}
-	} else {
-		runBackup(conf)
 	}
 }
 

metrics/prometheus/prometheus.go

@@ -23,7 +23,7 @@ var CountDestinationsConfigured = promauto.NewGauge(prometheus.GaugeOpts{
 var CountReposDiscovered = promauto.NewGaugeVec(prometheus.GaugeOpts{
 	Name: "gickup_repos_discovered",
 	Help: "The count of sources configured",
-}, []string{"source_name"})
+}, []string{"source_name", "config_number"})
 
 var JobsComplete = promauto.NewCounter(prometheus.CounterOpts{
 	Name: "gickup_jobs_complete",

onedev/onedev.go

@@ -2,6 +2,7 @@ package onedev
 
 import (
 	"fmt"
+
 	"github.com/cooperspencer/gickup/types"
 	"github.com/cooperspencer/onedev"
 	"github.com/rs/zerolog/log"