diff --git a/go.mod b/go.mod index 60032e3..a5ec84e 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,13 @@ module github.com/Debian/ratt -go 1.18 +go 1.24.0 + +toolchain go1.24.4 require pault.ag/go/debian v0.12.0 require ( golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 // indirect + golang.org/x/sync v0.17.0 // indirect pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a // indirect ) diff --git a/go.sum b/go.sum index d511cfd..58462a0 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/ratt.go b/ratt.go index cb02a94..4b91d71 100644 --- a/ratt.go +++ b/ratt.go @@ -21,9 +21,12 @@ import ( "os/exec" "path/filepath" "regexp" + "runtime" "sort" "strings" + "sync" + "golang.org/x/sync/errgroup" "pault.ag/go/debian/control" "pault.ag/go/debian/version" ) @@ -100,6 +103,14 @@ var ( false, "Output results in JSON format (currently only works in combination with -dry_run)") + parallel = flag.Bool("parallel", + false, + "Build packages in parallel") + + jobs = flag.Int("jobs", + runtime.NumCPU(), + "Number of parallel build jobs (default: number of CPU cores)") + listsPrefixRe = regexp.MustCompile(`/([^/]*_dists_.*)_InRelease$`) ) @@ -429,6 +440,80 @@ func getAptIndexPaths(dist string) ([]string, []string) { return fallbackIndexPaths() } +type buildJob struct { + src string + version version.Version +} + +func buildPackagesSequential(builder *sbuild, rebuild map[string][]version.Version) (map[string]*buildResult, []dryRunBuild) { + buildresults := make(map[string]*buildResult) + var dryRunBuilds []dryRunBuild + cnt := 1 + + for src, versions := range rebuild { + sort.Sort(sort.Reverse(version.Slice(versions))) + newest := versions[0] + log.Printf("Building package %d of %d: %s\n", cnt, len(rebuild), src) + cnt++ + result := builder.build(src, &newest) + if result.err != nil { + log.Printf("building %s failed: %v\n", src, result.err) + } + buildresults[src] = result + + if *dryRun { + cmd := builder.buildCommandLine(src, &newest) + dryRunBuilds = append(dryRunBuilds, dryRunBuild{ + Package: src, + Version: newest.String(), + SbuildCommand: strings.Join(cmd, " "), + }) + } + } + return buildresults, dryRunBuilds +} + +func buildPackagesParallel(builder *sbuild, rebuild map[string][]version.Version, numJobs int) (map[string]*buildResult, []dryRunBuild) { + var eg errgroup.Group + eg.SetLimit(numJobs) + + buildresults := make(map[string]*buildResult) + var dryRunBuilds []dryRunBuild + var resultsMu sync.Mutex + cnt := 1 + + for src, versions := range rebuild { + src, versions := src, versions // capture loop variables + eg.Go(func() error { + sort.Sort(sort.Reverse(version.Slice(versions))) + newest := versions[0] + log.Printf("Building package %d of %d: %s\n", cnt, len(rebuild), src) + cnt++ + result := builder.build(src, &newest) + if result.err != nil { + log.Printf("building %s failed: %v\n", src, result.err) + } + + resultsMu.Lock() + buildresults[src] = result + if *dryRun { + cmd := builder.buildCommandLine(src, &newest) + dryRunBuilds = append(dryRunBuilds, dryRunBuild{ + Package: src, + Version: newest.String(), + SbuildCommand: strings.Join(cmd, " "), + }) + } + resultsMu.Unlock() + return nil + }) + } + + eg.Wait() + log.Printf("Completed building %d packages with %d workers\n", len(buildresults), numJobs) + return buildresults, dryRunBuilds +} + func main() { flag.Parse() @@ -436,6 +521,10 @@ func main() { log.Fatal("-json can only be used together with -dry_run") } + if *jobs <= 0 { + log.Fatal("-jobs must be a positive number") + } + if flag.NArg() == 0 { log.Fatalf("Usage: %s [options] ...\n", os.Args[0]) } @@ -569,28 +658,15 @@ func main() { dryRun: *dryRun, extraDebs: debs, } - cnt := 1 + buildresults := make(map[string](*buildResult)) var dryRunBuilds []dryRunBuild - for src, versions := range rebuild { - sort.Sort(sort.Reverse(version.Slice(versions))) - newest := versions[0] - log.Printf("Building package %d of %d: %s \n", cnt, len(rebuild), src) - cnt++ - result := builder.build(src, &newest) - if result.err != nil { - log.Printf("building %s failed: %v\n", src, result.err) - } - buildresults[src] = result - if *dryRun { - cmd := builder.buildCommandLine(src, &newest) - dryRunBuilds = append(dryRunBuilds, dryRunBuild{ - Package: src, - Version: newest.String(), - SbuildCommand: strings.Join(cmd, " "), - }) - } + if *parallel { + log.Printf("Building packages in parallel using %d workers\n", *jobs) + buildresults, dryRunBuilds = buildPackagesParallel(builder, rebuild, *jobs) + } else { + buildresults, dryRunBuilds = buildPackagesSequential(builder, rebuild) } var toInclude []string