Skip to content

Add parallel build support with -parallel and -jobs options #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add parallel build support with -parallel and -jobs options #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
module github.com/Debian/ratt

go 1.18
go 1.24.0

toolchain go1.24.4

require pault.ag/go/debian v0.12.0

require (
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 // indirect
golang.org/x/sync v0.17.0 // indirect
pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a // indirect
)
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E=
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
114 changes: 95 additions & 19 deletions ratt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,12 @@ import (
"os/exec"
"path/filepath"
"regexp"
"runtime"
"sort"
"strings"
"sync"

"golang.org/x/sync/errgroup"
"pault.ag/go/debian/control"
"pault.ag/go/debian/version"
)
Expand Down Expand Up @@ -100,6 +103,14 @@ var (
false,
"Output results in JSON format (currently only works in combination with -dry_run)")

parallel = flag.Bool("parallel",
false,
"Build packages in parallel")

jobs = flag.Int("jobs",
runtime.NumCPU(),
"Number of parallel build jobs (default: number of CPU cores)")

listsPrefixRe = regexp.MustCompile(`/([^/]*_dists_.*)_InRelease$`)
)

Expand Down Expand Up @@ -429,13 +440,91 @@ func getAptIndexPaths(dist string) ([]string, []string) {
return fallbackIndexPaths()
}

type buildJob struct {
src string
version version.Version
}

func buildPackagesSequential(builder *sbuild, rebuild map[string][]version.Version) (map[string]*buildResult, []dryRunBuild) {
buildresults := make(map[string]*buildResult)
var dryRunBuilds []dryRunBuild
cnt := 1

for src, versions := range rebuild {
sort.Sort(sort.Reverse(version.Slice(versions)))
newest := versions[0]
log.Printf("Building package %d of %d: %s\n", cnt, len(rebuild), src)
cnt++
result := builder.build(src, &newest)
if result.err != nil {
log.Printf("building %s failed: %v\n", src, result.err)
}
buildresults[src] = result

if *dryRun {
cmd := builder.buildCommandLine(src, &newest)
dryRunBuilds = append(dryRunBuilds, dryRunBuild{
Package: src,
Version: newest.String(),
SbuildCommand: strings.Join(cmd, " "),
})
}
}
return buildresults, dryRunBuilds
}

func buildPackagesParallel(builder *sbuild, rebuild map[string][]version.Version, numJobs int) (map[string]*buildResult, []dryRunBuild) {
var eg errgroup.Group
eg.SetLimit(numJobs)

buildresults := make(map[string]*buildResult)
var dryRunBuilds []dryRunBuild
var resultsMu sync.Mutex
cnt := 1

for src, versions := range rebuild {
src, versions := src, versions // capture loop variables
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove this line, capturing loop variables has not been necessary since Go 1.22 (https://go.dev/blog/loopvar-preview).

By the way, your editor should show you a warning about this and offer a code action to remove the line. If you don’t see this, maybe you use an old version of gopls, or something is wrong with your editor or IDE’s LSP setup…?

eg.Go(func() error {
sort.Sort(sort.Reverse(version.Slice(versions)))
newest := versions[0]
log.Printf("Building package %d of %d: %s\n", cnt, len(rebuild), src)
cnt++
result := builder.build(src, &newest)
if result.err != nil {
log.Printf("building %s failed: %v\n", src, result.err)
}

resultsMu.Lock()
buildresults[src] = result
if *dryRun {
cmd := builder.buildCommandLine(src, &newest)
dryRunBuilds = append(dryRunBuilds, dryRunBuild{
Package: src,
Version: newest.String(),
SbuildCommand: strings.Join(cmd, " "),
})
}
resultsMu.Unlock()
return nil
})
}

eg.Wait()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While currently, the function we run on the errgroup never returns an error, please add error handling for this eg.Wait() call regardless (to make future changes less surprising).

log.Printf("Completed building %d packages with %d workers\n", len(buildresults), numJobs)
return buildresults, dryRunBuilds
}

func main() {
flag.Parse()

if *jsonOutput && !*dryRun {
log.Fatal("-json can only be used together with -dry_run")
}

if *jobs <= 0 {
log.Fatal("-jobs must be a positive number")
}

if flag.NArg() == 0 {
log.Fatalf("Usage: %s [options] <path-to-changes-file>...\n", os.Args[0])
}
Expand Down Expand Up @@ -569,28 +658,15 @@ func main() {
dryRun: *dryRun,
extraDebs: debs,
}
cnt := 1

buildresults := make(map[string](*buildResult))
var dryRunBuilds []dryRunBuild
for src, versions := range rebuild {
sort.Sort(sort.Reverse(version.Slice(versions)))
newest := versions[0]
log.Printf("Building package %d of %d: %s \n", cnt, len(rebuild), src)
cnt++
result := builder.build(src, &newest)
if result.err != nil {
log.Printf("building %s failed: %v\n", src, result.err)
}
buildresults[src] = result

if *dryRun {
cmd := builder.buildCommandLine(src, &newest)
dryRunBuilds = append(dryRunBuilds, dryRunBuild{
Package: src,
Version: newest.String(),
SbuildCommand: strings.Join(cmd, " "),
})
}
if *parallel {
log.Printf("Building packages in parallel using %d workers\n", *jobs)
buildresults, dryRunBuilds = buildPackagesParallel(builder, rebuild, *jobs)
} else {
buildresults, dryRunBuilds = buildPackagesSequential(builder, rebuild)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

buildPackagesSequential(builder, rebuild) could be replaced with buildPackagesParallel(builder, rebuild, 1), right? Then, we can delete the entire buildPackagesSequential function and remove all the duplicate code.

}

var toInclude []string
Expand Down