Merge branch 'main' into pkglint

chainguard-dev · Oct 10, 2023 · 4156090 · 4156090
2 parents 497f8ca + 2c1195f
commit 4156090
Show file tree

Hide file tree

Showing 14 changed files with 373 additions and 56 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,30 @@
+## Melange Pull Request Template
+
+<!--
+*** PULL REQUEST CHECKLIST: PLEASE START HERE ***
+
+The single most important feature of melange is that we can build Wolfi.
+
+Many changes to melange introduce a risk of breaking the build, and sometimes
+these are not flushed out until a package is changed (much) later.  This
+pertains to basic execution, SCA changes, linter changes, and more.
+-->
+
+### Functional Changes
+
+- [ ] This change can build all of Wolfi without errors (describe results in notes)
+
+Notes:
+
+### SCA Changes
+
+- [ ] Examining several representative APKs show no regression / the desired effect (details in notes)
+
+Notes:
+
+### Linter
+
+- [ ] The new check is clean across Wolfi
+- [ ] The new check is opt-in or a warning
+
+Notes:
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
@@ -42,16 +42,15 @@ archives:
   - files:
       - LICENSE
     wrap_in_directory: true
-    rlcp: true
 
 brews:
   - name: melange
-    tap:
+    repository:
       owner: chainguard-dev
       name: homebrew-tap
       branch: main
       token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}"
-    url_template: "https://github.com/chainguard-dev/melange/releases/download/{{ .Version }}/{{ .ArtifactName }}"
+    url_template: "https://github.com/chainguard-dev/melange/releases/download/v{{ .Version }}/{{ .ArtifactName }}"
     folder: Formula
     commit_author:
       name: guardian

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,153 @@
+# Major changes from 0.4.0 to 0.5.0
+
+* Rename Contexts to Builds by @jonjohnsonjr in https://github.com/chainguard-dev/melange/pull/525
+
+* Add missing context propagation by @jonjohnsonjr in https://github.com/chainguard-dev/melange/pull/527
+
+* Bug fix: silent env var replacement by @luhring in https://github.com/chainguard-dev/melange/pull/533
+
+* Add otel spans by @jonjohnsonjr in https://github.com/chainguard-dev/melange/pull/529
+
+* docs: explain how build cache works practically by @luhring in https://github.com/chainguard-dev/melange/pull/537
+
+* build: package: forcibly treat libc as a shared library by @kaniini in https://github.com/chainguard-dev/melange/pull/538
+
+* Change git-checkout depth default to 1 by @luhring in https://github.com/chainguard-dev/melange/pull/539
+
+* Fix/python version issue by @mesaglio in https://github.com/chainguard-dev/melange/pull/532
+
+* pull in apko with fix for blank SOURCE_DATE_EPOCH by @deitch in https://github.com/chainguard-dev/melange/pull/542
+
+* Remove use of deprecated WaitImmediate by @jonjohnsonjr in https://github.com/chainguard-dev/melange/pull/528
+
+* lima startup issues fixed by @deitch in https://github.com/chainguard-dev/melange/pull/543
+
+* add dir option to ruby pipelines as not all gemspecs live in the root… by @rawlingsj in https://github.com/chainguard-dev/melange/pull/544
+
+* K8s runner template bugs by @joshrwolf in https://github.com/chainguard-dev/melange/pull/550
+
+* K8s runner retry exec by @joshrwolf in https://github.com/chainguard-dev/melange/pull/549
+
+* Refactor some pipelines to more safely use pipeline expansions by @kaniini in https://github.com/chainguard-dev/melange/pull/554
+
+* use go-apk.FullFS for retrieving builder workspaces by @joshrwolf in https://github.com/chainguard-dev/melange/pull/548
+
+* Correct the variable name in the patch pipeline by @mattmoor in https://github.com/chainguard-dev/melange/pull/555
+
+* Avoid using pargzip for compression by @jonjohnsonjr in https://github.com/chainguard-dev/melange/pull/558
+
+* skip the cache mount for kubernetes runner builds by @joshrwolf in https://github.com/chainguard-dev/melange/pull/566
+
+* Make sure we log errors. by @mattmoor in https://github.com/chainguard-dev/melange/pull/570
+
+* Log errors bundling, enable GGCR Warn/Progress logs by @mattmoor in https://github.com/chainguard-dev/melange/pull/574
+
+* add k8s runner config loading from envvars by @joshrwolf in https://github.com/chainguard-dev/melange/pull/571
+
+* Remove `wget -q` from `fetch` by @mattmoor in https://github.com/chainguard-dev/melange/pull/575
+
+* Tweak the strip pipeline so that it never fails for deleted files by @mattmoor in https://github.com/chainguard-dev/melange/pull/573
+
+* convert/python: check if release is found by @Dentrax in https://github.com/chainguard-dev/melange/pull/572
+
+* Fix subpackage SBOM generation by @jonjohnsonjr in https://github.com/chainguard-dev/melange/pull/569
+
+* renovate: update to use new config infrastructure by @Elizafox in https://github.com/chainguard-dev/melange/pull/585
+
+* pipelines: meson/configure: explicitly invoke meson setup action by @kaniini in https://github.com/chainguard-dev/melange/pull/582
+
+* Updates on ci and release by @cpanato in https://github.com/chainguard-dev/melange/pull/583
+
+* Make var transforms work in bump by @Elizafox in https://github.com/chainguard-dev/melange/pull/586
+
+* container: bubblewrap: do not defer closing files by @kaniini in https://github.com/chainguard-dev/melange/pull/596
+
+* build: package: add pkgconf-based SCA to catalog SDKs which use it by @kaniini in https://github.com/chainguard-dev/melange/pull/590
+
+* Version transform block in melange by @Elizafox in https://github.com/chainguard-dev/melange/pull/588
+
+* Add docs about custom pipelines, defining and using. by @vaikas in https://github.com/chainguard-dev/melange/pull/604
+
+* Support for setting context in .melange.k8s.yaml by @tcnghia in https://github.com/chainguard-dev/melange/pull/605
+
+* allow override go version for uses: go/build and go/install by @rawlingsj in https://github.com/chainguard-dev/melange/pull/606
+
+* add melange sign command, slightly refactor and make public the signing methods by @joshrwolf in https://github.com/chainguard-dev/melange/pull/607
+
+* support substitutions in provides lists by @imjasonh in https://github.com/chainguard-dev/melange/pull/610
+
+* Add ${{targets.contextdir}} by @kaniini in https://github.com/chainguard-dev/melange/pull/622
+
+* add --force option to recreate apk indexes with given signatures by @joshrwolf in https://github.com/chainguard-dev/melange/pull/626
+
+* cli: index: add --signing-key, --source and --merge options by @kaniini in https://github.com/chainguard-dev/melange/pull/629
+
+* feat: support --recurse-submodules in git clone by @stormqueen1990 in https://github.com/chainguard-dev/melange/pull/639
+
+* readlinkfs: ignore some security-module specific xattrs by @kaniini in https://github.com/chainguard-dev/melange/pull/640
+
+* Add --wolfi-defaults flag, clean up flag handling. by @vaikas in https://github.com/chainguard-dev/melange/pull/641
+
+* Add a maven/configure-mirror pipeline to redirect to GCP. by @dlorenc in https://github.com/chainguard-dev/melange/pull/644
+
+* add builtin pipelines for python by @imjasonh in https://github.com/chainguard-dev/melange/pull/642
+
+* package: dereference symlinks for aliased pkg-config modules by @kaniini in https://github.com/chainguard-dev/melange/pull/653
+
+* feat: add output logs for the apkbuild converter by @stormqueen1990 in https://github.com/chainguard-dev/melange/pull/660
+
+* Change default python-version from 3.11 to 3. by @vaikas in https://github.com/chainguard-dev/melange/pull/649
+
+* feat: add new Perl pipelines for Makefile generation and cleanup by @stormqueen1990 in https://github.com/chainguard-dev/melange/pull/657
+
+* add ${{package.full-version}} = ${{package.version}}-r${{package.epoch}} by @vaikas in https://github.com/chainguard-dev/melange/pull/662
+
+* config: copy all subpackage variables when doing a range expansion by @kaniini in https://github.com/chainguard-dev/melange/pull/661
+
+* docs: add documentation for built-in pipelines by @stormqueen1990 in https://github.com/chainguard-dev/melange/pull/665
+
+* construct the package.full-version in higher context than just pipeline. by @vaikas in https://github.com/chainguard-dev/melange/pull/667
+
+* package: constrain library SCA to library search paths only by @kaniini in https://github.com/chainguard-dev/melange/pull/669
+
+* Add environment var overriding to the pipeline. by @Elizafox in https://github.com/chainguard-dev/melange/pull/676
+
+* package: only constrain library search paths for provides entries by @kaniini in https://github.com/chainguard-dev/melange/pull/678
+
+* Add pecl pipelines for phpize & install. by @vaikas in https://github.com/chainguard-dev/melange/pull/679
+
+* Add regression tests for workdir propagation, fix long-standing bug with propagation across referenced pipelines by @kaniini in https://github.com/chainguard-dev/melange/pull/681
+
+* git-checkout: Allow tags to matched annotated tag SHAs, don't allow by @wlynch in https://github.com/chainguard-dev/melange/pull/686
+
+* Binary package linting by @Elizafox in https://github.com/chainguard-dev/melange/pull/680
+
+* add goreleaser pipeline by @developer-guy in https://github.com/chainguard-dev/melange/pull/671
+
+* Disable linters on -compat packages by @Elizafox in https://github.com/chainguard-dev/melange/pull/691
+
+* log and continue when .pc file can't be loaded by @imjasonh in https://github.com/chainguard-dev/melange/pull/694
+
+* Add dev, opt, and srv linters by @Elizafox in https://github.com/chainguard-dev/melange/pull/697
+
+* Add worldwrite linter by @Elizafox in https://github.com/chainguard-dev/melange/pull/698
+
+* build: do not run linters on skipped subpackages by @kaniini in https://github.com/chainguard-dev/melange/pull/701
+
+* Add post-file walk linting and empty package linting by @Elizafox in https://github.com/chainguard-dev/melange/pull/700
+
+* Refactor the package linter into a submodule by @Elizafox in https://github.com/chainguard-dev/melange/pull/706
+
+* build: refactor package linter invocation by @kaniini in https://github.com/chainguard-dev/melange/pull/708
+
+* pipelines: strip: use -g by default when stripping by @kaniini in https://github.com/chainguard-dev/melange/pull/722
+
+* update alpine-go to latest git to fix indexing by @kaniini in https://github.com/chainguard-dev/melange/pull/723
+
+* Add stripped file linter by @Elizafox in https://github.com/chainguard-dev/melange/pull/724
+
+* Enable linters to warn (via callback) instead of just failing. by @mattmoor in https://github.com/chainguard-dev/melange/pull/739
+
 # Major changes from 0.3.0 to 0.4.0
 
 * The embedded apko component has been updated to 0.9.0.

diff --git a/docs/LINTER.md b/docs/LINTER.md
@@ -45,6 +45,6 @@ package:
   checks:
     disable:
       - setuidgid  # Package is meant to have setuid binaries
-        debug      # Toolchain problems require we keep debug info
+      - debug      # Toolchain problems require we keep debug info
         ...
 ```
diff --git a/docs/md/melange_build.md b/docs/md/melange_build.md
@@ -42,6 +42,7 @@ melange build [flags]
       --dependency-log string       log dependencies to a specified file
       --empty-workspace             whether the build workspace should be empty
       --env-file string             file to use for preloaded environment variables
+      --fail-on-lint-warning        turns linter warnings into failures
       --generate-index              whether to generate APKINDEX.tar.gz (default true)
       --guest-dir string            directory used for the build environment guest
   -h, --help                        help for build

diff --git a/go.mod b/go.mod
@@ -28,7 +28,7 @@ require (
 	github.com/kubescape/go-git-url v0.0.25
 	github.com/lima-vm/lima v0.17.2
 	github.com/opencontainers/image-spec v1.1.0-rc5
-	github.com/package-url/packageurl-go v0.1.1
+	github.com/package-url/packageurl-go v0.1.2
 	github.com/pkg/errors v0.9.1
 	github.com/psanford/memfs v0.0.0-20230130182539-4dbf7e3e865e
 	github.com/sirupsen/logrus v1.9.3
@@ -37,7 +37,7 @@ require (
 	github.com/yookoala/realpath v1.0.0
 	github.com/zealic/xignore v0.3.3
 	gitlab.alpinelinux.org/alpine/go v0.8.1-0.20230928153721-5381bfaecf9b
-	go.opentelemetry.io/otel v1.18.0
+	go.opentelemetry.io/otel v1.19.0
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	golang.org/x/sync v0.3.0
 	golang.org/x/sys v0.12.0
@@ -152,8 +152,8 @@ require (
 	go.lsp.dev/uri v0.3.0 // indirect
 	go.mongodb.org/mongo-driver v1.12.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/otel/metric v1.18.0 // indirect
-	go.opentelemetry.io/otel/trace v1.18.0 // indirect
+	go.opentelemetry.io/otel/metric v1.19.0 // indirect
+	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	golang.org/x/build v0.0.0-20230825202850-d75d44855ad1 // indirect
 	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect

diff --git a/go.sum b/go.sum
@@ -482,8 +482,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
 github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
-github.com/package-url/packageurl-go v0.1.1 h1:KTRE0bK3sKbFKAk3yy63DpeskU7Cvs/x/Da5l+RtzyU=
-github.com/package-url/packageurl-go v0.1.1/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c=
+github.com/package-url/packageurl-go v0.1.2 h1:0H2DQt6DHd/NeRlVwW4EZ4oEI6Bn40XlNPRqegcxuo4=
+github.com/package-url/packageurl-go v0.1.2/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
@@ -601,12 +601,12 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/otel v1.18.0 h1:TgVozPGZ01nHyDZxK5WGPFB9QexeTMXEH7+tIClWfzs=
-go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI=
-go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvzhtRrSQ=
-go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k=
-go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10=
-go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0=
+go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
+go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
+go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
+go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
+go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
+go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
 golang.org/x/build v0.0.0-20230825202850-d75d44855ad1 h1:N/sI0tXL5Slh4SwrARRCai/fw3UHWmko+ZfE3VD0z+8=
 golang.org/x/build v0.0.0-20230825202850-d75d44855ad1/go.mod h1:LblaorLo4w94wk4xZvKeLSaFLku6WbfOih5CoMhhvVc=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

diff --git a/pkg/build/build.go b/pkg/build/build.go
@@ -91,6 +91,7 @@ type Build struct {
 	Debug              bool
 	DebugRunner        bool
 	LogPolicy          []string
+	FailOnLintWarning  bool
 
 	EnabledBuildOptions []string
 }
@@ -252,6 +253,14 @@ func WithConfig(configFile string) Option {
 	}
 }
 
+// WithFailOnLintWarning sets whether or not to fail on linter warnings.
+func WithFailOnLintWarning(fail bool) Option {
+	return func(b *Build) error {
+		b.FailOnLintWarning = fail
+		return nil
+	}
+}
+
 // WithBuildDate sets the timestamps for the build context.
 // The string is parsed according to RFC3339.
 // An empty string is a special case and will default to
@@ -1131,9 +1140,18 @@ func (b *Build) BuildPackage(ctx context.Context) error {
 		lctx := linter.NewLinterContext(lt.pkgName, fsys)
 		linters := lt.checks.GetLinters()
 
-		err = lctx.LintPackageFs(fsys, linters)
+		var innerErr error
+		err = lctx.LintPackageFs(fsys, func(err error) {
+			if b.FailOnLintWarning {
+				innerErr = err
+			} else {
+				b.Logger.Warnf("WARNING: %v", err)
+			}
+		}, linters)
 		if err != nil {
 			return fmt.Errorf("package linter error: %w", err)
+		} else if innerErr != nil {
+			return fmt.Errorf("package linter warning: %w", err)
 		}
 	}
 

diff --git a/pkg/build/package.go b/pkg/build/package.go
@@ -54,6 +54,8 @@ import (
 // concurrent builds on giant machines, and uses only 1 core on tiny machines.
 var pgzipThreads = min(runtime.GOMAXPROCS(0), 8)
 
+var libDirs = []string{"lib", "usr/lib", "lib64", "usr/lib64"}
+
 func min(l, r int) int {
 	if l < r {
 		return l
@@ -203,6 +205,9 @@ provides = {{ $dep }}
 {{- range $dep := .Dependencies.Replaces }}
 replaces = {{ $dep }}
 {{- end }}
+{{- range $dep := .Dependencies.Vendored }}
+# vendored = {{ $dep }}
+{{- end }}
 {{- if .Dependencies.ProviderPriority }}
 provider_priority = {{ .Dependencies.ProviderPriority }}
 {{- end }}
@@ -397,7 +402,6 @@ func findInterpreter(bin *elf.File) (string, error) {
 // dereferenceCrossPackageSymlink attempts to dereference a symlink across multiple package
 // directories.
 func (pc *PackageBuild) dereferenceCrossPackageSymlink(path string) (string, error) {
-	libDirs := []string{"lib", "usr/lib", "lib64", "usr/lib64"}
 	targetPackageNames := []string{pc.PackageName, pc.Build.Configuration.Package.Name}
 	realPath, err := os.Readlink(filepath.Join(pc.WorkspaceSubdir(), path))
 	if err != nil {
@@ -529,11 +533,6 @@ func generateSharedObjectNameDeps(pc *PackageBuild, generated *config.Dependenci
 			// Ugh: libc.so.6 has an PT_INTERP set on itself to make the `/lib/libc.so.6 --about`
 			// functionality work.  So we always generate provides entries for libc.
 			if !pc.Options.NoProvides && (interp == "" || strings.HasPrefix(basename, "libc")) {
-				libDirs := []string{"lib", "usr/lib", "lib64", "usr/lib64"}
-				if !allowedPrefix(path, libDirs) {
-					return nil
-				}
-
 				sonames, err := ef.DynString(elf.DT_SONAME)
 				// most likely SONAME is not set on this object
 				if err != nil {
@@ -551,7 +550,11 @@ func generateSharedObjectNameDeps(pc *PackageBuild, generated *config.Dependenci
 						libver = "0"
 					}
 
-					generated.Provides = append(generated.Provides, fmt.Sprintf("so:%s=%s", soname, libver))
+					if allowedPrefix(path, libDirs) {
+						generated.Provides = append(generated.Provides, fmt.Sprintf("so:%s=%s", soname, libver))
+					} else {
+						generated.Vendored = append(generated.Vendored, fmt.Sprintf("so:%s=%s", soname, libver))
+					}
 				}
 			}
 		}
@@ -686,14 +689,23 @@ func (pc *PackageBuild) GenerateDependencies() error {
 		}
 	}
 
-	newruntime := append(pc.Dependencies.Runtime, generated.Runtime...)
+	// Only consider vendored deps for self-provided generated runtime deps.
+	// If a runtime dep is explicitly configured, assume we actually do need it.
+	// This gives us an escape hatch in melange config in case there is a runtime
+	// dep that we don't want to be satisfied by a vendored dep.
+	unvendored := removeSelfProvidedDeps(generated.Runtime, generated.Vendored)
+
+	newruntime := append(pc.Dependencies.Runtime, unvendored...)
 	pc.Dependencies.Runtime = dedup(newruntime)
 
 	newprovides := append(pc.Dependencies.Provides, generated.Provides...)
 	pc.Dependencies.Provides = dedup(newprovides)
 
 	pc.Dependencies.Runtime = removeSelfProvidedDeps(pc.Dependencies.Runtime, pc.Dependencies.Provides)
 
+	// Sets .PKGINFO `# vendored = ...` comments; does not affect resolution.
+	pc.Dependencies.Vendored = generated.Vendored
+
 	pc.Dependencies.Summarize(pc.Logger)
 
 	return nil