diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml new file mode 100644 index 0000000000..1872eb311a --- /dev/null +++ b/.github/workflows/dependency-check.yml @@ -0,0 +1,92 @@ +name: Dependency Check + +on: + pull_request: + paths: + - 'go.mod' + - 'go.sum' + - '.github/workflows/dependency-check.yml' + +jobs: + dependency-check: + runs-on: ubuntu-latest + name: Dependency Check + env: + V0_PATTERN: 'v0\.0\.0-[0-9]{14}-[0-9a-f]{7,}(\s*(\/\/.*)?)?$' + RELEASE_PATTERN: 'v[0-9]+\.[0-9]+\.[0-9]+(\+incompatible)?(\s*(\/\/.*)?)?$' + IGNORE_PATTERN: 'dependency-check-ignore:\s' + + steps: + - uses: actions/checkout@v3 + name: Check out the repository + with: + submodules: 'recursive' + - uses: ./.github/actions/install-go + + - id: all + name: Extract all dependencies from go.mod (include indirect dependencies and comments) + run: | + echo "dependencies<> $GITHUB_OUTPUT + # `go list` isn't used because: + # 1. it lists ALL the transitive dependencies, even those that are unused and don't make it to the go.mod file + # 2. It doesn't extract the inline `dependency-check-ignore` comments. + # Extract the lines from 'require (' to the first ')' including those lines in the go.mod file. + sed -n '/require (/,/)/p' go.mod | + + # Remove the 'require (' line. + sed '/require (/d' | + + # Remove the ')' line. + sed '/^)/d' | + + # Remove leading whitespace from each line. + sed 's/^[[:space:]]*//' | + + # Append the result to the file specified by the GITHUB_OUTPUT environment variable. + tee -a $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - id: unreleased + name: Find all dependencies that use prerelease versions (i.e., exclude vX.Y.Z and v0.0.0 versions) + env: + DEPENDENCIES: ${{ steps.all.outputs.dependencies }} + run: | + echo "dependencies<> $GITHUB_OUTPUT + grep -Pv "$V0_PATTERN|$RELEASE_PATTERN" <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - id: unexplained + name: Find all unreleased dependencies without a dependency-check-ignore comment + env: + DEPENDENCIES: ${{ steps.unreleased.outputs.dependencies }} + run: | + echo "dependencies<> $GITHUB_OUTPUT + grep -Pv "$IGNORE_PATTERN" <<< "$DEPENDENCIES" | tee -a $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - id: v0check + name: Check v0.0.0 dependencies for available tags + run: | + echo "tagged<> $GITHUB_OUTPUT + grep -P "$V0_PATTERN" go.mod | grep -Pv "$IGNORE_PATTERN" | while read -r line; do + dep=$(echo "$line" | cut -d' ' -f1) + if [ ! -z "$(go list -m -versions $dep 2>/dev/null | awk 'NF>1')" ]; then + echo "$dep" + fi + done | tee -a $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - if: steps.unexplained.outputs.dependencies != '' || steps.v0check.outputs.tagged != '' + name: Throw if any unexplained dependencies exist + env: + MESSAGE: | + Dependencies requiring attention found in this PR. Please follow the [dependency management conventions](https://github.com/filecoin-project/lotus/blob/master/CONTRIBUTING.md#dependency-management). + + ${{ steps.unexplained.outputs.dependencies != '' && 'Unexplained unreleased dependencies:' || '' }} + ${{ steps.unexplained.outputs.dependencies }} + + ${{ steps.v0check.outputs.tagged != '' && 'Unexplained v0.0.0 dependencies with available tags:' || '' }} + ${{ steps.v0check.outputs.tagged }} + run: | + echo "::error::${MESSAGE//$'\n'/%0A}" + exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66c0003c33..1149a8e17b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,7 +49,7 @@ Note that this is enforced with https://github.com/filecoin-project/lotus/blob/m ## CHANGELOG Management -To expedite the release process, the CHANGELOG is built-up incrementally. +To expedite the release process, the CHANGELOG is built-up incrementally. We enforce that each PR updates CHANGELOG.md or signals that the change doesn't need it. If the PR affects users (e.g., new feature, bug fix, system requirements change), update the CHANGELOG.md and add details to the UNRELEASED section. If the change does not require a CHANGELOG.md entry, do one of the following: @@ -58,6 +58,20 @@ If the change does not require a CHANGELOG.md entry, do one of the following: Note that this is enforced with https://github.com/filecoin-project/lotus/blob/master/.github/workflows/changelog.yml +## Dependency Management + +We strive to use release dependencies because: +1. Security / reliability - While there's no guarantee that a released version doesn't have bugs or issues, it seems fair to assume that non-released versions have even more. For example, https://github.com/filecoin-project/lotus/issues/12467 was triggered because of a bug in non-released library that lotus was depending on when the latest released version didn't have the bug. +2. Faster builds +3. Makes Lotus a better citizen when it's imported by other projects. + +We enforce that each dependency on an unreleased version of a package is explicitly documented in the `go.mod` file via an inline comment of the form `dependency-check-ignore: `. +* If you are adding such a dependency, please add a suitable comment to the `go.mod` file as well. +* This requirement applies both to direct and indirect dependencies. +* This requirement applies to packages that have released versions (i.e., is not a `v0.0.0`). +* This is enforced with https://github.com/filecoin-project/lotus/blob/master/.github/workflows/dependency-check.yml +* This enforcement was initially done per [#7131](https://github.com/filecoin-project/lotus/issues/7131). + ## Markdown Conventions We optimize our markdown files for viewing on GitHub. That isn't to say other syntaxes can't be used, but that is the flavor we focus on and at the minimum don't want to break. diff --git a/go.mod b/go.mod index 5c59754bb0..f872e98412 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/filecoin-project/go-jsonrpc v0.7.0 github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-paramfetch v0.0.4 - github.com/filecoin-project/go-state-types v0.16.0-rc2 + github.com/filecoin-project/go-state-types v0.16.0-rc2 // dependency-check-ignore: unknown github.com/filecoin-project/go-statemachine v1.0.3 github.com/filecoin-project/go-statestore v0.2.0 github.com/filecoin-project/go-storedcounter v0.1.0 @@ -78,7 +78,7 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/gregdhill/go-openrpc v0.0.0-20220114144539-ae6f44720487 github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 - github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e + github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // dependency-check-ignore: TODO: needs upgrading to tagged version github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru/arc/v2 v2.0.7 github.com/hashicorp/golang-lru/v2 v2.0.7 @@ -134,15 +134,15 @@ require ( github.com/samber/lo v1.39.0 github.com/sirupsen/logrus v1.9.2 github.com/stretchr/testify v1.10.0 - github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // dependency-check-ignore: unknown github.com/triplewz/poseidon v0.0.2 github.com/urfave/cli/v2 v2.25.5 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba github.com/whyrusleeping/cbor-gen v0.2.0 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/xeipuuv/gojsonschema v1.2.0 - github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 - github.com/yugabyte/pgx/v5 v5.5.3-yb-2 + github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 // dependency-check-ignore: unknown + github.com/yugabyte/pgx/v5 v5.5.3-yb-2 // dependency-check-ignore: unknown github.com/zondax/ledger-filecoin-go v0.11.1 github.com/zyedidia/generic v1.2.1 go.opencensus.io v0.24.0 @@ -315,7 +315,7 @@ require ( github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.48.2 // indirect - github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect + github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect; dependency-check-ignore: unknown github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v2.18.12+incompatible // indirect @@ -337,7 +337,7 @@ require ( github.com/zondax/ledger-go v0.14.3 // indirect gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect - go.dedis.ch/kyber/v4 v4.0.0-pre2.0.20240924132404-4de33740016e // indirect + go.dedis.ch/kyber/v4 v4.0.0-pre2.0.20240924132404-4de33740016e // indirect; dependency-check-ignore: unknown go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.18.0 // indirect @@ -352,7 +352,7 @@ require ( gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect + howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect; dependency-check-ignore: required by github.com/elastic/go-sysinfo lukechampine.com/blake3 v1.3.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect )