fix(splitstore): merry christmas lotus! Remove ~120 G from lotus datastore #3548
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - master | |
| - release/* | |
| workflow_dispatch: | |
| defaults: | |
| run: | |
| shell: bash | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| contents: read | |
| jobs: | |
| discover: | |
| name: Discover Test Groups | |
| runs-on: ubuntu-latest | |
| outputs: | |
| groups: ${{ steps.test.outputs.groups }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| - id: test | |
| env: | |
| # Unit test groups other than unit-rest | |
| utests: | | |
| [ | |
| {"name": "unit-cli", "packages": ["./cli/...", "./cmd/...", "./api/..."]}, | |
| {"name": "unit-storage", "packages": ["./storage/...", "./extern/..."]}, | |
| {"name": "unit-node", "packages": ["./node/..."]} | |
| ] | |
| # Other tests that require special configuration | |
| otests: | | |
| [ | |
| { | |
| "name": "multicore-sdr", | |
| "packages": ["./storage/sealer/ffiwrapper"], | |
| "go_test_flags": "-run=TestMulticoreSDR", | |
| "test_rustproofs_logs": "1" | |
| }, { | |
| "name": "conformance", | |
| "packages": ["./conformance"], | |
| "go_test_flags": "-run=TestConformance", | |
| "skip_conformance": "0" | |
| } | |
| ] | |
| # Mapping from test group names to custom runner labels | |
| # The jobs default to running on the default hosted runners (4 CPU, 16 RAM). | |
| # We use self-hosted xlarge (4 CPU, 8 RAM; and large - 2 CPU, 4 RAM) runners | |
| # to extend the available runner capacity (60 default hosted runners). | |
| # We use self-hosted 4xlarge (16 CPU, 32 RAM; and 2xlarge - 8 CPU, 16 RAM) self-hosted | |
| # to support resource intensive jobs. | |
| runners: | | |
| { | |
| "itest-niporep_manual": ["self-hosted", "linux", "x64", "4xlarge"], | |
| "itest-sector_pledge": ["self-hosted", "linux", "x64", "4xlarge"], | |
| "itest-worker": ["self-hosted", "linux", "x64", "4xlarge"], | |
| "itest-manual_onboarding": ["self-hosted", "linux", "x64", "4xlarge"], | |
| "itest-gateway": ["self-hosted", "linux", "x64", "2xlarge"], | |
| "itest-sector_import_full": ["self-hosted", "linux", "x64", "2xlarge"], | |
| "itest-sector_import_simple": ["self-hosted", "linux", "x64", "2xlarge"], | |
| "itest-wdpost": ["self-hosted", "linux", "x64", "2xlarge"], | |
| "unit-storage": ["self-hosted", "linux", "x64", "2xlarge"], | |
| "itest-cli": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-deals_invalid_utf8_label": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-decode_params": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-dup_mpool_messages": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-eth_account_abstraction": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-eth_api": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-eth_balance": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-eth_bytecode": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-eth_config": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-eth_conformance": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-eth_deploy": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-eth_fee_history": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-eth_transactions": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-fevm_address": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-fevm_events": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-gas_estimation": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-get_messages_in_ts": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-lite_migration": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-lookup_robust_address": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-mempool": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-mpool_msg_uuid": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-mpool_push_with_uuid": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-msgindex": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-multisig": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-net": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-nonce": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-path_detach_redeclare": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-pending_deal_allocation": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-remove_verifreg_datacap": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-sector_miner_collateral": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-sector_numassign": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-self_sent_txn": ["self-hosted", "linux", "x64", "xlarge"], | |
| "itest-verifreg": ["self-hosted", "linux", "x64", "xlarge"], | |
| "multicore-sdr": ["self-hosted", "linux", "x64", "xlarge"], | |
| "unit-node": ["self-hosted", "linux", "x64", "xlarge"] | |
| } | |
| # A list of test groups that require Proof Parameters to be fetched | |
| parameters: | | |
| [ | |
| "conformance", | |
| "itest-api", | |
| "itest-direct_data_onboard_verified", | |
| "itest-direct_data_onboard", | |
| "itest-manual_onboarding", | |
| "itest-niporep_manual", | |
| "itest-net", | |
| "itest-path_detach_redeclare", | |
| "itest-sealing_resources", | |
| "itest-sector_import_full", | |
| "itest-sector_import_simple", | |
| "itest-sector_pledge", | |
| "itest-sector_unseal", | |
| "itest-wdpost_no_miner_storage", | |
| "itest-wdpost_worker_config", | |
| "itest-wdpost", | |
| "itest-worker", | |
| "multicore-sdr", | |
| "unit-cli", | |
| "unit-storage" | |
| ] | |
| run: | | |
| # Create a list of integration test groups | |
| itests="$( | |
| find ./itests -name "*_test.go" | \ | |
| jq -R '{ | |
| "name": "itest-\(. | split("/") | .[2] | sub("_test.go$";""))", | |
| "packages": [.] | |
| }' | jq -s | |
| )" | |
| # Create a list of packages that are covered by the integration and unit tests | |
| packages="$(jq -n --argjson utests "$utests" '$utests | map(.packages) | flatten | . + ["./itests/..."]')" | |
| # Create a new group for the unit tests that are not yet covered | |
| rest="$( | |
| find . -name "*_test.go" | cut -d/ -f2 | sort | uniq | \ | |
| jq -R '"./\(.)/..."' | \ | |
| jq -s --argjson p "$packages" '{"name": "unit-rest", "packages": (. - $p)}' | |
| )" | |
| # Combine the groups for integration tests, unit tests, the new unit-rest group, and the other tests | |
| groups="$(jq -n --argjson i "$itests" --argjson u "$utests" --argjson r "$rest" --argjson o "$otests" '$i + $u + [$r] + $o')" | |
| # Apply custom runner labels to the groups | |
| groups="$(jq -n --argjson g "$groups" --argjson r "$runners" '$g | map(. + {"runner": (.name as $n | $r | .[$n]) })')" | |
| # Apply the needs_parameters flag to the groups | |
| groups="$(jq -n --argjson g "$groups" --argjson p "$parameters" '$g | map(. + {"needs_parameters": ([.name] | inside($p)) })')" | |
| # Output the groups | |
| echo "groups=$groups" | |
| echo "groups=$(jq -nc --argjson g "$groups" '$g')" >> $GITHUB_OUTPUT | |
| cache: | |
| name: Cache Dependencies | |
| runs-on: ubuntu-latest | |
| outputs: | |
| fetch_params_key: ${{ steps.fetch_params.outputs.key }} | |
| fetch_params_path: ${{ steps.fetch_params.outputs.path }} | |
| make_deps_key: ${{ steps.make_deps.outputs.key }} | |
| make_deps_path: ${{ steps.make_deps.outputs.path }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| - id: fetch_params | |
| env: | |
| CACHE_KEY: fetch-params-${{ hashFiles('./extern/filecoin-ffi/parameters.json') }} | |
| CACHE_PATH: | | |
| /var/tmp/filecoin-proof-parameters/ | |
| run: | | |
| echo -e "key=$CACHE_KEY" | tee -a $GITHUB_OUTPUT | |
| echo -e "path<<EOF\n$CACHE_PATH\nEOF" | tee -a $GITHUB_OUTPUT | |
| - id: make_deps | |
| env: | |
| CACHE_KEY: ${{ runner.os }}-${{ runner.arch }}-make-deps-${{ hashFiles('./.git/modules/extern/filecoin-ffi/HEAD') }}-p | |
| CACHE_PATH: | | |
| ./extern/filecoin-ffi/filcrypto.h | |
| ./extern/filecoin-ffi/libfilcrypto.a | |
| ./extern/filecoin-ffi/filcrypto.pc | |
| run: | | |
| echo -e "key=$CACHE_KEY" | tee -a $GITHUB_OUTPUT | |
| echo -e "path<<EOF\n$CACHE_PATH\nEOF" | tee -a $GITHUB_OUTPUT | |
| - id: restore_fetch_params | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ steps.fetch_params.outputs.key }} | |
| path: ${{ steps.fetch_params.outputs.path }} | |
| lookup-only: true | |
| - id: restore_make_deps | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ steps.make_deps.outputs.key }} | |
| path: ${{ steps.make_deps.outputs.path }} | |
| lookup-only: true | |
| - if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/install-system-dependencies | |
| - if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/install-go | |
| - if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true' | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: FFI_PORTABLE=1 make deps | |
| - if: steps.restore_fetch_params.outputs.cache-hit != 'true' | |
| run: make lotus | |
| - if: steps.restore_fetch_params.outputs.cache-hit != 'true' | |
| run: ./lotus fetch-params 2048 | |
| - if: steps.restore_fetch_params.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| key: ${{ steps.fetch_params.outputs.key }} | |
| path: ${{ steps.fetch_params.outputs.path }} | |
| - if: steps.restore_make_deps.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| key: ${{ steps.make_deps.outputs.key }} | |
| path: ${{ steps.make_deps.outputs.path }} | |
| test: | |
| needs: [discover, cache] | |
| name: Test (${{ matrix.name }}) | |
| runs-on: ${{ github.repository_owner == 'filecoin-project' && matrix.runner || 'ubuntu-latest' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.discover.outputs.groups) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| - uses: ./.github/actions/install-system-dependencies | |
| - uses: ./.github/actions/install-go | |
| - name: Install gotestsum | |
| run: go install gotest.tools/gotestsum@latest | |
| - name: Restore cached make deps outputs | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ needs.cache.outputs.make_deps_key }} | |
| path: ${{ needs.cache.outputs.make_deps_path }} | |
| fail-on-cache-miss: true | |
| - if: ${{ matrix.needs_parameters }} | |
| name: Restore cached fetch params outputs | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ needs.cache.outputs.fetch_params_key }} | |
| path: ${{ needs.cache.outputs.fetch_params_path }} | |
| fail-on-cache-miss: true | |
| # TODO: Install statediff (used to be used for conformance) | |
| - name: Create temporary directory for reports | |
| id: reports | |
| run: mktemp -d | xargs -0 -I{} echo "path={}" | tee -a $GITHUB_OUTPUT | |
| # TODO: Track coverage (used to be tracked for conformance) | |
| - name: Run tests | |
| env: | |
| NAME: ${{ matrix.name }} | |
| LOTUS_SRC_DIR: ${{ github.workspace }} | |
| REPORTS_PATH: ${{ steps.reports.outputs.path }} | |
| SKIP_CONFORMANCE: ${{ matrix.skip_conformance || '1' }} | |
| TEST_RUSTPROOFS_LOGS: ${{ matrix.test_rustproofs_logs || '0' }} | |
| FORMAT: ${{ matrix.format || 'standard-verbose' }} | |
| PACKAGES: ${{ join(matrix.packages, ' ') }} | |
| run: | | |
| gotestsum \ | |
| --format "$FORMAT" \ | |
| --junitfile "$REPORTS_PATH/$NAME.xml" \ | |
| --jsonfile "$REPORTS_PATH/$NAME.json" \ | |
| --packages="$PACKAGES" \ | |
| -- ${{ matrix.go_test_flags || '' }} | |
| - name: Modify junit.xml for BuildPulse | |
| env: | |
| NAME: ${{ matrix.name }} | |
| REPORTS_PATH: ${{ steps.reports.outputs.path }} | |
| PACKAGES: ${{ join(matrix.packages, ' ') }} | |
| if: always() | |
| run: | | |
| # Modify test suite name and classname attributes in JUnit XML for better grouping | |
| # in BuildPulse. itests are run with go test ./itests/file_test.go and therefore Go | |
| # assigns the name and classname attributes to "command-line-arguments". Others get the | |
| # package name for both. | |
| if [[ "${{ matrix.name }}" == itest-* ]]; then | |
| PACKAGE_NAME=$(basename "$PACKAGES" .go) | |
| sed -i 's/ name="command-line-arguments"/ name="itests"/g' "$REPORTS_PATH/$NAME.xml" | |
| sed -i 's/classname="command-line-arguments"/classname="'"$PACKAGE_NAME"'"/g' "$REPORTS_PATH/$NAME.xml" | |
| else | |
| sed -i 's# name="github.com/filecoin-project/lotus/\(.*\)"# name="'${{ matrix.name }}':\1"#g' "$REPORTS_PATH/$NAME.xml" | |
| fi | |
| cat "$REPORTS_PATH/$NAME.xml" | |
| - if: success() || failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.name }} | |
| path: | | |
| ${{ steps.reports.outputs.path }}/${{ matrix.name }}.xml | |
| ${{ steps.reports.outputs.path }}/${{ matrix.name }}.json | |
| continue-on-error: true |