Merge branch 'master' into bench-9.1.1

.github/workflows/haskell.yml

@@ -4,6 +4,16 @@ on:
   pull_request: # Required for workflows to be able to be approved from forks
   merge_group:
 
+  push:
+    # we need this to populate cache for `main` branch to make it available to the child branches, see
+    # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
+    branches:
+      - master
+  # GH caches are removed when not accessed within 7 days - this schedule runs the job every 6 days making
+  # sure that we always have some caches on main
+  schedule:
+    - cron: '0 0 */6 * *'
+
   # DO NOT DELETE.
   # This is required for nightly builds and is invoked by nightly-trigger.yml
   # on a schedule trigger.
@@ -20,25 +30,37 @@ on:
 
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.sys.os }}
 
     strategy:
       fail-fast: false
       matrix:
         # If you edit these versions, make sure the version in the lonely macos-latest job below is updated accordingly
-        ghc: ["9.6", "9.8"]
-        cabal: ["3.10.2.1"]
-        os: [windows-latest, ubuntu-latest]
+        # TODO add 9.8 again to the versions list when GHC-9.8 gets released with stm > 2.5.2,
+        # see https://github.com/haskell/stm/issues/76
+        ghc: ["9.6", "9.10"]
+        cabal: ["3.12"]
+        sys:
+          - { os: windows-latest, shell: 'C:/msys64/usr/bin/bash.exe -e {0}' }
+          - { os: ubuntu-latest, shell: bash }
         include:
           # Using include, to make sure there will only be one macOS job, even if the matrix gets expanded later on.
           # We want a single job, because macOS runners are scarce.
-          - os: macos-latest
-            cabal: "3.10.2.1"
+          - cabal: "3.12"
             ghc: "9.6"
+            sys:
+              os: macos-13
+              shell: bash
+    defaults:
+      run:
+        shell: ${{ matrix.sys.shell }}
 
     env:
       # Modify this value to "invalidate" the cabal cache.
-      CABAL_CACHE_VERSION: "2024-04-24"
+      CABAL_CACHE_VERSION: "2024-07-26"
+      # these two are msys2 env vars, they have no effect on non-msys2 installs.
+      MSYS2_PATH_TYPE: inherit
+      MSYSTEM: MINGW64
 
     concurrency:
       group: >
@@ -47,7 +69,7 @@ jobs:
         c+${{ github.job }}
         d+${{ matrix.ghc }}
         e+${{ matrix.cabal }}
-        f+${{ matrix.os }}
+        f+${{ matrix.sys.os }}
         g+${{ (startsWith(github.ref, 'refs/heads/gh-readonly-queue/') && github.run_id) || github.event.pull_request.number || github.ref }}
       cancel-in-progress: true
 
@@ -60,7 +82,7 @@ jobs:
         c+${{ github.job }}
         d+${{ matrix.ghc }}
         e+${{ matrix.cabal }}
-        f+${{ matrix.os }}
+        f+${{ matrix.sys.os }}
         g+${{ (startsWith(github.ref, 'refs/heads/gh-readonly-queue/') && github.run_id) || github.event.pull_request.number || github.ref }}
 
     - name: Install Haskell
@@ -81,7 +103,6 @@ jobs:
       run: cabal update
 
     - name: Configure build
-      shell: bash
       run: |
         cp .github/workflows/cabal.project.local.ci cabal.project.local
         echo "# cabal.project.local"
@@ -96,11 +117,12 @@ jobs:
     - name: Record dependencies
       id: record-deps
       run: |
-        # The tests call out to msys2 commands. We generally do not want to mix toolchains, so
-        # we are very deliberate about only adding msys64 to the path where absolutely necessary.
-        ${{ (runner.os == 'Windows' && '$env:PATH=("C:\msys64\mingw64\bin;{0}" -f $env:PATH)') || '' }}
         cat dist-newstyle/cache/plan.json | jq -r '."install-plan"[] | select(.style != "local") | .id' | sort | uniq > dependencies.txt
 
+    # Use a fresh cache each month
+    - name: Store month number as environment variable used in cache version
+      run:  echo "MONTHNUM=$(date -u '+%m')" >> $GITHUB_ENV
+
     # From the dependency list we restore the cached dependencies.
     # We use the hash of `dependencies.txt` as part of the cache key because that will be stable
     # until the `index-state` values in the `cabal.project` file changes.
@@ -111,7 +133,11 @@ jobs:
         path: |
           ${{ steps.setup-haskell.outputs.cabal-store }}
           dist-newstyle
-        key: cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }}
+        key:
+          cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.MONTHNUM }}-${{ hashFiles('dependencies.txt') }}
+        # try to restore previous cache from this month if there's no cache for the dependencies set
+        restore-keys: |
+          cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.MONTHNUM }}-
 
     # Now we install the dependencies. If the cache was found and restored in the previous step,
     # this should be a no-op, but if the cache key was not found we need to build stuff so we can
@@ -127,7 +153,8 @@ jobs:
         path: |
           ${{ steps.setup-haskell.outputs.cabal-store }}
           dist-newstyle
-        key: cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }}
+        key:
+          ${{ steps.cache.outputs.cache-primary-key }}
 
     # Now we build.
     - name: Build all
@@ -145,7 +172,6 @@ jobs:
       if: ${{ failure() }}
       env:
         TMP: ${{ runner.temp }}
-      shell: bash
       run: |
         cd $TMP
         find . -name 'module' -type f -exec dirname {} \; | xargs -L1 basename | sort -u | xargs tar -czvf workspaces.tgz
@@ -154,11 +180,10 @@ jobs:
       if: ${{ failure() }}
       uses: actions/upload-artifact@v4
       with:
-        name: failed-test-workspaces-${{ matrix.os }}-ghc${{ matrix.ghc }}-cabal${{ matrix.cabal }}.tgz
+        name: failed-test-workspaces-${{ matrix.sys.os }}-ghc${{ matrix.ghc }}-cabal${{ matrix.cabal }}.tgz
         path: ${{ runner.temp }}/workspaces.tgz
 
     - name: "Tar artifacts"
-      shell: bash
       run: |
         mkdir -p artifacts
 
@@ -176,7 +201,6 @@ jobs:
 
     - name: Delete socket files in chairman tests in preparation for uploading artifacts
       if: ${{ always() }}
-      shell: bash
       run: |
         if [ -d "${{ runner.temp }}/chairman" ]; then
           find "${{ runner.temp }}/chairman" -type s -exec rm -f {} \;
@@ -187,7 +211,7 @@ jobs:
       if: ${{ always() }}
       continue-on-error: true
       with:
-        name: chairman-test-artifacts-${{ matrix.os }}-${{ matrix.ghc }}
+        name: chairman-test-artifacts-${{ matrix.sys.os }}-${{ matrix.ghc }}
         path: ${{ runner.temp }}/chairman/
 
     # Uncomment the following back in for debugging. Remember to launch a `pwsh` from

Makefile

@@ -7,8 +7,8 @@ include nix.mk
 PROJECT_NAME = cardano-node
 NUM_PROC     = $(nproc --all)
 
-## One of:  shey alra mary alzo bage
-ERA     ?= bage
+## One of:  shey alra mary alzo bage coay
+ERA     ?= coay
 
 PROFILE ?= default-${ERA}
 BACKEND ?= supervisor
@@ -74,64 +74,10 @@ ps:                                              ## Plain-text list of profiles
 ##
 ## Profile-based cluster shells (autogenerated targets)
 ##
-PROFILES_BASE             := default default-p2p plutus plutus-secp-ecdsa plutus-secp-schnorr oldtracing idle tracer-only
-PROFILES_FAST             := fast fast-solo fast-p2p fast-plutus fast-notracer fast-oldtracing faststartup-24M
-PROFILES_CI_TEST          := ci-test ci-test-p2p ci-test-plutus ci-test-notracer ci-test-rtview ci-test-dense10 ci-test-hydra
-PROFILES_CI_BENCH         := ci-bench ci-bench-p2p ci-bench-plutus ci-bench-plutus-secp-ecdsa ci-bench-plutus-secp-schnorr ci-bench-notracer ci-bench-rtview ci-bench-lmdb ci-bench-drep
-PROFILES_CI_BENCH         += ci-bench-plutusv3-blst ci-bench-plutus24
-PROFILES_TRACE_BENCH      := trace-bench trace-bench-notracer trace-bench-oldtracing trace-bench-rtview
-PROFILES_TRACE_FULL       := trace-full trace-full-rtview
-PROFILES_EPOCHTRANS       := epoch-transition
-PROFILES_PLUTUSCALL       := plutuscall-loop-plain plutuscall-secp-ecdsa-plain plutuscall-secp-schnorr-plain
-PROFILES_PLUTUSCALL       += plutuscall-loop-half plutuscall-secp-ecdsa-half plutuscall-secp-schnorr-half
-PROFILES_PLUTUSCALL       += plutuscall-loop-double plutuscall-secp-ecdsa-double plutuscall-secp-schnorr-double
-PROFILES_MODEL            := model-value model-secp-ecdsa-plain model-secp-ecdsa-half model-secp-ecdsa-double
-PROFILES_MODEL            += model-value-test
-PROFILES_10               := 10 10-p2p 10-plutus 10-notracer
-PROFILES_FORGE_STRESS     := forge-stress forge-stress-p2p forge-stress-plutus forge-stress-plutus-solo forge-stress-notracer forge-stress-large forge-stress-solo forge-stress-solo-xs
-PROFILES_FORGE_STRESS_PRE := forge-stress-pre forge-stress-pre-plutus forge-stress-pre-notracer forge-stress-pre-solo forge-stress-pre-solo-xl forge-stress-pre-solo-xs
-PROFILES_FORGE_STRESS_RTS := forge-stress-pre-rtsA4m forge-stress-pre-rtsA64m forge-stress-pre-rtsN3 forge-stress-pre-rtsA4mN3 forge-stress-pre-rtsA64mN3 forge-stress-pre-rtsxn
-PROFILES_CHAINSYNC        := chainsync-early-byron  chainsync-early-byron-notracer  chainsync-early-byron-oldtracing
-PROFILES_CHAINSYNC        += chainsync-early-alonzo chainsync-early-alonzo-notracer chainsync-early-alonzo-oldtracing chainsync-early-alonzo-p2p
-PROFILES_VENDOR           := dish dish-plutus dish-10M dish-10M-plutus
-# The dedicated P&T Nomad cluster on AWS
-# Cloud version of "default", "ci-test" and "ci-bench" plus value (52+explorer)
-# Not all local profiles are compatible or tested (yet) with a cloud runs
-PROFILES_NOMAD_PERF       := default-nomadperf ci-test-nomadperf ci-bench-nomadperf value-nomadperf oldtracing-nomadperf ci-test-oldtracing-nomadperf ci-bench-oldtracing-nomadperf value-oldtracing-nomadperf
-PROFILES_NOMAD_PERF       += plutus-nomadperf plutus24-nomadperf fast-nomadperf latency-nomadperf
-PROFILES_NOMAD_PERF       += plutusv3-blst-nomadperf plutusv3-blst-half-nomadperf plutusv3-blst-double-nomadperf
-PROFILES_NOMAD_PERF       += plutus-secp-ecdsa-nomadperf plutus-secp-schnorr-nomadperf
-PROFILES_NOMAD_PERF_DREP  := value-drep1k-nomadperf value-drep2k-nomadperf value-drep10k-nomadperf value-drep100k-nomadperf
-PROFILES_NOMAD_PERF_DREP  += plutus-drep1k-nomadperf plutus-drep2k-nomadperf plutus-drep10k-nomadperf plutus-drep100k-nomadperf
-PROFILES_NOMAD_PERF_NOP2P := default-nomadperf-nop2p oldtracing-nomadperf-nop2p ci-test-nomadperf-nop2p ci-bench-nomadperf-nop2p
-PROFILES_NOMAD_PERF_NOP2P += value-nomadperf-nop2p value-oldtracing-nomadperf-nop2p plutus-nomadperf-nop2p fast-nomadperf-nop2p
-PROFILES_NOMAD_PERFSSD    := value-nomadperfssd fast-nomadperfssd latency-nomadperfssd
-# single node profiles on the NomadSSD cluster on AWS
-PROFILES_UTXOSCALE_SOLO	  := utxoscale-solo-24M64G-nomadperfssd utxoscale-solo-12M64G-nomadperfssd utxoscale-solo-12M16G-nomadperfssd
-
-LOCAL_PROFILES += $(PROFILES_BASE)
-LOCAL_PROFILES += $(PROFILES_FAST)
-LOCAL_PROFILES += $(PROFILES_CI_TEST)
-LOCAL_PROFILES += $(PROFILES_CI_BENCH)
-LOCAL_PROFILES += $(PROFILES_TRACE_BENCH)
-LOCAL_PROFILES += $(PROFILES_TRACE_FULL)
-LOCAL_PROFILES += $(PROFILES_EPOCHTRANS)
-LOCAL_PROFILES += $(PROFILES_PLUTUSCALL)
-LOCAL_PROFILES += $(PROFILES_MODEL)
-LOCAL_PROFILES += $(PROFILES_10)
-LOCAL_PROFILES += $(PROFILES_FORGE_STRESS)
-LOCAL_PROFILES += $(PROFILES_FORGE_STRESS_PRE)
-LOCAL_PROFILES += $(PROFILES_FORGE_STRESS_RTS)
-LOCAL_PROFILES += $(PROFILES_CHAINSYNC)
-LOCAL_PROFILES += $(PROFILES_VENDOR)
-CLOUD_PROFILES += $(PROFILES_NOMAD_PERF)
-CLOUD_PROFILES += $(PROFILES_NOMAD_PERF_DREP)
-CLOUD_PROFILES += $(PROFILES_NOMAD_PERF_NOP2P)
-CLOUD_PROFILES += $(PROFILES_NOMAD_PERFSSD)
-CLOUD_PROFILES += $(PROFILES_UTXOSCALE_SOLO)
-
-
-## Note:  to enable a shell for a profile, just add its name (one of names from 'make ps') to SHELL_PROFILES
+## wb_profiles.mk is autogenerated by `cardano-profile lib-make`. It contains all known profile families and flavours (minus the era, backend and shell type suffixes).
+## After adding or removing profile definitions in `cardano-profile`, you'll probably want to also commit a regenerated version of that file.
+##
+include wb_profiles.mk
 
 $(eval $(call define_profile_targets,           $(LOCAL_PROFILES)))
 $(eval $(call define_profile_targets_nomadcloud,$(CLOUD_PROFILES)))

README.rst

@@ -1,19 +1,16 @@
 .. raw:: html
 
   <p align="center">
-    <a href="https://github.com/intersectmbo/cardano-node/releases"><img src="https://img.shields.io/github/release-pre/intersectmbo/cardano-node.svg?style=for-the-badge" /></a>
+    <a href="https://github.com/intersectmbo/cardano-node/releases">
+      <img src="https://img.shields.io/github/release-pre/intersectmbo/cardano-node.svg?style=for-the-badge" />
+    </a>
+    <a href="https://github.com/intersectmbo/cardano-node/actions/workflows/haskell.yml?query=branch%3Amaster">
+      <img alt="GitHub Workflow Status (master)" src="https://img.shields.io/github/actions/workflow/status/intersectmbo/cardano-node/haskell.yml?branch=master&label=master&style=for-the-badge" />
+    </a>
+    <a href="https://github.com/intersectmbo/cardano-node/actions/workflows/haskell.yml?query=branch%3Anightly">
+      <img alt="GitHub Workflow Status (nightly)" src="https://img.shields.io/github/actions/workflow/status/intersectmbo/cardano-node/haskell.yml?branch=nightly&label=nightly&style=for-the-badge" />
+    </a>
   </p>
-
-  <table align="center">
-    <tr><td colspan="2" align="center">GitHub Actions</td></tr>
-    <tr>
-      <td>
-        <a href="https://github.com/intersectmbo/cardano-node/actions/workflows/haskell.yml"><img alt="GitHub Workflow Status (master)" src="https://img.shields.io/github/actions/workflow/status/intersectmbo/cardano-node/haskell.yml?branch=master" /></a>
-        <a href="https://github.com/intersectmbo/cardano-node/actions/workflows/haskell.yml"><img alt="GitHub Workflow Status (branch)" src="https://img.shields.io/github/actions/workflow/status/intersectmbo/cardano-node/haskell.yml?branch=nightly" /></a>
-      </td>
-    </tr>
-  </table>
-
 .. contents:: Contents
 
 *******************************************
@@ -47,7 +44,7 @@ Obtaining ``cardano-node``
 Building from source
 ====================
 
-Documentation for building the node can be found `here <https://developers.cardano.org/docs/get-started/installing-cardano-node>`_.
+Documentation for building the node can be found `here <https://developers.cardano.org/docs/get-started/cardano-node/installing-cardano-node>`_.
 
 Executables
 ===========
@@ -154,7 +151,7 @@ Using ``cardano-cli``
 *********************
 
 A CLI utility to support a variety of key material operations (genesis, migration, pretty-printing..) for different system generations.
-Usage documentation can be found at ``https://github.com/input-output-hk/cardano-cli/tree/master/cardano-cli/README.md``.
+Usage documentation can be found at `<https://developers.cardano.org/docs/get-started/cardano-cli/get-started/>`_.
 
 The general synopsis is as follows:
 

bench/cardano-profile/app/cardano-profile.hs

@@ -22,14 +22,7 @@ data Cli = Names | All | ByName String | LibMK | ToJson String | FromJson String
 
 main :: IO ()
 main = do
-  cli <- OA.execParser
-    (OA.info
-      cliParser
-      (   OA.fullDesc
-       <> OA.progDesc "Cardano benchmarking profile generator"
-       <> OA.header "names | all | make PROFILE_NAME | from-json FILE.json | to-json FILE.hs"
-      )
-    )
+  cli <- getOpts
   case cli of
     -- Print all profile names.
     Names -> BSL8.putStrLn $ Aeson.encode Profiles.names
@@ -68,6 +61,12 @@ lookupOverlay = do
                     (Just (Aeson.Object keyMap)) -> return keyMap
                     _ -> error ""
 
+getOpts :: IO Cli
+getOpts = OA.execParser $
+  OA.info
+    (cliParser OA.<**> OA.helper)
+    (OA.fullDesc <> OA.progDesc "Cardano benchmarking profile generator (-h for help)")
+
 --------------------------------------------------------------------------------
 
 cliParser :: OA.Parser Cli

bench/cardano-profile/cardano-profile.cabal

@@ -1,7 +1,7 @@
 cabal-version: 3.0
 
 name:                   cardano-profile
-version:                8.4.0
+version:                8.6.0
 synopsis:               A Cardano benchmarking profile generator
 description:            A Cardano benchmarking profile generator.
 category:               Cardano,
@@ -17,10 +17,7 @@ extra-source-files:     README.md
 data-files:             data/all-profiles.json
                         data/ci-test-bage.json
                         data/genesis/epoch-timeline.json
-                        data/genesis/overlays/v8-preview.json
-                        data/genesis/overlays/v9-preview.json
-                        data/genesis/overlays/stepshalf.json
-                        data/genesis/overlays/doublebudget.json
+                        data/genesis/overlays/*.json
 
 common project-config
   build-depends:        base >= 4.14 && < 5
@@ -73,9 +70,6 @@ executable cardano-profile
   import:               project-config
   hs-source-dirs:       app/
   main-is:              cardano-profile.hs
-  ghc-options:          -threaded
-                        -rtsopts
-                        "-with-rtsopts=-T"
   build-depends:        aeson
                       , vector
                       , bytestring
@@ -96,6 +90,3 @@ test-suite cardano-profile-test
                       , tasty
                       , tasty-hunit
                       , cardano-profile
-  ghc-options:          -threaded
-                        -rtsopts
-                        "-with-rtsopts=-T"