Chore/update nim version (#1052)

* Move to version 2.0.6

* Update nim-confutils submodule to latest version

* Update dependencies

* Update Nim version to 2.0.12

* Add gcsafe pragma

* Add missing import

* Update specific conf for Nim 2.x

* Fix method signatures

* Revert erasure coding attempt to fix bug

* More gcsafe pragma

* Duplicate code from libp2p because it is not exported anymore

* Fix camelcase function names

* Use alreadySeen because need is not a bool anymore

* newLPStreamReadError does not exist anymore so use another error

* Replace ValidIpAddress by IpAddress

* Add gcsafe pragma

* Restore maintenance parameter deleted by mistake when removing esasure coding fix attempt code

* Update method signatures

* Copy LPStreamReadError code from libp2p which was removed

* Fix camel case

* Fix enums in tests

* Fix camel case

* Extract node components to a variable to make Nim 2 happy

* Update the tests using ValidIpAddress to IpAddress

* Fix cast for value which is already an option

* Set nim version to 2.0.x for CI

* Set nim version to 2.0.x for CI

* Move to miniupnp version 2.2.4 to avoid symlink error

* Set core.symlinks to false for Windows for miniupnp >= 2.2.5 support

* Update to Nim 2.0.14

* Update CI nim versions to 2.0.14

* Try with GCC 14

* Replace apt-fast by apt-get

* Update ubuntu runner to latest

* Use Ubuntu 20.04 for coverage

* Disable CI cache for coverage

* Add coverage property description

* Remove commented test

* Check the node value of seen instead of using alreadySeen

* Fix the merge. The taskpool work was reverted.

* Update nim-ethers submodule

* Remove deprecated ValidIpAddress. Fix missing case and imports.

* Fix a weird issue where nim-confutils cannot find NatAny

* Fix tests and remove useless static keyword

.github/actions/nimbus-build-system/action.yml

@@ -11,13 +11,16 @@ inputs:
     default: "amd64"
   nim_version:
     description: "Nim version"
-    default: "version-1-6"
+    default: "v2.0.14"
   rust_version:
     description: "Rust version"
     default: "1.79.0"
   shell:
     description: "Shell to run commands in"
     default: "bash --noprofile --norc -e -o pipefail"
+  coverage:
+    description: "True if the process is used for coverage"
+    default: false
 runs:
   using: "composite"
   steps:
@@ -31,17 +34,17 @@ runs:
       if: inputs.os == 'linux' && (inputs.cpu == 'amd64' || inputs.cpu == 'arm64')
       shell: ${{ inputs.shell }} {0}
       run: |
-        sudo apt-fast update -qq
-        sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
+        sudo apt-get update -qq
+        sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
           --no-install-recommends -yq lcov
 
     - name: APT (Linux i386)
       if: inputs.os == 'linux' && inputs.cpu == 'i386'
       shell: ${{ inputs.shell }} {0}
       run: |
         sudo dpkg --add-architecture i386
-        sudo apt-fast update -qq
-        sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
+        sudo apt-get update -qq
+        sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
           --no-install-recommends -yq gcc-multilib g++-multilib
 
     - name: Homebrew (macOS)
@@ -78,11 +81,21 @@ runs:
           mingw-w64-i686-ntldd-git
           mingw-w64-i686-rust
 
-    - name: MSYS2 (Windows All) - Downgrade to gcc 13
+    - name: MSYS2 (Windows All) - Update to gcc 14
       if: inputs.os == 'windows'
       shell: ${{ inputs.shell }} {0}
       run: |
-        pacman -U --noconfirm https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-13.2.0-6-any.pkg.tar.zst https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-libs-13.2.0-6-any.pkg.tar.zst
+        pacman -U --noconfirm https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-14.2.0-2-any.pkg.tar.zst https://repo.msys2.org/mingw/ucrt64/mingw-w64-ucrt-x86_64-gcc-libs-14.2.0-2-any.pkg.tar.zst
+
+    - name: Install gcc 14 on Linux
+      # We don't want to install gcc 14 for coverage (Ubuntu 20.04)
+      if : ${{ inputs.os == 'linux' && !inputs.coverage }}
+      shell: ${{ inputs.shell }} {0}
+      run: |
+        # Add GCC-14 to alternatives
+        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 14
+        # Set GCC-14 as the default
+        sudo update-alternatives --set gcc /usr/bin/gcc-14
 
     - name: Derive environment variables
       shell: ${{ inputs.shell }} {0}
@@ -159,6 +172,7 @@ runs:
     - name: Restore Nim toolchain binaries from cache
       id: nim-cache
       uses: actions/cache@v4
+      if : ${{ !inputs.coverage  }}
       with:
         path: NimBinaries
         key: ${{ inputs.os }}-${{ inputs.cpu }}-nim-${{ inputs.nim_version }}-cache-${{ env.cache_nonce }}-${{ github.run_id }}
@@ -168,9 +182,16 @@ runs:
       shell: ${{ inputs.shell }} {0}
       run: echo "NIM_COMMIT=${{ inputs.nim_version }}" >> ${GITHUB_ENV}
 
+    - name: MSYS2 (Windows All) - Disable git symbolic links (since miniupnp 2.2.5)
+      if: inputs.os == 'windows'
+      shell: ${{ inputs.shell }} {0}
+      run: |
+        git config --global core.symlinks false
+
     - name: Build Nim and Codex dependencies
       shell: ${{ inputs.shell }} {0}
       run: |
+        gcc --version
         make -j${ncpu} CI_CACHE=NimBinaries ${ARCH_OVERRIDE} QUICK_AND_DIRTY_COMPILER=1 update
         echo
         ./env.sh nim --version

.github/workflows/ci-reusable.yml

@@ -40,6 +40,7 @@ jobs:
           os: ${{ matrix.os }}
           shell: ${{ matrix.shell }}
           nim_version: ${{ matrix.nim_version }}
+          coverage: false
 
       ## Part 1 Tests ##
       - name: Unit tests

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ on:
 
 env:
   cache_nonce: 0 # Allows for easily busting actions/cache caches
-  nim_version: pinned
+  nim_version: v2.0.14
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
@@ -27,10 +27,10 @@ jobs:
         uses: fabiocaccamo/create-matrix-action@v4
         with:
           matrix: |
-            os {linux},   cpu {amd64}, builder {ubuntu-20.04},   tests {unittest},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {linux},   cpu {amd64}, builder {ubuntu-20.04},   tests {contract},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {linux},   cpu {amd64}, builder {ubuntu-20.04},   tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-            os {linux},   cpu {amd64}, builder {ubuntu-20.04},   tests {tools},       nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {amd64}, builder {ubuntu-latest},   tests {unittest},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {amd64}, builder {ubuntu-latest},   tests {contract},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {amd64}, builder {ubuntu-latest},   tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {amd64}, builder {ubuntu-latest},   tests {tools},       nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {macos},   cpu {amd64}, builder {macos-13},       tests {unittest},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {macos},   cpu {amd64}, builder {macos-13},       tests {contract},    nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
             os {macos},   cpu {amd64}, builder {macos-13},       tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
@@ -48,6 +48,10 @@ jobs:
       cache_nonce: ${{ needs.matrix.outputs.cache_nonce }}
 
   coverage:
+    # Force to stick to ubuntu 20.04 for coverage because
+    # lcov was updated to 2.x version in ubuntu-latest
+    # and cause a lot of issues.
+    # See https://github.com/linux-test-project/lcov/issues/238
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout sources
@@ -61,6 +65,7 @@ jobs:
         with:
           os: linux
           nim_version: ${{ env.nim_version }}
+          coverage: true
 
       - name: Generate coverage data
         run: |

Makefile

@@ -15,8 +15,8 @@
 #
 # If NIM_COMMIT is set to "nimbusbuild", this will use the
 # version pinned by nimbus-build-system.
-PINNED_NIM_VERSION := 38640664088251bbc88917b4bacfd86ec53014b8 # 1.6.21
-
+#PINNED_NIM_VERSION := 38640664088251bbc88917b4bacfd86ec53014b8 # 1.6.21
+PINNED_NIM_VERSION := v2.0.14
 ifeq ($(NIM_COMMIT),)
 NIM_COMMIT := $(PINNED_NIM_VERSION)
 else ifeq ($(NIM_COMMIT),pinned)

codex.nim

@@ -47,7 +47,7 @@ when isMainModule:
   let config = CodexConf.load(
     version = codexFullVersion,
     envVarsPrefix = "codex",
-    secondarySources = proc (config: CodexConf, sources: auto) =
+    secondarySources = proc (config: CodexConf, sources: auto) {.gcsafe, raises: [ConfigurationError].} =
             if configFile =? config.configFile:
               sources.addConfigFile(Toml, configFile)
   )

codex/clock.nim

@@ -8,7 +8,7 @@ type
   SecondsSince1970* = int64
   Timeout* = object of CatchableError
 
-method now*(clock: Clock): SecondsSince1970 {.base, upraises: [].} =
+method now*(clock: Clock): SecondsSince1970 {.base, gcsafe, upraises: [].} =
   raiseAssert "not implemented"
 
 method waitUntil*(clock: Clock, time: SecondsSince1970) {.base, async.} =

codex/codex.nim

@@ -11,7 +11,6 @@ import std/sequtils
 import std/strutils
 import std/os
 import std/tables
-import std/cpuinfo
 
 import pkg/chronos
 import pkg/presto
@@ -24,7 +23,6 @@ import pkg/stew/shims/net as stewnet
 import pkg/datastore
 import pkg/ethers except Rng
 import pkg/stew/io2
-import pkg/taskpools
 
 import ./node
 import ./conf
@@ -56,7 +54,6 @@ type
     codexNode: CodexNodeRef
     repoStore: RepoStore
     maintenance: BlockMaintainer
-    taskpool: Taskpool
 
   CodexPrivateKey* = libp2p.PrivateKey # alias
   EthWallet = ethers.Wallet
@@ -174,10 +171,6 @@ proc start*(s: CodexServer) {.async.} =
 proc stop*(s: CodexServer) {.async.} =
   notice "Stopping codex node"
 
-
-  s.taskpool.syncAll()
-  s.taskpool.shutdown()
-
   await allFuturesThrowing(
     s.restServer.stop(),
     s.codexNode.switch.stop(),
@@ -266,15 +259,12 @@ proc new*(
     else:
       none Prover
 
-    taskpool = Taskpool.new(num_threads = countProcessors())
-
     codexNode = CodexNodeRef.new(
       switch = switch,
       networkStore = store,
       engine = engine,
-      prover = prover,
       discovery = discovery,
-      taskpool = taskpool)
+      prover = prover)
 
     restServer = RestServerRef.new(
       codexNode.initRestApi(config, repoStore, config.apiCorsAllowedOrigin),
@@ -290,5 +280,4 @@ proc new*(
     codexNode: codexNode,
     restServer: restServer,
     repoStore: repoStore,
-    maintenance: maintenance,
-    taskpool: taskpool)
+    maintenance: maintenance)

codex/conf.nim

@@ -43,6 +43,7 @@ import ./units
 import ./utils
 import ./nat
 import ./utils/natutils
+
 from ./validationconfig import MaxSlots, ValidationGroups
 
 export units, net, codextypes, logutils, completeCmdArg, parseCmdArg, NatConfig
@@ -119,9 +120,9 @@ type
 
     metricsAddress* {.
       desc: "Listening address of the metrics server"
-      defaultValue: ValidIpAddress.init("127.0.0.1")
+      defaultValue: defaultAddress(config)
       defaultValueDesc: "127.0.0.1"
-      name: "metrics-address" }: ValidIpAddress
+      name: "metrics-address" }: IpAddress
 
     metricsPort* {.
       desc: "Listening HTTP port of the metrics server"
@@ -147,7 +148,7 @@ type
     nat* {.
       desc: "Specify method to use for determining public address. " &
             "Must be one of: any, none, upnp, pmp, extip:<IP>"
-      defaultValue: NatConfig(hasExtIp: false, nat: NatAny)
+      defaultValue: defaultNatConfig()
       defaultValueDesc: "any"
       name: "nat" }: NatConfig
 
@@ -410,6 +411,12 @@ type
 logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog
 logutils.formatIt(LogFormat.json, EthAddress): %it
 
+func defaultAddress*(conf: CodexConf): IpAddress =
+  result = static parseIpAddress("127.0.0.1")
+
+func defaultNatConfig*(): NatConfig =
+  result = NatConfig(hasExtIp: false, nat: NatStrategy.NatAny)
+
 func persistence*(self: CodexConf): bool =
   self.cmd == StartUpCmd.persistence
 
@@ -442,13 +449,17 @@ const
 
 proc parseCmdArg*(T: typedesc[MultiAddress],
                   input: string): MultiAddress
-                 {.upraises: [ValueError, LPError].} =
+                 {.upraises: [ValueError] .} =
   var ma: MultiAddress
-  let res = MultiAddress.init(input)
-  if res.isOk:
-    ma = res.get()
-  else:
-    warn "Invalid MultiAddress", input=input, error = res.error()
+  try:
+    let res = MultiAddress.init(input)
+    if res.isOk:
+      ma = res.get()
+    else:
+      warn "Invalid MultiAddress", input=input, error = res.error()
+      quit QuitFailure
+  except LPError as exc:
+    warn "Invalid MultiAddress uri", uri = input, error = exc.msg
     quit QuitFailure
   ma
 
@@ -458,6 +469,9 @@ proc parseCmdArg*(T: type SignedPeerRecord, uri: string): T =
     if not res.fromURI(uri):
       warn "Invalid SignedPeerRecord uri", uri = uri
       quit QuitFailure
+  except LPError as exc:
+    warn "Invalid SignedPeerRecord uri", uri = uri, error = exc.msg
+    quit QuitFailure
   except CatchableError as exc:
     warn "Invalid SignedPeerRecord uri", uri = uri, error = exc.msg
     quit QuitFailure
@@ -476,7 +490,7 @@ func parseCmdArg*(T: type NatConfig, p: string): T {.raises: [ValueError].} =
     else:
       if p.startsWith("extip:"):
         try:
-          let ip = ValidIpAddress.init(p[6..^1])
+          let ip = parseIpAddress(p[6..^1])
           NatConfig(hasExtIp: true, extIp: ip)
         except ValueError:
           let error = "Not a valid IP address: " & p[6..^1]
@@ -516,7 +530,11 @@ proc readValue*(r: var TomlReader, val: var SignedPeerRecord) =
     error "invalid SignedPeerRecord configuration value", error = err.msg
     quit QuitFailure
 
-  val = SignedPeerRecord.parseCmdArg(uri)
+  try:
+    val = SignedPeerRecord.parseCmdArg(uri)
+  except LPError as err:
+    warn "Invalid SignedPeerRecord uri", uri = uri, error = err.msg
+    quit QuitFailure
 
 proc readValue*(r: var TomlReader, val: var MultiAddress) =
   without input =? r.readValue(string).catch, err:

codex/discovery.nim

@@ -17,6 +17,7 @@ import pkg/questionable/results
 import pkg/stew/shims/net
 import pkg/contractabi/address as ca
 import pkg/codexdht/discv5/[routing_table, protocol as discv5]
+from pkg/nimcrypto import keccak256
 
 import ./rng
 import ./errors
@@ -124,7 +125,7 @@ method provide*(d: Discovery, host: ca.Address) {.async, base.} =
 
 method removeProvider*(
   d: Discovery,
-  peerId: PeerId): Future[void] {.base.} =
+  peerId: PeerId): Future[void] {.base, gcsafe.} =
   ## Remove provider from providers table
   ##
 
@@ -169,7 +170,7 @@ proc stop*(d: Discovery) {.async.} =
 proc new*(
     T: type Discovery,
     key: PrivateKey,
-    bindIp = ValidIpAddress.init(IPv4_any()),
+    bindIp = IPv4_any(),
     bindPort = 0.Port,
     announceAddrs: openArray[MultiAddress],
     bootstrapNodes: openArray[SignedPeerRecord] = [],
@@ -199,7 +200,7 @@ proc new*(
 
   self.protocol = newProtocol(
     key,
-    bindIp = bindIp.toNormalIp,
+    bindIp = bindIp,
     bindPort = bindPort,
     record = self.providerRecord.get,
     bootstrapRecords = bootstrapNodes,

codex/erasure/backend.nim

@@ -22,7 +22,7 @@ type
   EncoderBackend* = ref object of ErasureBackend
   DecoderBackend* = ref object of ErasureBackend
 
-method release*(self: ErasureBackend) {.base.} =
+method release*(self: ErasureBackend) {.base, gcsafe.} =
   ## release the backend
   ##
   raiseAssert("not implemented!")
@@ -31,7 +31,7 @@ method encode*(
     self: EncoderBackend,
     buffers,
     parity: var openArray[seq[byte]]
-): Result[void, cstring] {.base.} =
+): Result[void, cstring] {.base, gcsafe.} =
   ## encode buffers using a backend
   ##
   raiseAssert("not implemented!")
@@ -41,7 +41,7 @@ method decode*(
     buffers,
     parity,
     recovered: var openArray[seq[byte]]
-): Result[void, cstring] {.base.} =
+): Result[void, cstring] {.base, gcsafe.} =
   ## decode buffers using a backend
   ##
   raiseAssert("not implemented!")