Merge branch 'master' into arraybuf-setlen

.github/workflows/ci.yml

@@ -18,23 +18,35 @@ jobs:
         target:
           - os: linux
             cpu: amd64
+          - os: linux-gcc-14 # This is to use ubuntu 24 and install gcc 14. Should be removed when ubuntu-latest is 26.04
+            cpu: amd64
           - os: linux
             cpu: i386
           - os: macos
             cpu: amd64
+          - os: macos
+            cpu: arm64
           - os: windows
             cpu: amd64
-          #- os: windows
-            #cpu: i386
-        branch: [version-1-6, version-2-0, devel]
+        branch: [version-2-0, version-2-2, devel]
         include:
           - target:
               os: linux
             builder: ubuntu-latest
             shell: bash
+          - target:
+              os: linux-gcc-14 # This is to use ubuntu 24 and install gcc 14. Should be removed when ubuntu-latest is 26.04
+            builder: ubuntu-24.04
+            shell: bash
           - target:
               os: macos
-            builder: macos-12
+              cpu: amd64
+            builder: macos-13
+            shell: bash
+          - target:
+              os: macos
+              cpu: arm64
+            builder: macos-latest
             shell: bash
           - target:
               os: windows
@@ -72,13 +84,6 @@ jobs:
           chmod 755 external/bin/gcc external/bin/g++
           echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
 
-      - name: 'Install dependencies (macOS)'
-        if: runner.os == 'macOS' && matrix.branch == 'devel'
-        run: |
-          brew install [email protected]
-          ln -s $(brew --prefix)/opt/openssl/lib/libcrypto.1.1.dylib /usr/local/lib
-          ln -s $(brew --prefix)/opt/openssl/lib/libssl.1.1.dylib /usr/local/lib/
-
       - name: MSYS2 (Windows i386)
         if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
         uses: msys2/setup-msys2@v2
@@ -127,6 +132,8 @@ jobs:
         run: |
           if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
             PLATFORM=x64
+          elif [[ '${{ matrix.target.cpu }}' == 'arm64' ]]; then
+            PLATFORM=arm64
           else
             PLATFORM=x86
           fi
@@ -158,6 +165,15 @@ jobs:
             bash build_nim.sh nim csources dist/nimble NimBinaries
           echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
 
+      - name: Use gcc 14
+        # Should be removed when ubuntu-latest is 26.04
+        if : ${{ matrix.target.os == 'linux-gcc-14' }}
+        run: |
+          # Add GCC-14 to alternatives
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 14
+          # Set GCC-14 as the default
+          sudo update-alternatives --set gcc /usr/bin/gcc-14
+
       - name: Run tests
         run: |
           if [[ "${{ matrix.target.os }}" == "windows" ]]; then
@@ -166,6 +182,7 @@ jobs:
           fi
           nim --version
           nimble --version
+          gcc --version
           nimble install -y --depsOnly
           rm -f nimble.lock
           env NIMLANG=c nimble test

stew.nimble

@@ -33,6 +33,6 @@ task test, "Run all tests":
       "--threads:off",
       "--threads:on -d:nimTypeNames",
       "--threads:on -d:noIntrinsicsBitOpts -d:noIntrinsicsEndians"]:
-    run args, "tests/all_tests"
+    run args & " --mm:refc", "tests/all_tests"
     if (NimMajor, NimMinor) > (1, 6):
-      run args & " --mm:refc", "tests/all_tests"
+      run args & " --mm:orc", "tests/all_tests"

stew/io2.nim

@@ -35,6 +35,7 @@ when defined(windows):
     FILE_APPEND_DATA = 0x00000004'u32
 
     FILE_FLAG_NO_BUFFERING = 0x20000000'u32
+    FILE_FLAG_WRITE_THROUGH = 0x80000000'u32
     FILE_ATTRIBUTE_READONLY = 0x00000001'u32
     FILE_ATTRIBUTE_DIRECTORY = 0x00000010'u32
 
@@ -200,6 +201,8 @@ when defined(windows):
        sideEffect.}
   proc getTempPathW(nBufferLength: uint32, lpBuffer: WideCString): uint32 {.
        importc: "GetTempPathW", dynlib: "kernel32", stdcall, sideEffect.}
+  proc flushFileBuffers(hFile: uint): int32 {.
+       importc: "FlushFileBuffers", dynlib: "kernel32", stdcall, sideEffect.}
 
   const
     NO_ERROR = IoErrorCode(0)
@@ -1024,15 +1027,16 @@ proc openFile*(pathName: string, flags: set[OpenFlags],
     when defined(linux) or defined(freebsd) or defined(netbsd) or
          defined(dragonflybsd):
       if OpenFlags.Direct in flags:
-        cflags = cflags or O_DIRECT
+        cflags = cflags or O_DIRECT or O_SYNC
     if OpenFlags.Inherit notin flags:
       cflags = cflags or O_CLOEXEC
     if OpenFlags.NonBlock in flags:
       cflags = cflags or posix.O_NONBLOCK
 
     while true:
-      let omask = setUmask(0)
-      let ores = posix.open(cstring(pathName), cflags, Mode(createMode))
+      let
+        omask = setUmask(0)
+        ores = posix.open(cstring(pathName), cflags, Mode(createMode))
       discard setUmask(omask)
       if ores == -1:
         let errCode = ioLastError()
@@ -1041,7 +1045,7 @@ proc openFile*(pathName: string, flags: set[OpenFlags],
         else:
           return err(errCode)
       else:
-        when defined(macosx):
+        when defined(macosx) or defined(macos):
           if OpenFlags.Direct in flags:
             while true:
               let fres = posix.fcntl(cint(ores), F_NOCACHE, 1)
@@ -1105,7 +1109,7 @@ proc openFile*(pathName: string, flags: set[OpenFlags],
     if OpenFlags.NonBlock in flags:
       dwFlags = dwFlags or FILE_FLAG_OVERLAPPED
     if OpenFlags.Direct in flags:
-      dwFlags = dwFlags or FILE_FLAG_NO_BUFFERING
+      dwFlags = dwFlags or FILE_FLAG_NO_BUFFERING or FILE_FLAG_WRITE_THROUGH
     if OpenFlags.Inherit in flags:
       sa.bInheritHandle = 1
 
@@ -1352,6 +1356,49 @@ proc setFilePos*(handle: IoHandle, offset: int64,
     else:
       ok()
 
+proc updateFilePos*(handle: IoHandle, offset: int64,
+                    whence: SeekPosition): IoResult[int64] =
+  ## Procedure shall set the file offset for the open file associated with the
+  ## file descriptor ``handle``, as follows:
+  ##   * If whence is ``SeekPosition.SeekBegin``, the file offset shall be set
+  ##     to ``offset`` bytes.
+  ##   * If whence is ``SeekPosition.SeekCur``, the file offset shall be set to
+  ##     its current location plus ``offset``.
+  ##   * If whence is ``SeekPosition.SeekEnd``, the file offset shall be set to
+  ##     the size of the file plus ``offset``.
+  ##
+  ## Returns the resulting offset location as measured in bytes from the
+  ## beginning of the file.
+  when defined(windows):
+    var noffset = 0'i64
+    let pos =
+      case whence
+      of SeekBegin:
+        FILE_BEGIN
+      of SeekCurrent:
+        FILE_CURRENT
+      of SeekEnd:
+        FILE_END
+    let res = setFilePointerEx(uint(handle), offset, addr noffset, pos)
+    if res == 0:
+      err(ioLastError())
+    else:
+      ok(noffset)
+  else:
+    let pos =
+      case whence
+      of SeekBegin:
+        posix.SEEK_SET
+      of SeekCurrent:
+        posix.SEEK_CUR
+      of SeekEnd:
+        posix.SEEK_END
+    let res = int64(posix.lseek(cint(handle), Off(offset), pos))
+    if res == -1'i64:
+      err(ioLastError())
+    else:
+      ok(res)
+
 proc truncate*(handle: IoHandle, length: int64): IoResult[void] =
   ## Procedure cause the regular file referenced by handle ``handle`` to be
   ## truncated to a size of precisely ``length`` bytes.
@@ -1391,6 +1438,26 @@ proc truncate*(pathName: string, length: int64): IoResult[void] =
     else:
       ok()
 
+proc fsync*(handle: IoHandle): IoResult[void] =
+  ## Ensure that all data for the open file descriptor named by ``handle``
+  ## is to be transferred to the storage device associated with the file.
+  when defined(windows):
+    let res = flushFileBuffers(uint(handle))
+    if res == 0:
+      return err(ioLastError())
+    ok()
+  else:
+    while true:
+      let res = posix.fsync(cint(handle))
+      if res == -1:
+        let errCode = ioLastError()
+        if errCode == EINTR:
+          continue
+        else:
+          return err(errCode)
+      else:
+        return ok()
+
 proc checkFileSize*(value: int64): IoResult[int] =
   ## Checks if ``value`` fits into supported by Nim string/sequence indexing
   ## mechanism.

tests/test_io2.nim

@@ -629,6 +629,31 @@ suite "OS Input/Output procedures test suite":
       positions[3] == 10'i64
       positions[4] == 20'i64
 
+  test "updateFilePos(handle) test":
+    proc performTest(path: string): IoResult[tuple[s0, s1, s2, s3, s4: int64]] =
+      let flags = {OpenFlags.Write, OpenFlags.Truncate, OpenFlags.Create}
+      let handle = ? openFile(path, flags)
+      let msg = "AAAAABBBBBCCCCCDDDDD"
+      discard ? io2.writeFile(handle, msg)
+      let
+        pos0 = ? updateFilePos(handle, 0'i64, SeekBegin)
+        pos1 = ? updateFilePos(handle, 5'i64, SeekCurrent)
+        pos2 = ? updateFilePos(handle, 5'i64, SeekCurrent)
+        pos3 = ? updateFilePos(handle, 0'i64, SeekEnd)
+        pos4 = ? updateFilePos(handle, -5'i64, SeekEnd)
+      ? closeFile(handle)
+      ? removeFile(path)
+      ok((pos0, pos1, pos2, pos3, pos4))
+    let res = performTest("testblob4")
+    check res.isOk()
+    let positions = res.get()
+    check:
+      positions[0] == 0'i64
+      positions[1] == 5'i64
+      positions[2] == 10'i64
+      positions[3] == 20'i64
+      positions[4] == 15'i64
+
   test "lockFile(handle)/unlockFile(handle) test":
     type
       TestResult = object