From 4d4dfb7b32fc3e9137baf4a9416cf5f503fe4026 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Wed, 4 Sep 2024 00:15:41 +0300 Subject: [PATCH 1/3] Add cross-platform fsync() primitive and improve Direct mode. (#228) --- stew/io2.nim | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/stew/io2.nim b/stew/io2.nim index 2e54d6a..ba5a5ba 100644 --- a/stew/io2.nim +++ b/stew/io2.nim @@ -35,6 +35,7 @@ when defined(windows): FILE_APPEND_DATA = 0x00000004'u32 FILE_FLAG_NO_BUFFERING = 0x20000000'u32 + FILE_FLAG_WRITE_THROUGH = 0x80000000'u32 FILE_ATTRIBUTE_READONLY = 0x00000001'u32 FILE_ATTRIBUTE_DIRECTORY = 0x00000010'u32 @@ -200,6 +201,8 @@ when defined(windows): sideEffect.} proc getTempPathW(nBufferLength: uint32, lpBuffer: WideCString): uint32 {. importc: "GetTempPathW", dynlib: "kernel32", stdcall, sideEffect.} + proc flushFileBuffers(hFile: uint): int32 {. + importc: "FlushFileBuffers", dynlib: "kernel32", stdcall, sideEffect.} const NO_ERROR = IoErrorCode(0) @@ -1024,15 +1027,16 @@ proc openFile*(pathName: string, flags: set[OpenFlags], when defined(linux) or defined(freebsd) or defined(netbsd) or defined(dragonflybsd): if OpenFlags.Direct in flags: - cflags = cflags or O_DIRECT + cflags = cflags or O_DIRECT or O_SYNC if OpenFlags.Inherit notin flags: cflags = cflags or O_CLOEXEC if OpenFlags.NonBlock in flags: cflags = cflags or posix.O_NONBLOCK while true: - let omask = setUmask(0) - let ores = posix.open(cstring(pathName), cflags, Mode(createMode)) + let + omask = setUmask(0) + ores = posix.open(cstring(pathName), cflags, Mode(createMode)) discard setUmask(omask) if ores == -1: let errCode = ioLastError() @@ -1041,7 +1045,7 @@ proc openFile*(pathName: string, flags: set[OpenFlags], else: return err(errCode) else: - when defined(macosx): + when defined(macosx) or defined(macos): if OpenFlags.Direct in flags: while true: let fres = posix.fcntl(cint(ores), F_NOCACHE, 1) @@ -1105,7 +1109,7 @@ proc openFile*(pathName: string, flags: set[OpenFlags], if OpenFlags.NonBlock in flags: dwFlags = dwFlags or FILE_FLAG_OVERLAPPED if OpenFlags.Direct in flags: - dwFlags = dwFlags or FILE_FLAG_NO_BUFFERING + dwFlags = dwFlags or FILE_FLAG_NO_BUFFERING or FILE_FLAG_WRITE_THROUGH if OpenFlags.Inherit in flags: sa.bInheritHandle = 1 @@ -1391,6 +1395,26 @@ proc truncate*(pathName: string, length: int64): IoResult[void] = else: ok() +proc fsync*(handle: IoHandle): IoResult[void] = + ## Ensure that all data for the open file descriptor named by ``handle`` + ## is to be transferred to the storage device associated with the file. + when defined(windows): + let res = flushFileBuffers(uint(handle)) + if res == 0: + return err(ioLastError()) + ok() + else: + while true: + let res = posix.fsync(cint(handle)) + if res == -1: + let errCode = ioLastError() + if errCode == EINTR: + continue + else: + return err(errCode) + else: + return ok() + proc checkFileSize*(value: int64): IoResult[int] = ## Checks if ``value`` fits into supported by Nim string/sequence indexing ## mechanism. From d4634c5405ac188e7050d348332edb6c3b09a527 Mon Sep 17 00:00:00 2001 From: Eugene Kabanov Date: Wed, 4 Sep 2024 06:12:02 +0300 Subject: [PATCH 2/3] Add updateFilePos() primitive and tests. (#229) --- stew/io2.nim | 43 +++++++++++++++++++++++++++++++++++++++++++ tests/test_io2.nim | 25 +++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/stew/io2.nim b/stew/io2.nim index ba5a5ba..86bb3c2 100644 --- a/stew/io2.nim +++ b/stew/io2.nim @@ -1356,6 +1356,49 @@ proc setFilePos*(handle: IoHandle, offset: int64, else: ok() +proc updateFilePos*(handle: IoHandle, offset: int64, + whence: SeekPosition): IoResult[int64] = + ## Procedure shall set the file offset for the open file associated with the + ## file descriptor ``handle``, as follows: + ## * If whence is ``SeekPosition.SeekBegin``, the file offset shall be set + ## to ``offset`` bytes. + ## * If whence is ``SeekPosition.SeekCur``, the file offset shall be set to + ## its current location plus ``offset``. + ## * If whence is ``SeekPosition.SeekEnd``, the file offset shall be set to + ## the size of the file plus ``offset``. + ## + ## Returns the resulting offset location as measured in bytes from the + ## beginning of the file. + when defined(windows): + var noffset = 0'i64 + let pos = + case whence + of SeekBegin: + FILE_BEGIN + of SeekCurrent: + FILE_CURRENT + of SeekEnd: + FILE_END + let res = setFilePointerEx(uint(handle), offset, addr noffset, pos) + if res == 0: + err(ioLastError()) + else: + ok(noffset) + else: + let pos = + case whence + of SeekBegin: + posix.SEEK_SET + of SeekCurrent: + posix.SEEK_CUR + of SeekEnd: + posix.SEEK_END + let res = int64(posix.lseek(cint(handle), Off(offset), pos)) + if res == -1'i64: + err(ioLastError()) + else: + ok(res) + proc truncate*(handle: IoHandle, length: int64): IoResult[void] = ## Procedure cause the regular file referenced by handle ``handle`` to be ## truncated to a size of precisely ``length`` bytes. diff --git a/tests/test_io2.nim b/tests/test_io2.nim index 83af09d..f1318f7 100644 --- a/tests/test_io2.nim +++ b/tests/test_io2.nim @@ -629,6 +629,31 @@ suite "OS Input/Output procedures test suite": positions[3] == 10'i64 positions[4] == 20'i64 + test "updateFilePos(handle) test": + proc performTest(path: string): IoResult[tuple[s0, s1, s2, s3, s4: int64]] = + let flags = {OpenFlags.Write, OpenFlags.Truncate, OpenFlags.Create} + let handle = ? openFile(path, flags) + let msg = "AAAAABBBBBCCCCCDDDDD" + discard ? io2.writeFile(handle, msg) + let + pos0 = ? updateFilePos(handle, 0'i64, SeekBegin) + pos1 = ? updateFilePos(handle, 5'i64, SeekCurrent) + pos2 = ? updateFilePos(handle, 5'i64, SeekCurrent) + pos3 = ? updateFilePos(handle, 0'i64, SeekEnd) + pos4 = ? updateFilePos(handle, -5'i64, SeekEnd) + ? closeFile(handle) + ? removeFile(path) + ok((pos0, pos1, pos2, pos3, pos4)) + let res = performTest("testblob4") + check res.isOk() + let positions = res.get() + check: + positions[0] == 0'i64 + positions[1] == 5'i64 + positions[2] == 10'i64 + positions[3] == 20'i64 + positions[4] == 15'i64 + test "lockFile(handle)/unlockFile(handle) test": type TestResult = object From 90a9bfd84389ad9ff1639a54404bed20c861ded8 Mon Sep 17 00:00:00 2001 From: Miran Date: Wed, 11 Sep 2024 02:30:21 +0200 Subject: [PATCH 3/3] update ci.yml to test Nim 2.2; also test gcc-14 (#230) * update ci.yml to test Nim 2.2; also test gcc-14 * the installation of openssl 1.1 is maybe not needed? --- .github/workflows/ci.yml | 39 ++++++++++++++++++++++++++++----------- stew.nimble | 4 ++-- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab4ad54..56248a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,23 +18,35 @@ jobs: target: - os: linux cpu: amd64 + - os: linux-gcc-14 # This is to use ubuntu 24 and install gcc 14. Should be removed when ubuntu-latest is 26.04 + cpu: amd64 - os: linux cpu: i386 - os: macos cpu: amd64 + - os: macos + cpu: arm64 - os: windows cpu: amd64 - #- os: windows - #cpu: i386 - branch: [version-1-6, version-2-0, devel] + branch: [version-2-0, version-2-2, devel] include: - target: os: linux builder: ubuntu-latest shell: bash + - target: + os: linux-gcc-14 # This is to use ubuntu 24 and install gcc 14. Should be removed when ubuntu-latest is 26.04 + builder: ubuntu-24.04 + shell: bash - target: os: macos - builder: macos-12 + cpu: amd64 + builder: macos-13 + shell: bash + - target: + os: macos + cpu: arm64 + builder: macos-latest shell: bash - target: os: windows @@ -72,13 +84,6 @@ jobs: chmod 755 external/bin/gcc external/bin/g++ echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH - - name: 'Install dependencies (macOS)' - if: runner.os == 'macOS' && matrix.branch == 'devel' - run: | - brew install openssl@1.1 - ln -s $(brew --prefix)/opt/openssl/lib/libcrypto.1.1.dylib /usr/local/lib - ln -s $(brew --prefix)/opt/openssl/lib/libssl.1.1.dylib /usr/local/lib/ - - name: MSYS2 (Windows i386) if: runner.os == 'Windows' && matrix.target.cpu == 'i386' uses: msys2/setup-msys2@v2 @@ -127,6 +132,8 @@ jobs: run: | if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then PLATFORM=x64 + elif [[ '${{ matrix.target.cpu }}' == 'arm64' ]]; then + PLATFORM=arm64 else PLATFORM=x86 fi @@ -158,6 +165,15 @@ jobs: bash build_nim.sh nim csources dist/nimble NimBinaries echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH + - name: Use gcc 14 + # Should be removed when ubuntu-latest is 26.04 + if : ${{ matrix.target.os == 'linux-gcc-14' }} + run: | + # Add GCC-14 to alternatives + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 14 + # Set GCC-14 as the default + sudo update-alternatives --set gcc /usr/bin/gcc-14 + - name: Run tests run: | if [[ "${{ matrix.target.os }}" == "windows" ]]; then @@ -166,6 +182,7 @@ jobs: fi nim --version nimble --version + gcc --version nimble install -y --depsOnly rm -f nimble.lock env NIMLANG=c nimble test diff --git a/stew.nimble b/stew.nimble index 6e0a356..faa0867 100644 --- a/stew.nimble +++ b/stew.nimble @@ -33,6 +33,6 @@ task test, "Run all tests": "--threads:off", "--threads:on -d:nimTypeNames", "--threads:on -d:noIntrinsicsBitOpts -d:noIntrinsicsEndians"]: - run args, "tests/all_tests" + run args & " --mm:refc", "tests/all_tests" if (NimMajor, NimMinor) > (1, 6): - run args & " --mm:refc", "tests/all_tests" \ No newline at end of file + run args & " --mm:orc", "tests/all_tests"