From 7bce62d198a2cd3dfcb6bf871583687f5ed3c6c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Thu, 25 Sep 2025 15:58:51 +0200 Subject: [PATCH 1/2] Add copilot instructions Apparently this file speeds up process of AI contributions to this repository. It was almost fully generated by copilot. --- .github/copilot-instructions.md | 127 ++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000000..d8087945ce --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,127 @@ +# UMF (Unified Memory Framework) - AI Coding Guide + +## Project Architecture + +UMF is a C library for constructing memory allocators and pools, built around a two-layer architecture: + +- **Memory Providers** (`src/provider/`): Handle coarse-grained OS-level memory allocation (mmap, CUDA, Level Zero, etc.) +- **Memory Pools** (`src/pool/`): Handle fine-grained allocation using providers as backing store (jemalloc, scalable, disjoint) + +Key architectural patterns: +- Provider/pool separation enables mixing any provider with any pool allocator +- Operations structures (`*_ops_t`) define plugin interfaces for extensibility +- Handle-based API (`*_handle_t`) abstracts implementation details +- Result codes (`umf_result_t`) for consistent error handling + +## Development Workflows + +### Build System +```bash +# Standard build +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build -j $(nproc) + +# Enable all features for development +# GPU tests will work only in an environment with proper hardware and drivers +cmake -B build -DCMAKE_BUILD_TYPE=Debug \ + -DUMF_BUILD_TESTS=ON -DUMF_BUILD_GPU_TESTS=OFF \ + -DUMF_BUILD_EXAMPLES=ON -DUMF_DEVELOPER_MODE=ON \ + -DUMF_FORMAT_CODE_STYLE=ON +``` + +### Version Management +- Version determined by: + 1. `git describe` (preferred) + 2. `VERSION` file fallback + 3. "0.0.0" default +- `set_version_variables()` in `cmake/helpers.cmake` handles version detection +- For releases: create `VERSION` file with semver format (e.g., "1.0.3-dev") + +### Code Formatting +- **Always format code before committing**: `make format-apply` +- Requires build with `-DUMF_FORMAT_CODE_STYLE=ON` +- Uses clang-format-15.0, cmake-format-0.6, and black for Python + +### Testing Patterns +- Use `build_umf_test()` CMake function in `test/CMakeLists.txt` +- GPU tests require `UMF_BUILD_GPU_TESTS=ON` and hardware/drivers +- IPC tests use producer/consumer pattern with shell scripts +- Platform-specific tests: `.c` files for portability, `.cpp` for C++ features, utils, and selected tests + +### CI/CD Structure +- `pr_push.yml`: Main workflow calling reusable workflows. It's called for each PR change or push to main/stable branches +- Separate workflows for different configurations: `reusable_gpu.yml`, `reusable_sanitizers.yml`, etc. +- Provider-specific testing: Level Zero, CUDA runners with actual hardware + +## Coding Conventions + +### Naming Patterns +- Public API: `umf*` prefix (e.g., `umfMemoryProviderCreate`) +- Internal functions: `snake_case` without prefix +- Structures: `*_t` suffix for types, `*_handle_t` for opaque handles +- Constants: `UMF_*` uppercase with underscores + +### Memory Management Patterns +- Always pair create/destroy functions (e.g., `umfMemoryProviderCreate`/`umfMemoryProviderDestroy`) +- Use `umf_result_t` return codes, never throw exceptions +- Provider params have separate create/destroy lifecycle +- Thread-local storage (`__TLS`) for error state in providers + +### Provider Implementation Pattern +```c +// Standard provider structure +typedef struct my_provider_t { + // Provider-specific state +} my_provider_t; + +static umf_result_t my_initialize(const void *params, void **provider); +static umf_result_t my_finalize(void *provider); +static umf_result_t my_alloc(void *provider, size_t size, size_t alignment, void **ptr); +static umf_result_t my_free(void *provider, void *ptr, size_t size); + +static const umf_memory_provider_ops_t MY_PROVIDER_OPS = { + .version = UMF_PROVIDER_OPS_VERSION_CURRENT, + .initialize = my_initialize, + .finalize = my_finalize, + .alloc = my_alloc, + .free = my_free, + // ... other required ops +}; +``` + +## Key Files and Patterns + +### Core APIs +- `include/umf.h`: Main header, include this for basic usage +- `include/umf/memory_provider_ops.h`: Provider plugin interface +- `include/umf/memory_pool_ops.h`: Pool plugin interface + +### Common Utilities +- `src/utils/`: Logging (`utils_log.h`), concurrency (`utils_concurrency.h`), assertions +- `src/critnib/`: Concurrent radix tree for address tracking +- `src/base_alloc/`: Base allocation utilities + +### Platform Abstractions +- `libumf_linux.c`/`libumf_windows.c`: OS-specific implementations +- `topology.c`: HWLOC integration for NUMA topology discovery +- Provider files handle platform-specific allocation (CUDA, Level Zero, OS memory) + +## Integration Points + +### NUMA Support +- Uses HWLOC for topology discovery (`topology.c`, `umf_hwloc.h`) +- NUMA policies in `mempolicy.c`: bind, interleave, split modes +- Memory spaces (`memspace.c`) and targets (`memtarget.c`) for NUMA abstraction + +### GPU Integration +- Level Zero provider: `provider_level_zero.c` for Intel GPUs +- CUDA provider: `provider_cuda.c` for NVIDIA GPUs +- Examples in `examples/level_zero_shared_memory/` and `examples/cuda_shared_memory/` + +### IPC (Inter-Process Communication) +- Linux-specific implementation using file descriptor passing +- Requires `PTRACE_MODE_ATTACH_REALCREDS` permission +- Uses `memfd_create()` or `memfd_secret()` for anonymous shared memory + +When implementing new providers or pools, follow the existing patterns in +`src/provider/provider_os_memory.c` and `src/pool/pool_scalable.c` as reference implementations. From b9c94bb632569b655be3b3144cf2a200eb2178d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Thu, 25 Sep 2025 16:03:34 +0200 Subject: [PATCH 2/2] Add support for versioning in git archives While releasing UMF we should update VERSION file with the current version. It will be used only if git is not available or tags were not fetched. To underline it is not a "detailed version" we will add "-dev" in CMake version parsing, so it will stand out (comparing to e.g. versions like "1.0.3-dev1-..."). --- .github/copilot-instructions.md | 2 +- CMakeLists.txt | 3 ++- RELEASE_STEPS.md | 4 ++++ cmake/helpers.cmake | 30 ++++++++++++++++++++++++------ 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index d8087945ce..a897ef815b 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -35,7 +35,7 @@ cmake -B build -DCMAKE_BUILD_TYPE=Debug \ 2. `VERSION` file fallback 3. "0.0.0" default - `set_version_variables()` in `cmake/helpers.cmake` handles version detection -- For releases: create `VERSION` file with semver format (e.g., "1.0.3-dev") +- For releases: create `VERSION` file with semver format (e.g., "1.0.3") ### Code Formatting - **Always format code before committing**: `make format-apply` diff --git a/CMakeLists.txt b/CMakeLists.txt index ac77c61575..2a60a22502 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,8 @@ if(UMF_CMAKE_VERSION VERSION_EQUAL "0.0.0") message( WARNING "UMF version is set to 0.0.0, which most likely is not expected! " - "Please checkout the git tags to get a proper version.") + "Please install git and checkout the git tags to get a proper version." + ) endif() if(PROJECT_VERSION_PATCH GREATER 0) diff --git a/RELEASE_STEPS.md b/RELEASE_STEPS.md index 32ab7b5c37..75ca16bae8 100644 --- a/RELEASE_STEPS.md +++ b/RELEASE_STEPS.md @@ -50,7 +50,11 @@ Prepare changes for the release: - Once all changes are done, build locally (and/or verify changes on CI), including: - Verify if scanners/linters/checkers passed - Verify if version is set properly, especially in `.dll` and `.so` files +- Create/update a VERSION file for GitHub ZIP downloads (users without git): + - `echo "$VERSION" > VERSION` + - It will always contain "the last released version". In logs/CMake build it will introduce itself as `$VERSION-dev`, only if git is not available - Commit these changes and tag the release: + - `git add VERSION` - `git commit -a -S -m "$VERSION release"` - `git tag -a -s -m "Version $VERSION" v$VERSION` - Verify if commit and tag are properly signed: diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index 8fea0a7431..8ee14bd377 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -11,10 +11,12 @@ include(CheckCCompilerFlag) include(CheckCXXCompilerFlag) # This function establishes version variables based on the git describe output. -# If there's no git available in the system, the version will be set to "0.0.0". -# If git reports only a hash, the version will be set to "0.0.0.git.". -# Otherwise we'll use 3-component version: major.minor.patch, just for CMake's -# sake. A few extra variables will be set for Win dll metadata. +# If there's no git available in the system, it falls back to reading a VERSION +# file from the project root. If neither git nor VERSION file is available, the +# version will be set to "0.0.0". If git reports only a hash, the version will +# be set to "0.0.0.git.". Otherwise we'll use 3-component version: +# major.minor.patch, just for CMake's sake. A few extra variables will be set +# for Win dll metadata. # # Important note: CMake does not support rc or git information. According to # semver rules, 1.5.1-rc1 should be less than 1.5.1, but it seems hard to @@ -78,8 +80,24 @@ function(set_version_variables) OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) if(NOT GIT_VERSION) - # no git or it reported no version. Use default ver: "0.0.0" - return() + # no git or it reported no version. Try fallback to VERSION file + if(EXISTS "${UMF_CMAKE_SOURCE_DIR}/VERSION") + file(READ "${UMF_CMAKE_SOURCE_DIR}/VERSION" FILE_VERSION) + string(STRIP ${FILE_VERSION} FILE_VERSION) + if(FILE_VERSION) + set(GIT_VERSION "v${FILE_VERSION}-dev") + message( + STATUS + "Using version from VERSION file: ${FILE_VERSION}. To get detailed version, use git and fetch tags." + ) + else() + # VERSION file exists but is empty, use default ver: "0.0.0" + return() + endif() + else() + # no git and no VERSION file. Use default ver: "0.0.0" + return() + endif() endif() # v1.5.0 - we're exactly on a tag -> UMF ver: "1.5.0"