diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000000..a897ef815b --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,127 @@ +# UMF (Unified Memory Framework) - AI Coding Guide + +## Project Architecture + +UMF is a C library for constructing memory allocators and pools, built around a two-layer architecture: + +- **Memory Providers** (`src/provider/`): Handle coarse-grained OS-level memory allocation (mmap, CUDA, Level Zero, etc.) +- **Memory Pools** (`src/pool/`): Handle fine-grained allocation using providers as backing store (jemalloc, scalable, disjoint) + +Key architectural patterns: +- Provider/pool separation enables mixing any provider with any pool allocator +- Operations structures (`*_ops_t`) define plugin interfaces for extensibility +- Handle-based API (`*_handle_t`) abstracts implementation details +- Result codes (`umf_result_t`) for consistent error handling + +## Development Workflows + +### Build System +```bash +# Standard build +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build -j $(nproc) + +# Enable all features for development +# GPU tests will work only in an environment with proper hardware and drivers +cmake -B build -DCMAKE_BUILD_TYPE=Debug \ + -DUMF_BUILD_TESTS=ON -DUMF_BUILD_GPU_TESTS=OFF \ + -DUMF_BUILD_EXAMPLES=ON -DUMF_DEVELOPER_MODE=ON \ + -DUMF_FORMAT_CODE_STYLE=ON +``` + +### Version Management +- Version determined by: + 1. `git describe` (preferred) + 2. `VERSION` file fallback + 3. "0.0.0" default +- `set_version_variables()` in `cmake/helpers.cmake` handles version detection +- For releases: create `VERSION` file with semver format (e.g., "1.0.3") + +### Code Formatting +- **Always format code before committing**: `make format-apply` +- Requires build with `-DUMF_FORMAT_CODE_STYLE=ON` +- Uses clang-format-15.0, cmake-format-0.6, and black for Python + +### Testing Patterns +- Use `build_umf_test()` CMake function in `test/CMakeLists.txt` +- GPU tests require `UMF_BUILD_GPU_TESTS=ON` and hardware/drivers +- IPC tests use producer/consumer pattern with shell scripts +- Platform-specific tests: `.c` files for portability, `.cpp` for C++ features, utils, and selected tests + +### CI/CD Structure +- `pr_push.yml`: Main workflow calling reusable workflows. It's called for each PR change or push to main/stable branches +- Separate workflows for different configurations: `reusable_gpu.yml`, `reusable_sanitizers.yml`, etc. +- Provider-specific testing: Level Zero, CUDA runners with actual hardware + +## Coding Conventions + +### Naming Patterns +- Public API: `umf*` prefix (e.g., `umfMemoryProviderCreate`) +- Internal functions: `snake_case` without prefix +- Structures: `*_t` suffix for types, `*_handle_t` for opaque handles +- Constants: `UMF_*` uppercase with underscores + +### Memory Management Patterns +- Always pair create/destroy functions (e.g., `umfMemoryProviderCreate`/`umfMemoryProviderDestroy`) +- Use `umf_result_t` return codes, never throw exceptions +- Provider params have separate create/destroy lifecycle +- Thread-local storage (`__TLS`) for error state in providers + +### Provider Implementation Pattern +```c +// Standard provider structure +typedef struct my_provider_t { + // Provider-specific state +} my_provider_t; + +static umf_result_t my_initialize(const void *params, void **provider); +static umf_result_t my_finalize(void *provider); +static umf_result_t my_alloc(void *provider, size_t size, size_t alignment, void **ptr); +static umf_result_t my_free(void *provider, void *ptr, size_t size); + +static const umf_memory_provider_ops_t MY_PROVIDER_OPS = { + .version = UMF_PROVIDER_OPS_VERSION_CURRENT, + .initialize = my_initialize, + .finalize = my_finalize, + .alloc = my_alloc, + .free = my_free, + // ... other required ops +}; +``` + +## Key Files and Patterns + +### Core APIs +- `include/umf.h`: Main header, include this for basic usage +- `include/umf/memory_provider_ops.h`: Provider plugin interface +- `include/umf/memory_pool_ops.h`: Pool plugin interface + +### Common Utilities +- `src/utils/`: Logging (`utils_log.h`), concurrency (`utils_concurrency.h`), assertions +- `src/critnib/`: Concurrent radix tree for address tracking +- `src/base_alloc/`: Base allocation utilities + +### Platform Abstractions +- `libumf_linux.c`/`libumf_windows.c`: OS-specific implementations +- `topology.c`: HWLOC integration for NUMA topology discovery +- Provider files handle platform-specific allocation (CUDA, Level Zero, OS memory) + +## Integration Points + +### NUMA Support +- Uses HWLOC for topology discovery (`topology.c`, `umf_hwloc.h`) +- NUMA policies in `mempolicy.c`: bind, interleave, split modes +- Memory spaces (`memspace.c`) and targets (`memtarget.c`) for NUMA abstraction + +### GPU Integration +- Level Zero provider: `provider_level_zero.c` for Intel GPUs +- CUDA provider: `provider_cuda.c` for NVIDIA GPUs +- Examples in `examples/level_zero_shared_memory/` and `examples/cuda_shared_memory/` + +### IPC (Inter-Process Communication) +- Linux-specific implementation using file descriptor passing +- Requires `PTRACE_MODE_ATTACH_REALCREDS` permission +- Uses `memfd_create()` or `memfd_secret()` for anonymous shared memory + +When implementing new providers or pools, follow the existing patterns in +`src/provider/provider_os_memory.c` and `src/pool/pool_scalable.c` as reference implementations. diff --git a/CMakeLists.txt b/CMakeLists.txt index ac77c61575..2a60a22502 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,8 @@ if(UMF_CMAKE_VERSION VERSION_EQUAL "0.0.0") message( WARNING "UMF version is set to 0.0.0, which most likely is not expected! " - "Please checkout the git tags to get a proper version.") + "Please install git and checkout the git tags to get a proper version." + ) endif() if(PROJECT_VERSION_PATCH GREATER 0) diff --git a/RELEASE_STEPS.md b/RELEASE_STEPS.md index 32ab7b5c37..75ca16bae8 100644 --- a/RELEASE_STEPS.md +++ b/RELEASE_STEPS.md @@ -50,7 +50,11 @@ Prepare changes for the release: - Once all changes are done, build locally (and/or verify changes on CI), including: - Verify if scanners/linters/checkers passed - Verify if version is set properly, especially in `.dll` and `.so` files +- Create/update a VERSION file for GitHub ZIP downloads (users without git): + - `echo "$VERSION" > VERSION` + - It will always contain "the last released version". In logs/CMake build it will introduce itself as `$VERSION-dev`, only if git is not available - Commit these changes and tag the release: + - `git add VERSION` - `git commit -a -S -m "$VERSION release"` - `git tag -a -s -m "Version $VERSION" v$VERSION` - Verify if commit and tag are properly signed: diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index 8fea0a7431..8ee14bd377 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -11,10 +11,12 @@ include(CheckCCompilerFlag) include(CheckCXXCompilerFlag) # This function establishes version variables based on the git describe output. -# If there's no git available in the system, the version will be set to "0.0.0". -# If git reports only a hash, the version will be set to "0.0.0.git.". -# Otherwise we'll use 3-component version: major.minor.patch, just for CMake's -# sake. A few extra variables will be set for Win dll metadata. +# If there's no git available in the system, it falls back to reading a VERSION +# file from the project root. If neither git nor VERSION file is available, the +# version will be set to "0.0.0". If git reports only a hash, the version will +# be set to "0.0.0.git.". Otherwise we'll use 3-component version: +# major.minor.patch, just for CMake's sake. A few extra variables will be set +# for Win dll metadata. # # Important note: CMake does not support rc or git information. According to # semver rules, 1.5.1-rc1 should be less than 1.5.1, but it seems hard to @@ -78,8 +80,24 @@ function(set_version_variables) OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) if(NOT GIT_VERSION) - # no git or it reported no version. Use default ver: "0.0.0" - return() + # no git or it reported no version. Try fallback to VERSION file + if(EXISTS "${UMF_CMAKE_SOURCE_DIR}/VERSION") + file(READ "${UMF_CMAKE_SOURCE_DIR}/VERSION" FILE_VERSION) + string(STRIP ${FILE_VERSION} FILE_VERSION) + if(FILE_VERSION) + set(GIT_VERSION "v${FILE_VERSION}-dev") + message( + STATUS + "Using version from VERSION file: ${FILE_VERSION}. To get detailed version, use git and fetch tags." + ) + else() + # VERSION file exists but is empty, use default ver: "0.0.0" + return() + endif() + else() + # no git and no VERSION file. Use default ver: "0.0.0" + return() + endif() endif() # v1.5.0 - we're exactly on a tag -> UMF ver: "1.5.0"