Merge branch 'master' into pycli

.github/workflows/container.yml

@@ -11,7 +11,27 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - name: Free disk space
+        run: |
+          # inspired by: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh
+          df -h
+          # 100 largest packages, in ascending order
+          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
+          sudo apt-get remove -y google-cloud-cli
+          sudo apt-get remove -y azure-cli
+          sudo apt-get remove -y microsoft-edge-stable
+          sudo apt-get remove -y '^dotnet-.*'
+          sudo apt-get remove -y '^temurin-.*-jdk'
+          sudo apt-get remove -y google-chrome-stable
+          sudo apt-get remove -y '^llvm-.*-dev'
+          sudo apt-get remove -y firefox
+          sudo apt-get remove -y powershell
+          sudo apt-get remove -y mono-devel
+          sudo apt-get autoremove -y
+          sudo apt-get clean
+          # disk space after removing packages
+          df -h
       - name: Build image
         uses: redhat-actions/buildah-build@v2
         with:

.github/workflows/python.yml

@@ -9,7 +9,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v3
@@ -20,8 +20,7 @@ jobs:
       - name: Test
         run: |
           cd pineappl_py
-          pip install virtualenv
-          virtualenv env
+          python -m venv env
           . env/bin/activate
           pip install maturin
           maturin develop --extras test

.github/workflows/release.yml

@@ -31,9 +31,9 @@ jobs:
       - name: Compile library
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --target=${{ matrix.target }}
-          cargo install cargo-c
+          cargo install --locked cargo-c
           cd pineappl_capi
-          cargo cinstall --verbose --destdir=prefix --library-type=cdylib --prefix=/ --target=${{ matrix.target }}
+          cargo cinstall --destdir=prefix --library-type=cdylib --locked --prefix=/ --target=${{ matrix.target }} --verbose
           cd prefix
           tar czf ../../pineappl_capi-${{ matrix.target }}.tar.gz .
       - name: Upload artifact
@@ -43,6 +43,56 @@ jobs:
           name: pineappl_capi-${{ matrix.target }}
           path: pineappl_capi-${{ matrix.target }}.tar.gz
 
+  cli-macos:
+    strategy:
+      matrix:
+        include:
+          - os: macos-13
+            target: x86_64-apple-darwin
+          - os: macos-14
+            # we cannot cross-compile, because we need also all dependencies for the specified target
+            target: aarch64-apple-darwin
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Compile binary
+        run: |
+          brew tap davidchall/hep
+          # install LHAPDF
+          brew install lhapdf
+          # install APPLgrid's dependencies; disable ROOT because its static libraries are missing
+          brew install --only-dependencies --without-hoppet --without-lhapdf --without-root applgrid
+          # install zlib, which is a dependency of APPLgrid but somehow missing
+          brew install zlib
+          # APPLgrid wants to be linked against zlib, and we need to find its static library via pkg-config
+          export PKG_CONFIG_PATH=$(find $(brew --cellar) -name '*.pc' -exec dirname {} + | sort -u | tr '\n' ':')
+          # manually compile APPLgrid, because we need the file `appl_igrid.h` and the files it includes, which are possibly generated
+          export HOMEBREW_TEMP="$(pwd)"/tmp
+          mkdir -p ${HOMEBREW_TEMP}
+          brew install --build-from-source --keep-tmp --without-hoppet --without-lhapdf --without-root applgrid
+          export APPL_IGRID_DIR=$(find ${HOMEBREW_TEMP} -name appl_igrid.h -exec dirname {} +)
+          # do not link against `gfortran`
+          sed 's/-lgfortran//g' $(command -v applgrid-config) > applgrid-config.new
+          mv applgrid-config.new $(command -v applgrid-config)
+          chmod +x $(command -v applgrid-config)
+          # install fastNLO
+          brew install --without-fastjet fastnlo
+          # install Rust
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-host=${{ matrix.target }}
+          # build the CLI
+          cargo install --all-features --locked --path pineappl_cli  --root=prefix --target=${{ matrix.target }}
+          # build manpages
+          mkdir -p prefix/share/man/man1
+          cargo xtask install-manpages prefix/share/man/man1
+          cd prefix
+          tar czf ../pineappl_cli-${{ matrix.target }}.tar.gz .
+      - name: Upload artifact
+        # as long as we need v3 in `cli-linux` we also must use it here
+        uses: actions/upload-artifact@v3
+        with:
+          name: pineappl_cli-${{ matrix.target }}
+          path: pineappl_cli-${{ matrix.target }}.tar.gz
+
   capi-linux:
     runs-on: ubuntu-latest
     container: ghcr.io/nnpdf/pineappl-ci:latest
@@ -55,7 +105,7 @@ jobs:
       - name: Compile library
         run: |
           cd pineappl_capi
-          cargo cinstall --verbose --destdir=prefix --library-type=cdylib --prefix=/ --target=${{ matrix.target }}
+          cargo cinstall --destdir=prefix --library-type=cdylib --locked --prefix=/ --target=${{ matrix.target }} --verbose
           cd prefix
           tar czf ../../pineappl_capi-${{ matrix.target }}.tar.gz .
           # print the glibc version requirement
@@ -67,13 +117,40 @@ jobs:
           name: pineappl_capi-${{ matrix.target }}
           path: pineappl_capi-${{ matrix.target }}.tar.gz
 
+  cli-linux:
+    runs-on: ubuntu-latest
+    container: ghcr.io/nnpdf/pineappl-ci:latest
+    strategy:
+      matrix:
+        target: [x86_64-unknown-linux-gnu]
+    steps:
+      # checkout@v4 uses a newer version of Node that's incompatible with our container's GLIBC
+      - uses: actions/checkout@v3
+      - name: Compile binary
+        run: |
+          cargo install --all-features --locked --path pineappl_cli --root=prefix --target=${{ matrix.target }} -vv
+          # build manpages
+          mkdir -p prefix/share/man/man1
+          cargo xtask install-manpages prefix/share/man/man1
+          cd prefix
+          tar czf ../pineappl_cli-${{ matrix.target }}.tar.gz .
+          # print the dynamically linked libraries
+          ldd bin/pineappl
+      - name: Upload artifact
+        # upload-artifact@v4 uses a newer version of Node that's incompatible with our container's GLIBC
+        uses: actions/upload-artifact@v3
+        with:
+          name: pineappl_cli-${{ matrix.target }}
+          path: pineappl_cli-${{ matrix.target }}.tar.gz
+
   # publish a release on github and upload pre-built CAPI
   publish-release:
-    needs: [capi-macos, capi-linux]
+    needs: [capi-macos, cli-macos, capi-linux, cli-linux]
     runs-on: ubuntu-latest
     if: "startsWith(github.ref, 'refs/tags/')"
     steps:
       - uses: actions/checkout@v4
+      # version must match the one used in `actions/upload-artifact`
       - uses: actions/download-artifact@v3
         with:
           path: artifacts
@@ -93,7 +170,7 @@ jobs:
           else
               gh release edit v${version} -n "" --prerelease
           fi
-          find artifacts -name 'pineappl_capi*' -type f -exec gh release upload v${version} {} +
+          find artifacts -name 'pineappl*' -type f -exec gh release upload v${version} {} +
           gh release edit v${version} --draft=false
 
   publish-crates:
@@ -144,48 +221,8 @@ jobs:
           name: ${{ github.job }}-${{ matrix.target }}
           path: dist
 
-  wheels-eol-linux:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        target: [x86_64]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Patch Cargo.toml
-        run: |
-          # - older maturin versions don't support metadata inheritance - overwrite it explicitly
-          # - use '@' separator since 'repository' contains slashes
-          # - downgrade both `numpy` and `pyo3` to the last version that support Python 3.6
-          sed -i \
-              -e "[email protected] = true@$(grep '^categories' Cargo.toml)@" \
-              -e "[email protected] = true@$(grep '^edition' Cargo.toml)@" \
-              -e "[email protected] = true@$(grep '^keywords' Cargo.toml)@" \
-              -e "[email protected] = true@$(grep '^license' Cargo.toml)@" \
-              -e "[email protected] = true@$(grep '^repository' Cargo.toml)@" \
-              -e "[email protected] = true@$(grep '^rust-version' Cargo.toml)@" \
-              -e "[email protected] = true@$(grep '^version' Cargo.toml)@" \
-              -e "s@^numpy = .*@numpy = \"0.14.1\"@" \
-              -e "s@^pyo3 = .*@pyo3 = { features = [\"extension-module\"], version = \"0.14.5\" }@" \
-              pineappl_py/Cargo.toml
-          # the previous command is very fragile, so print the file contents to check it
-          cat pineappl_py/Cargo.toml
-      - name: Build wheels
-        uses: PyO3/maturin-action@v1
-        with:
-          target: ${{ matrix.target }}
-          args: --release -i python3.6 --out dist --no-sdist --manifest-path pineappl_py/Cargo.toml
-          sccache: 'true'
-          manylinux: auto
-          # this is the latest version to support Python 3.6
-          maturin-version: 0.12.20
-      - name: Upload wheels
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ github.job }}-${{ matrix.target }}
-          path: dist
-
   wheels-macos:
-    runs-on: macos-latest
+    runs-on: macos-13
     strategy:
       matrix:
         target: [aarch64, x86_64]
@@ -271,10 +308,9 @@ jobs:
           path: dist
 
   release-wheels:
-    name: Release
     runs-on: ubuntu-latest
     if: "startsWith(github.ref, 'refs/tags/')"
-    needs: [wheels-eol-linux, wheels-linux, wheels-macos, wheels-sdist, wheels-windows]
+    needs: [wheels-linux, wheels-macos, wheels-sdist, wheels-windows]
     steps:
       - uses: actions/download-artifact@v4
         with:

.github/workflows/rust.yml

@@ -9,7 +9,6 @@ defaults:
 
 env:
   CARGO_TERM_COLOR: always
-  RUSTFLAGS: '-Cinstrument-coverage'
 
 jobs:
   build:
@@ -61,9 +60,11 @@ jobs:
           echo "RUSTDOCFLAGS=-Cinstrument-coverage -Z unstable-options --persist-doctests $(pwd)/target/debug/doctestbins" >> "$GITHUB_ENV"
 
       - name: Run tests
+        env:
+          RUSTFLAGS: '-Cinstrument-coverage'
         run: |
           # we need stderr, but we can't run test twice because it'll regenerate/modify the binaries which interferes with `llvm-cov`
-          cargo test --all-features --no-fail-fast 2> >(tee stderr 1>&2)
+          cargo test --features=applgrid,evolve,fastnlo,fktable --no-fail-fast 2> >(tee stderr 1>&2)
           # from https://stackoverflow.com/a/51141872/812178
           sed -i 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' stderr
 
@@ -73,11 +74,12 @@ jobs:
           ( sed -nE 's/[[:space:]]+Running( unittests|) [^[:space:]]+ \(([^)]+)\)/\2/p' stderr && echo target/debug/doctestbins/*/rust_out | tr ' ' "\n" ) | \
               xargs printf ' --object %s' | \
               xargs $(rustc --print target-libdir)/../bin/llvm-cov export \
-                  --ignore-filename-regex='index.crates.io' \
-                  --ignore-filename-regex='rustc' \
-                  --ignore-filename-regex='pineappl/tests' \
-                  --ignore-filename-regex='pineappl_capi' \
-                  --ignore-filename-regex='pineappl_cli/tests' \
+                  --ignore-filename-regex=index.crates.io \
+                  --ignore-filename-regex=rustc \
+                  --ignore-filename-regex=managed-lhapdf \
+                  --ignore-filename-regex=pineappl/tests \
+                  --ignore-filename-regex=pineappl_capi \
+                  --ignore-filename-regex=pineappl_cli/tests \
                   --instr-profile=pineappl.profdata \
                   --skip-functions \
                   --object target/debug/pineappl \

.gitignore

@@ -2,3 +2,5 @@
 
 # Devenv files, to handle the development shell defined by the flake
 .devenv/
+
+test-data/*

CHANGELOG.md

@@ -9,18 +9,94 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- added new type `Convolution`
+- added new methods `Grid::convolutions` and `Grid::set_convolution`
+- added the function `pineappl_grid_convolve_with_one` and
+  `pineappl_grid_convolve_with_two` which replace the deprecated function
+  similarly named with `convolute` in CAPI
+- added `PidBasis::charge_conjugate` and `PidBasis::guess`
+- added `Grid::set_pid_basis` method
+- added `Grid::subgrids` and `Grid::subgrids_mut` methods
+- added new switch `conv_fun_uncert_from` to subcommand `plot` to allow
+  choosing with convolution function uncertainty should be plotted
+
+### Changed
+
+- moved `Order` and `ParseOrderError` to the new module `boc`
+- renamed switch `--split-lumi` of `pineappl write` to `--split-channels`. The
+  old switch can still be used
+- renamed switch `--lumis` of `pineappl read` to `--channels`. The old switch
+  can still be used
+- renamed switch `--ignore-lumis` of `pineappl diff` to `--ignore-channels`.
+  The old switch can still be used
+- renamed `Grid::lumi` to `Grid::channels`, `Grid::split_lumi` to
+  `Grid::split_channels`, `Grid::rewrite_lumi` to `Grid::rewrite_channels` and
+  `Grid::set_lumis` to `Grid::set_channels`. The term 'channel' is now used
+  everywhere instead of 'lumi', 'luminosity function', etc.
+- renamed the struct `LumiEntry` to `Channel` and `ParseLumiEntryError` to
+  `ParseChannelError`. Both structures have been moved to the module `boc`
+- renamed the macro `lumi_entry` to `channel`
+- renamed `Grid::set_channels` to `Grid::channels_mut`
+- renamed `TryFromGridError::InvalidLumi` to `TryFromGridError::InvalidChannel`
+- changed member `lumi_id_types` of `OperatorInfo` and `OperatorSliceInfo` to
+  `pid_basis`, which is now of type `PidBasis`
+- renamed module `pineappl::lumi` to `pineappl::convolutions`
+- renamed switch `--pdf` to `--conv-fun` in the subcommand `uncert`. This
+  switch now optionally accepts a list of indices, which determines the
+  corresponding convolution function (PDF/FF), for which the uncertainty should
+  calculated
+- renamed `no_pdf_unc` to `no_conv_fun_unc` in subcommand `plot`
+
+### Removed
+
+- removed support for Python 3.6
+- removed the deprecated evolution methods `Grid::axes`, `Grid::convolute_eko`
+  and the structs `EkoInfo` and `GridAxes`
+- removed methods `Grid::has_pdf1`, `Grid::has_pdf2`, `Grid::initial_state_1`
+  and `Grid::initial_state_2`
+- removed `pids::charge_conjugate`; this function has been replaced with the
+  new function `PidBasis::charge_conjugate`
+- removed `pids::determine_lumi_id_types`; this function has been replaced with
+  the new function `PidBasis::guess`
+- removed `TryFromGridError::MetadataMissing`
+- removed `Grid::subgrid` and `Grid::set_subgrid` methods; these functions have
+  been replaced with `Grid::subgrids` and `Grid::subgrids_mut`
+- removed the switch `--pdf-with-scale-cov` from `pineappl uncert`
+
+## [0.7.4] - 23/05/2024
+
+### Added
+
 - added `Grid::evolve_with_slice_iter`, `AlphasTable` and `OperatorSliceInfo`,
   which define a new interface supporting very large evolution kernels that
   have been introduced in EKO v0.13. This interface will replace `Grid::evolve`
+- added `--dont-sort` switch to `pineappl channels`, which displays the channel
+  sizes ordered by channel index (instead of channel size)
+- added `Grid::rotate_pid_basis` and `pineappl write --rotate-pid-basis`. This
+  allows to change the meaning of the used particle IDs, and supported formats
+  are PDG MC IDs and the evolution basis
+- added `pineappl write --rewrite-order` that lets the user change the
+  exponents of each order
 
 ### Changed
 
+- changed the official name of the CLI subcommand `convolute` to `convolve`,
+  because the latter is the proper verb of 'convolution'. The old name
+  `convolute` is now an alias of `convolve`, which means both can be used. The
+  methods `Grid::convolute*` are left unchanged and will be renamed in later
+  version
+- changed switch `--silence-lhapdf` to `--lhapdf-banner` and suppress LHAPDF's
+  banners by default, unless `--lhapdf-banner` is given
 - `Grid::evolve` has now been marked deprecated
+- switched from `lhapdf` to `managed-lhapdf` crate which automatically
+  downloads PDF sets when they are needed
 
 ### Fixed
 
 - fixed yet another problem that prevent the Python interface for Python 3.6
   from being successfully installed
+- fixed `Grid::delete_channels` and its CLI variant `pineappl write
+  --delete-channels`. This command wasn't working properly before
 
 ## [0.7.3] - 23/02/2024
 
@@ -545,7 +621,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - first release
 
-[Unreleased]: https://github.com/NNPDF/pineappl/compare/v0.7.3...HEAD
+[Unreleased]: https://github.com/NNPDF/pineappl/compare/v0.7.4...HEAD
+[0.7.4]: https://github.com/NNPDF/pineappl/compare/v0.7.3...v0.7.4
 [0.7.3]: https://github.com/NNPDF/pineappl/compare/v0.7.2...v0.7.3
 [0.7.2]: https://github.com/NNPDF/pineappl/compare/v0.7.1...v0.7.2
 [0.7.1]: https://github.com/NNPDF/pineappl/compare/v0.7.0...v0.7.1