Merge from master and resolve conflicts

NNPDF · Dec 6, 2024 · 7c8577a · 7c8577a
2 parents 1f2b922 + 13fda2a
commit 7c8577a
Show file tree

Hide file tree

Showing 921 changed files with 129,897 additions and 30,521 deletions.
diff --git a/.github/workflows/fitbot.yml b/.github/workflows/fitbot.yml
@@ -10,57 +10,36 @@ on:
 env:
   N3FIT_MAXNREP: 20 # total number of replicas to fit
   POSTFIT_NREP: 16 # requested replicas for postfit
-  REFERENCE_SET: NNBOT-73468bfff-2024-11-01 # reference set for exact results
+  REFERENCE_SET: NNBOT-b1374cba6-2024-11-28 # reference set for exact results
   STABLE_REFERENCE_SET: NNBOT-c0f99b7b3-2024-02-28 # reference set for last tag
-  CONDA_PY: 312
   PYTHONHASHSEED: "0"
 
 jobs:
-  build:
+  fitbot:
     # perform build only if PR has run-fit-bot label
     if: contains(github.event.pull_request.labels.*.name, 'run-fit-bot')
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.12"]
-    runs-on: ${{ matrix.os }}
-    env:
-      NETRC_FILE: ${{ secrets.NETRC_FILE }}
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
       with:
         fetch-tags: true
         fetch-depth: 0
     - uses: conda-incubator/setup-miniconda@v3
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: "3.12"
         use-mamba: true
-        channels: https://packages.nnpdf.science/public,conda-forge
+        channels: conda-forge
         show-channel-urls: true
         auto-update-conda: true
-    - name: Install SSH key
-      uses: shimataro/ssh-key-action@v2
-      with:
-        key: ${{ secrets.NNPDF_SSA }}
-        name: id_rsa
-        known_hosts: ${{ secrets.NNPDF_HOSTS_KEY }}
-    - name: Setup conda and install conda-build
-      shell: bash -l {0}
-      run: |
-        echo "$NETRC_FILE" | base64 --decode > ~/.netrc
-        conda install conda-build --yes
-    - name: Build recipe
+        activate-environment: nnpdfenv
+    - name: Install lhapdf, pandoc and mongodb
       shell: bash -l {0}
       run: |
-        CONDA_PY=$CONDA_PY conda build --no-test -q conda-recipe
-    # install local build
-    - name: Installing NNPDF conda package
+        conda install lhapdf pandoc mongodb
+    - name: Install nnpdf with pip
       shell: bash -l {0}
       run: |
-        conda update -n base conda
-        conda install -n base conda-libmamba-solver
-        conda config --set solver libmamba
-        conda create -n nnpdfenv -c file:///usr/share/miniconda/envs/test/conda-bld/linux-64/ nnpdf
+        pip install .
     # create fancy runcards
     - name: Preparing fit runcard
       run: |
@@ -69,29 +48,32 @@ jobs:
         export TIMESTAMP=`date --iso-8601`
         echo "RUNCARD=NNBOT-$COMMIT-$TIMESTAMP" >> $GITHUB_ENV
         echo "RUNFOLDER=n3fit/runcards/examples" >> $GITHUB_ENV
-    # downloading theory and t0
+    # downloading theory, t0 and eko
     - name: Downloading requirements
       shell: bash -l {0}
       run: |
-        conda activate nnpdfenv
         cd $RUNFOLDER
         cp developing.yml $RUNCARD.yml
         vp-setupfit $RUNCARD.yml
-    # running n3fit replicas
+    # run n3fit replicas sequentially
     - name: Running n3fit
       shell: bash -l {0}
       run: |
-        conda activate nnpdfenv
         cd $RUNFOLDER
         for ((i=1; i<=$N3FIT_MAXNREP; i+=1)); do n3fit $RUNCARD.yml $i ; done
     # performing DGLAP
     - name: Running dglap
       shell: bash -l {0}
       run: |
-        conda activate nnpdfenv
         cd $RUNFOLDER
         evolven3fit evolve $RUNCARD
-    # running postfit
+    # running postfit and upload the fit to the server
+    - name: Install SSH key
+      uses: shimataro/ssh-key-action@v2
+      with:
+        key: ${{ secrets.NNPDF_SSA }}
+        name: id_rsa
+        known_hosts: ${{ secrets.NNPDF_HOSTS_KEY }}
     - name: Postfit and upload fit
       shell: bash -l {0}
       run: |

diff --git a/.github/workflows/python_installation.yml b/.github/workflows/python_installation.yml
@@ -43,4 +43,4 @@ jobs:
       shell: bash -l {0}
       run: |
         conda activate test
-        pytest --pyargs --mpl validphys n3fit
+        pytest --pyargs --mpl validphys n3fit --mpl-default-tolerance 18
diff --git a/.github/workflows/redo_regressions.yml b/.github/workflows/redo_regressions.yml
@@ -13,15 +13,7 @@ env:
 jobs:
   build:
     if: contains(github.event.pull_request.labels.*.name, 'redo-regressions')
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.11"]
-        include:
-          - os: ubuntu-latest
-            CONDA_OS: linux-64
-      fail-fast: false
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     env:
       NETRC_FILE: ${{ secrets.NETRC_FILE }}
       NNPDF_SSH_KEY: ${{ secrets.NNPDF_SSH_KEY }}
@@ -30,10 +22,9 @@ jobs:
       with:
         fetch-depth: 0
         ref: ${{ github.event.pull_request.head.ref }}
-
     - uses: conda-incubator/setup-miniconda@v3
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: "3.12"
         use-mamba: true
         auto-update-conda: true
         activate-environment: test

diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml
@@ -4,27 +4,20 @@ on: [push]
 
 env:
   # https://keras.io/getting_started/faq/#how-can-i-obtain-reproducible-results-using-keras-during-development
+  # However, results might change between different versions of keras.
   PYTHONHASHSEED: "0"
 
 jobs:
   build:
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.11"]
-        include:
-          - os: ubuntu-latest
-            CONDA_OS: linux-64
-      fail-fast: false
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     env:
       NETRC_FILE: ${{ secrets.NETRC_FILE }}
       NNPDF_SSH_KEY: ${{ secrets.NNPDF_SSH_KEY }}
     steps:
     - uses: actions/checkout@v4
     - uses: conda-incubator/setup-miniconda@v3
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: "3.12"
         use-mamba: true
         auto-update-conda: true
         activate-environment: test

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -12,10 +12,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-14]
-        python-version: ["3.12"] # Test only the latest python in all system
-        include:
-          - os: ubuntu-latest # Test also the oldest python supported in ubuntu
-            python-version: "3.9"
+        python-version: ["3.10"] # We need an older python version to avoid conflict with the pymongo pin
       fail-fast: false
     runs-on: ${{ matrix.os }}
     env:
@@ -45,7 +42,7 @@ jobs:
       run: |
         conda build -q conda-recipe
     - name: Upload noarch conda package to NNPDF server
-      if: ${{ (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && startsWith(matrix.os, 'ubuntu') && matrix.python-version == '3.9' }}
+      if: ${{ (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && startsWith(matrix.os, 'ubuntu') }}
       shell: bash -l {0}
       run: |
         KEY=$( mktemp )
@@ -54,7 +51,7 @@ jobs:
           $CONDA_PREFIX/conda-bld/noarch/*.tar.bz2 \
           [email protected]:~/packages/conda/noarch
     - name: Build and upload sphinx documentation to NNPDF server
-      if: startsWith(matrix.os, 'ubuntu') && github.ref == 'refs/heads/master' && matrix.python-version == '3.9'
+      if: startsWith(matrix.os, 'ubuntu') && github.ref == 'refs/heads/master'
       shell: bash -l {0}
       run: |
         KEY=$( mktemp )

diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
@@ -20,16 +20,17 @@ requirements:
     run:
         - python >=3.9,<3.13
         - tensorflow >=2.10,<2.17 # 2.17 works ok but the conda-forge package for macos doesn't
-        - psutil # to ensure n3fit affinity is with the right processors
+        - psutil
         - hyperopt
         - mongodb
         - pymongo <4
+        - pyopenssl >=23.2
         - seaborn
         - lhapdf
         - numpy
         - pkg-config
         - reportengine
-        - matplotlib >=3.3.0,<3.8  # see https://github.com/NNPDF/nnpdf/pull/1809
+        - matplotlib >=3.9
         - blessings >=1.7
         - scipy >=0.19.1
         - pandas
@@ -43,7 +44,7 @@ requirements:
         - joblib
         - sphinx_rtd_theme >0.5
         - sphinxcontrib-bibtex
-        - ruamel.yaml <0.18
+        - ruamel.yaml >=0.15
 
 test:
     requires:

diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh
@@ -3,8 +3,9 @@ set -u
 set -v
 set -e
 
-#Python tests for the installed validphys package
-pytest --pyargs --mpl validphys
+# Python tests for the installed validphys package
+# Note that the default tolerance in the conda test is higher than the pip test
+pytest --pyargs --mpl validphys --mpl-default-tolerance 22
 
 platformstr=`uname`
 

diff --git a/doc/sphinx/source/tutorials/general_th_covmat.rst b/doc/sphinx/source/tutorials/general_th_covmat.rst
@@ -41,8 +41,6 @@ Instructions
 
     ############################################################################
     theorycovmatconfig:
-      use_scalevar_uncertainties: False
-      use_user_uncertainties: True
       user_covmat_path: "IeGM9CY8RxGcb5r6bIEYlQ==/topthcovmat.csv"
       use_thcovmat_in_sampling: True
       use_thcovmat_in_fitting: True
@@ -57,16 +55,10 @@ generation). The default is ``True`` for both.
       Changing either of these to ``False`` will affect the fit outcome and should
       be avoided unless you know what you are doing.
 
-
-4. The flag ``use_scalevar_uncertainties``  corresponds to whether or not a
-   theory covmat is to be constructed from a pointprescription with instructions
-   inside ``theorycovmatconfig``. Usually in cases where a custom covmat is
-   being loaded, this will be set to ``False``.
-
-5. Make sure that datasets are grouped under one big experiment called "BIGEXP",
+1. Make sure that datasets are grouped under one big experiment called "BIGEXP",
    just like in :ref:`vptheorycov-index`.
 
-6. For an example runcard, see `here <https://github.com/NNPDF/nnpdf/tree/master/validphys2/examples/fit_with_user_covmat.yaml.>`_
+2. For an example runcard, see `here <https://github.com/NNPDF/nnpdf/tree/master/validphys2/examples/fit_with_user_covmat.yaml.>`_
 
 
 Including both scale variation uncertainties and user uncertainties

diff --git a/extra_tests/regression_checks.py b/extra_tests/regression_checks.py
@@ -9,7 +9,7 @@
 import pytest
 
 from n3fit.tests.test_fit import EXE, check_fit_results
-from reportengine.compat import yaml
+from validphys.utils import yaml_safe
 
 REGRESSION_FOLDER = pathlib.Path(__file__).with_name("regression_fits")
 
@@ -27,7 +27,6 @@
     "no_lagrange": 27,
     "no_csr": 613,
     "polarized_evol": 34,
-    "multi_dense": 316,
     "t0theoryid": 100,
 }
 
@@ -38,7 +37,7 @@ def test_regression_fit(tmp_path, runcard, replica, regenerate):
     runcard_file = REGRESSION_FOLDER / runcard_name
     shutil.copy(runcard_file, tmp_path)
 
-    runcard_info = yaml.load(runcard_file.read_text())
+    runcard_info = yaml_safe.load(runcard_file.read_text())
     if (wname := runcard_info.get("load")) is not None:
         shutil.copy(REGRESSION_FOLDER / wname, tmp_path)