Merge branch 'gwastro:master' into add_duration

.github/workflows/basic-tests.yml

@@ -16,9 +16,9 @@ jobs:
         python-version: ['3.9', '3.10', '3.11', '3.12']
         test-type: [unittest, search, docs]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: installing system packages
@@ -51,7 +51,7 @@ jobs:
         tox -e py-inference
     - name: store documentation page
       if: matrix.test-type == 'docs' && matrix.python-version == '3.12'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: documentation-page
         path: _gh-pages
@@ -61,7 +61,7 @@ jobs:
     if: github.ref == 'refs/heads/master' && github.event_name == 'push'
     steps:
     - name: retrieve built documentation
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v4.1.7
       with:
         name: documentation-page
     - name: debug

.github/workflows/build_venv.yml

@@ -14,7 +14,7 @@ jobs:
         uses: actions/checkout@v1
       -
         name: "Set up Python"
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
       -

.github/workflows/distribution.yml

@@ -15,10 +15,10 @@ jobs:
         os: [ubuntu-20.04, macos-12]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Install cibuildwheel
@@ -29,29 +29,33 @@ jobs:
           CIBW_BUILD: cp39-* cp310-* cp311-* cp312-*
           CIBW_SKIP: "*musllinux*"
           CIBW_ARCHS_MACOS: x86_64 arm64
-      - uses: actions/upload-artifact@v2
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
         with:
+          name: wheel-${{ matrix.os }}
           path: ./wheelhouse/*.whl
   deploy_pypi:
-    name: Build and publish Python 🐍 distributions 📦 to PyPI
+    name: Package and publish to PyPI
     runs-on: ubuntu-20.04
     needs: build_wheels
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
-    - uses: actions/download-artifact@v2
+    - uses: actions/download-artifact@v4.1.7
       with:
-        path: ./
-    - name: build pycbc for pypi
+        pattern: wheel-*
+        merge-multiple: true
+        path: dist/
+    - name: Build source distribution
       run: |
         python setup.py sdist
-        mv artifact/* dist/
-    - name: Publish distribution 📦 to PyPI
+        ls -lh dist
+    - name: Publish to PyPI
       if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
       uses: pypa/gh-action-pypi-publish@master
       with:

.github/workflows/docker-build.yml

@@ -14,7 +14,7 @@ jobs:
         uses: actions/checkout@v1
       -
         name: "Set up Python"
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
       -

.github/workflows/inference-workflow.yml

@@ -8,7 +8,7 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
     - name: install condor
@@ -25,7 +25,7 @@ jobs:
         wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add -
         echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -o Acquire::Retries=3 update
-        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18
+        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.8-1+ubuntu18
     - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl*
     - name: Install pycbc
       run: |
@@ -48,12 +48,12 @@ jobs:
         find submitdir/work/ -type f -name '*.tar.gz' -delete
     - name: store log files
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: logs
         path: gw_output/submitdir/work
     - name: store result page
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: results
         path: html

.github/workflows/mac-test.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - run: |

.github/workflows/search-workflow.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
     - name: install condor
@@ -30,7 +30,7 @@ jobs:
         wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add -
         echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -o Acquire::Retries=3 update
-        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18
+        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.8-1+ubuntu18
     - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl*
     - name: Install pycbc
       run: |
@@ -55,12 +55,12 @@ jobs:
         find submitdir/work/ -type f -name '*.tar.gz' -delete
     - name: store log files
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: logs
         path: output/submitdir/work
     - name: store result page
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: results
         path: html

.github/workflows/tmpltbank-workflow.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
     - name: install condor
@@ -29,7 +29,7 @@ jobs:
         wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add -
         echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -o Acquire::Retries=3 update
-        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18
+        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.8-1+ubuntu18
     - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl*
     - name: Install pycbc
       run: |
@@ -51,7 +51,7 @@ jobs:
         find submitdir/work/ -type f -name '*.tar.gz' -delete
     - name: store log files
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: logs
         path: output/submitdir/work

.github/workflows/tut-test.yml

@@ -15,9 +15,9 @@ jobs:
         os: [ubuntu-20.04]
         python-version: ['3.10', '3.11', '3.12']
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: installing packages

.github/workflows/workflow-tests.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
     - name: install condor
@@ -34,7 +34,7 @@ jobs:
         wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add -
         echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -o Acquire::Retries=3 update
-        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18
+        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.8-1+ubuntu18
     - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl*
     - name: Install pycbc
       run: |
@@ -50,7 +50,7 @@ jobs:
         find submitdir/work/ -type f -name '*.tar.gz' -delete
     - name: store log files
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: logs-${{matrix.test-type}}
         path: examples/workflow/generic/${{matrix.test-type}}/submitdir/work

bin/all_sky_search/pycbc_bin_templates

@@ -48,7 +48,8 @@ with h5.File(args.output_file, 'w') as f:
         bin_tids = bin_dict[bin_name]
         grp = ifo_grp.create_group(bin_name)
         grp['tids'] = bin_tids
+    f.attrs['bank_file'] = args.bank_file
     f.attrs['f_lower'] = args.f_lower
-    f.attrs['background_bins'] = args.background_bins
+    f.attrs['background_bins'] = ' '.join(args.background_bins)
 
 logging.info('Finished')

bin/all_sky_search/pycbc_calculate_psd

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 """ Calculate psd estimates for analysis segments
 """
-import logging, argparse, numpy, multiprocessing, time, copy
+import logging, argparse, numpy, time, copy
 from six.moves import zip_longest
 import pycbc, pycbc.psd, pycbc.strain, pycbc.events
 from pycbc.io import HFile
+from pycbc.pool import BroadcastPool as Pool
 from pycbc.fft.fftw import set_measure_level
 from pycbc.workflow import resolve_td_option
 from ligo.segments import segmentlist, segment
@@ -101,7 +102,7 @@ segments = segmentlist(frozenset(segments))
 logging.info('%d psds to calculate', len(segments))
 
 if len(segments) > 0:
-    pool = multiprocessing.Pool(args.cores)
+    pool = Pool(args.cores)
     psds = pool.map_async(get_psd, zip(segments, range(len(segments))))
     psds = psds.get()
 else:

bin/all_sky_search/pycbc_coinc_statmap

@@ -291,9 +291,9 @@ if fore_locs.sum() > 0:
     f['foreground/ifar_exc'] = conv.sec_to_year(ifar_exc)
     f['foreground/fap_exc'] = fap_exc
 else:
-    f['foreground/ifar'] = numpy.array([])
+    f['foreground/ifar'] = ifar = numpy.array([])
     f['foreground/fap'] = numpy.array([])
-    f['foreground/ifar_exc'] = numpy.array([])
+    f['foreground/ifar_exc'] = ifar_exc = numpy.array([])
     f['foreground/fap_exc'] = numpy.array([])
 
 if 'name' in all_trigs.attrs:
@@ -331,6 +331,10 @@ else :
 #          will happen when ifar_foreground has no elements greater than
 #          the background time.
 
+# This is a workaround for the case that we have no foreground, so that the
+# while loop comparison doesnt fail straight away
+ifar_foreground = numpy.append(ifar_foreground, 0)
+
 while numpy.any(ifar_foreground >= background_time):
     # If the user wants to stop doing hierarchical removals after a set
     # number of iterations then break when that happens.
@@ -508,9 +512,9 @@ f.attrs['hierarchical_removal_iterations'] = h_iterations
 
 # Write whether hierarchical removals were removed against the
 # inclusive background or the exclusive background. Have to use
-# numpy.string_ datatype.
+# numpy.bytes_ datatype.
 if h_iterations != 0:
     hrm_method = args.hierarchical_removal_against
-    f.attrs['hierarchical_removal_method'] = numpy.string_(hrm_method)
+    f.attrs['hierarchical_removal_method'] = numpy.bytes_(hrm_method)
 
 logging.info("Done")

bin/all_sky_search/pycbc_sngls_statmap

@@ -436,10 +436,10 @@ f.attrs['hierarchical_removal_iterations'] = h_iterations
 
 # Write whether hierarchical removals were removed against the
 # inclusive background or the exclusive background. Have to use
-# numpy.string_ datatype.
+# numpy.bytes_ datatype.
 if h_iterations != 0:
     hrm_method = args.hierarchical_removal_against
-    f.attrs['hierarchical_removal_method'] = numpy.string_(hrm_method)
+    f.attrs['hierarchical_removal_method'] = numpy.bytes_(hrm_method)
 
 logging.info("Done")
 

bin/inference/pycbc_inference

@@ -108,7 +108,11 @@ fft.from_cli(opts)
 with ctx:
 
     # read configuration file
-    cp = configuration.WorkflowConfigParser.from_cli(opts)
+    cp_original = configuration.WorkflowConfigParser.from_cli(opts)
+    # some models will interally modify original cp for sampling,
+    # such as joint_primary_marginalized, we need to save original
+    # and let modify the copied one
+    cp = cp_original.__deepcopy__(cp_original)
 
     # create an empty checkpoint file, if needed
     condor_ckpt = cp.has_option('sampler', 'checkpoint-signal')
@@ -138,7 +142,7 @@ with ctx:
     if pool.is_main_process():
         for fn in [sampler.checkpoint_file, sampler.backup_file]:
             with loadfile(fn, 'a') as fp:
-                fp.write_config_file(cp)
+                fp.write_config_file(cp_original)
 
     # Run the sampler
     sampler.run()

bin/inference/pycbc_inference_plot_movie

@@ -43,7 +43,7 @@ import logging
 import subprocess
 import os
 import glob
-from multiprocessing import Pool
+from pycbc.pool import BroadcastPool as Pool
 
 import numpy
 

bin/live/pycbc_live_collate_triggers

@@ -231,14 +231,15 @@ with h5py.File(args.output_file, 'w') as destination:
                 sorted_key = triggers[key][:][sorted_indices]
                 triggers[key][:] = sorted_key
 
-
         logging.info("Setting up region references")
         # Datasets which need region references:
         region_ref_datasets = ('chisq_dof', 'chisq', 'coa_phase',
                                'end_time', 'sg_chisq', 'snr',
                                'template_duration', 'sigmasq')
         if 'psd_var_val' in triggers.keys():
             region_ref_datasets += ('psd_var_val',)
+        if 'dq_state' in triggers.keys():
+            region_ref_datasets += ('dq_state',)
 
         start_boundaries = template_boundaries
         end_boundaries = numpy.roll(start_boundaries, -1)