From 36b9370412554cfa8f28735e380ebcfbd1b3688f Mon Sep 17 00:00:00 2001 From: "Benedikt J. Daurer" Date: Thu, 21 Mar 2024 15:23:33 +0000 Subject: [PATCH 1/7] Wrap nccl.get_unique_id in try/except (#549) --- ptypy/accelerate/cuda_cupy/multi_gpu.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ptypy/accelerate/cuda_cupy/multi_gpu.py b/ptypy/accelerate/cuda_cupy/multi_gpu.py index 79f511423..6ccbce9a7 100644 --- a/ptypy/accelerate/cuda_cupy/multi_gpu.py +++ b/ptypy/accelerate/cuda_cupy/multi_gpu.py @@ -98,7 +98,10 @@ def __init__(self): # get a unique identifier for the NCCL communicator and # broadcast it to all MPI processes (assuming one device per process) if self.rank == 0: - self.id = nccl.get_unique_id() + try: + self.id = nccl.get_unique_id() + except: + self.id = None else: self.id = None From 9a270a08d8a587cb1b83b9f0bfe6c8b4a0f54d79 Mon Sep 17 00:00:00 2001 From: kahntm Date: Tue, 7 Jan 2025 17:21:15 +0100 Subject: [PATCH 2/7] probe from numpy array will not be rescaled to photon count --- ptypy/core/illumination.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ptypy/core/illumination.py b/ptypy/core/illumination.py index ddf1f98f7..93320d2b8 100644 --- a/ptypy/core/illumination.py +++ b/ptypy/core/illumination.py @@ -366,6 +366,7 @@ def init_storage(storage, pars, energy=None, **kwargs): model *= np.sqrt(p.photons) / np.prod(s.shape) elif type(p.model) is np.ndarray: model = p.model + p.photons = None elif p.model in resources.probes: model = resources.probes[p.model](s.shape) elif str(p.model) == 'recon': From fa2a7fa2dc9d7c6abb6d4417d7472e926c18826b Mon Sep 17 00:00:00 2001 From: Maik Kahnt Date: Wed, 8 Jan 2025 08:35:36 +0100 Subject: [PATCH 3/7] unify illumination.photons functionality for model, ptyr and np.array probe inits --- ptypy/core/illumination.py | 17 +++++++++-------- ptypy/core/manager.py | 24 ++++++++++++++++-------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/ptypy/core/illumination.py b/ptypy/core/illumination.py index 93320d2b8..c714a5976 100644 --- a/ptypy/core/illumination.py +++ b/ptypy/core/illumination.py @@ -134,11 +134,13 @@ userlevel = 0 [photons] - type = int, float, None - default = None + type = int, float, str, None + default = 'maxdiff' help = Number of photons in the incident illumination - doc = A value specified here will take precedence over calculated statistics from the loaded data. - lowlim = 0 + doc = A value specified here will take precedence over calculated statistics from the loaded data. Choices: + - ``None`` : modeled or loaded probe remains unscaled in intensity + - ``int`` or ``float`` : modeled or loaded probe intensity is scaled to that number of photons + - ``'maxdiff'`` : modeled or loaded probe intensity is scaled to match the brightest diffraction pattern userlevel = 2 [propagation] @@ -311,7 +313,8 @@ def init_storage(storage, pars, energy=None, **kwargs): p = DEFAULT.copy(depth=3) model = None - if hasattr(pars, 'items') or hasattr(pars, 'items'): + + if hasattr(pars, 'items'): # This is a dict p.update(pars, in_place_depth=3) @@ -366,7 +369,6 @@ def init_storage(storage, pars, energy=None, **kwargs): model *= np.sqrt(p.photons) / np.prod(s.shape) elif type(p.model) is np.ndarray: model = p.model - p.photons = None elif p.model in resources.probes: model = resources.probes[p.model](s.shape) elif str(p.model) == 'recon': @@ -378,7 +380,6 @@ def init_storage(storage, pars, energy=None, **kwargs): 'Attempt to load layer `%s` of probe storage with ID `%s` from `%s`' % (str(layer), str(ID), p.recon.rfile)) model = u.load_from_ptyr(p.recon.rfile, 'probe', ID, layer) - p.photons = None # This could be more sophisticated, # i.e. matching the real space grids etc. elif str(p.model) == 'stxm': @@ -476,7 +477,7 @@ def _process(model, aperture_pars=None, prop_pars=None, photons=1e7, model = prop(model) # apply photon count - if photons is not None: + if (type(photons) is int) or (type(photons) is float): model *= np.sqrt(photons / u.norm2(model)) return model diff --git a/ptypy/core/manager.py b/ptypy/core/manager.py index ec376c8ff..042e0da26 100644 --- a/ptypy/core/manager.py +++ b/ptypy/core/manager.py @@ -1130,18 +1130,26 @@ def _initialize_probe(self, probe_ids): # Bypass additional tests if input is a string (previous reconstruction) if illu_pars != str(illu_pars): - # if photon count is None, assign a number from the stats. phot = illu_pars.get('photons') phot_max = self.diff.max_power - - if phot is None: + if phot == 'maxdiff': + # probe intensity to be scaled to the brightest diffraction pattern logger.info( - 'Found no photon count for probe in parameters.\nUsing photon count %.2e from photon report' % phot_max) + 'Probe intensity is being rescaled to match the brightest diffraction pattern.\nUsing photon count %.2e from photon report' % phot_max) illu_pars['photons'] = phot_max - elif np.abs(np.log10(phot) - np.log10(phot_max)) > 1: - logger.warning( - 'Photon count from input parameters (%.2e) differs from statistics (%.2e) by more than a magnitude' % ( - phot, phot_max)) + elif phot is None: + # probe intensity to remain untouched + pass + elif (type(phot) is int) or (type(phot) is float): + # probe intensity to be scaled to a specific value + if phot < 0: + logger.warning( + f'Given photon count is negative. Using the absolute of the given value: {-1 * phot:.2e}') + phot = -1 * phot + if np.abs(np.log10(phot) - np.log10(phot_max)) > 1: + logger.warning( + 'Photon count from input parameters (%.2e) differs from statistics (%.2e) by more than a magnitude' % ( + phot, phot_max)) if (self.p.coherence.num_probe_modes > 1) and (type(illu_pars) is not np.ndarray): From 0ff03fce037f9af13b5f6d6b8e89f02197b2eb98 Mon Sep 17 00:00:00 2001 From: Maik Kahnt Date: Wed, 8 Jan 2025 11:03:08 +0100 Subject: [PATCH 4/7] fixed if statement to account for new possible data types of illumination.photons --- ptypy/core/illumination.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptypy/core/illumination.py b/ptypy/core/illumination.py index c714a5976..29cb7e7be 100644 --- a/ptypy/core/illumination.py +++ b/ptypy/core/illumination.py @@ -365,7 +365,7 @@ def init_storage(storage, pars, energy=None, **kwargs): if p.model is None: model = np.ones(s.shape, s.dtype) - if p.photons is not None: + if (type(p.photons) is int) or (type(p.photons) is float): model *= np.sqrt(p.photons) / np.prod(s.shape) elif type(p.model) is np.ndarray: model = p.model From 0cbb99902368b770ca44d13b91c7515e287e17f6 Mon Sep 17 00:00:00 2001 From: Timothy Poon <62692924+ptim0626@users.noreply.github.com> Date: Mon, 9 Dec 2024 11:38:21 +0000 Subject: [PATCH 5/7] Drop Python <= 3.8 support and add 3.13 (#588) * Drop Python <= 3.8 support * Add py 3.13 in CI tests * Use conda-incubator to set up conda env * Ensure correct activation throughout the tests * Migrate to importlib in script2rst * Migrate to importlib in resources * Migrate to packaging.version --- .github/workflows/test.yml | 32 ++++++++----------- dependencies_dev.yml | 1 + dependencies_full.yml | 1 + doc/script2rst.py | 30 ++++++++--------- ptypy/accelerate/cuda_cupy/multi_gpu.py | 8 ++--- ptypy/accelerate/cuda_pycuda/multi_gpu.py | 10 +++--- ptypy/resources/__init__.py | 14 ++++---- pyproject.toml | 6 ++-- .../cuda_cupy_tests/multi_gpu_test.py | 10 +++--- .../cuda_pycuda_tests/multi_gpu_test.py | 6 ++-- 10 files changed, 58 insertions(+), 60 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d53a9a632..b37bc8c30 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,16 +20,19 @@ on: jobs: build-linux: runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} strategy: max-parallel: 10 fail-fast: false matrix: python-version: - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" + - "3.13" conda-env: - "core" - "full" @@ -37,20 +40,14 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - uses: conda-incubator/setup-miniconda@v3 with: - python-version: ${{ matrix.python-version }} - - name: Add conda to system path - run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH - conda --version - conda info - - name: Make sure conda is updated - run: | - conda update conda - conda --version + auto-update-conda: true + auto-activate-base: false + conda-remove-defaults: true + channels: conda-forge + activate-environment: ptypy_env + python-version: ${{ matrix.python-version }} - name: Install ${{ matrix.conda-env }} dependencies run: | # replace python version in dependencies @@ -58,16 +55,13 @@ jobs: if [ ${{ matrix.conda-env }} == 'full' ] && [ ${{ matrix.python-version }} == '3.12' ]; then sed -i '/- pyfftw/d' dependencies_${{ matrix.conda-env }}.yml fi - # if [ ${{ matrix.conda-env }} == 'full' ] && [ ${{ matrix.python-version }} == '3.8' ]; then - # sed -i '/- mpi4py/d' dependencies_${{ matrix.conda-env }}.yml - # fi # if [ ${{ matrix.conda-env }} == 'full' ] && [ ${{ matrix.python-version }} == '3.9' ]; then # sed -i '/- mpi4py/d' dependencies_${{ matrix.conda-env }}.yml # fi conda install --solver=classic mpich - conda env update --file dependencies_${{ matrix.conda-env }}.yml --name base + conda env update --file dependencies_${{ matrix.conda-env }}.yml --name ptypy_env conda install --solver=classic flake8 pytest pytest-cov - conda list + conda list - name: Prepare ptypy run: | # Install ptypy diff --git a/dependencies_dev.yml b/dependencies_dev.yml index 5462c145a..6563f9dd8 100644 --- a/dependencies_dev.yml +++ b/dependencies_dev.yml @@ -10,6 +10,7 @@ dependencies: - pyzmq - pep8 - mpi4py + - packaging - pillow - pyfftw - pip diff --git a/dependencies_full.yml b/dependencies_full.yml index e43241fce..205bdb19e 100644 --- a/dependencies_full.yml +++ b/dependencies_full.yml @@ -9,6 +9,7 @@ dependencies: - h5py - pyzmq - mpi4py + - packaging - pillow - pyfftw - pyyaml diff --git a/doc/script2rst.py b/doc/script2rst.py index b05148e46..b30d94667 100644 --- a/doc/script2rst.py +++ b/doc/script2rst.py @@ -1,5 +1,6 @@ import sys import io +from importlib.resources import files import contextlib import os @@ -9,13 +10,12 @@ 'simupod.py', 'ownengine.py', 'subclassptyscan.py'] +_ptypy_dir = files('ptypy') if len(sys.argv) == 1: - import pkg_resources - for script in scripts: - scr = pkg_resources.resource_filename('ptypy', tutorial_dir+script) - if not os.path.exists(scr): + scr = _ptypy_dir / (tutorial_dir + script) + if not scr.exists(): print('Using backup tutorial for %s' % script) scr = '../tutorial/'+script #subprocess.call(['python',sys.argv[0]+' '+scr]) # doesn't work @@ -50,13 +50,13 @@ def stdoutIO(stdout=None): frst.write(""" .. note:: This tutorial was generated from the python source - :file:`[ptypy_root]/tutorial/%(fname)s` using :file:`ptypy/doc/%(this)s`. + :file:`[ptypy_root]/tutorial/%(fname)s` using :file:`ptypy/doc/%(this)s`. You are encouraged to modify the parameters and rerun the tutorial with:: - + $ python [ptypy_root]/tutorial/%(fname)s """ % {'fname': os.path.split(script_name)[-1], 'this': sys.argv[0]}) - + was_comment = True while True: @@ -86,7 +86,7 @@ def stdoutIO(stdout=None): frst.write(' '+line2[1:].strip()+'\n') frst.write('\n') continue - + if line.startswith('"""'): frst.write('.. parsed-literal::\n\n') while True: @@ -95,11 +95,11 @@ def stdoutIO(stdout=None): break frst.write(' ' + line2) continue - + decorator = False indent = False for key in indent_keys: - if line.startswith(key): + if line.startswith(key): indent = True break @@ -125,12 +125,12 @@ def stdoutIO(stdout=None): pt = fpy.tell() exec(func+'\n') continue - + wline = line.strip() if not wline: frst.write('\n') continue - + with stdoutIO() as sout: exec(wline) out = sout.getvalue() @@ -150,9 +150,9 @@ def stdoutIO(stdout=None): if was_comment: wline = '\n::\n\n'+wline was_comment = False - + frst.write(wline+'\n') - + #print out if out.strip(): print(out) @@ -160,5 +160,5 @@ def stdoutIO(stdout=None): frst.write(' '*3+l+'\n') out = '' - + diff --git a/ptypy/accelerate/cuda_cupy/multi_gpu.py b/ptypy/accelerate/cuda_cupy/multi_gpu.py index 6ccbce9a7..e631c3405 100644 --- a/ptypy/accelerate/cuda_cupy/multi_gpu.py +++ b/ptypy/accelerate/cuda_cupy/multi_gpu.py @@ -10,12 +10,12 @@ - OpenMPI in a conda install needs to have the environment variable --> if cuda support isn't enabled, the application simply crashes with a seg fault -2) For NCCL peer-to-peer transfers, the EXCLUSIVE compute mode cannot be used. +2) For NCCL peer-to-peer transfers, the EXCLUSIVE compute mode cannot be used. It should be in DEFAULT mode. """ -from pkg_resources import parse_version +from packaging.version import parse import numpy as np import cupy as cp from ptypy.utils import parallel @@ -44,7 +44,7 @@ have_cuda_mpi = (mpi4py is not None) and \ "OMPI_MCA_opal_cuda_support" in os.environ and \ os.environ["OMPI_MCA_opal_cuda_support"] == "true" and \ - parse_version(parse_version(mpi4py.__version__).base_version) >= parse_version("3.1.0") and \ + parse(parse(mpi4py.__version__).base_version) >= parse("3.1.0") and \ not ('PTYPY_USE_MPI' in os.environ) @@ -114,7 +114,7 @@ def allReduceSum(self, arr): count, datatype = self.__get_NCCL_count_dtype(arr) - self.com.allReduce(arr.data.ptr, arr.data.ptr, count, datatype, nccl.NCCL_SUM, + self.com.allReduce(arr.data.ptr, arr.data.ptr, count, datatype, nccl.NCCL_SUM, cp.cuda.get_current_stream().ptr) def __get_NCCL_count_dtype(self, arr): diff --git a/ptypy/accelerate/cuda_pycuda/multi_gpu.py b/ptypy/accelerate/cuda_pycuda/multi_gpu.py index 73138c3ee..09ec60552 100644 --- a/ptypy/accelerate/cuda_pycuda/multi_gpu.py +++ b/ptypy/accelerate/cuda_pycuda/multi_gpu.py @@ -5,7 +5,7 @@ Findings: -1) NCCL works with unit tests, but not in the engines. It seems to +1) NCCL works with unit tests, but not in the engines. It seems to add something to the existing pycuda Context or create a new one, as a later event recording on an exit wave transfer fails with 'ivalid resource handle' Cuda Error. This error typically happens if for example @@ -22,14 +22,14 @@ - OpenMPI in a conda install needs to have the environment variable --> if cuda support isn't enabled, the application simply crashes with a seg fault -4) For NCCL peer-to-peer transfers, the EXCLUSIVE compute mode cannot be used. +4) For NCCL peer-to-peer transfers, the EXCLUSIVE compute mode cannot be used. It should be in DEFAULT mode. 5) NCCL support has been dropped from PyCUDA module, but can be used with CuPy module instead """ -from pkg_resources import parse_version +from packaging.version import parse import numpy as np from pycuda import gpuarray import pycuda.driver as cuda @@ -54,7 +54,7 @@ have_cuda_mpi = (mpi4py is not None) and \ "OMPI_MCA_opal_cuda_support" in os.environ and \ os.environ["OMPI_MCA_opal_cuda_support"] == "true" and \ - parse_version(parse_version(mpi4py.__version__).base_version) >= parse_version("3.1.0") and \ + parse(parse(mpi4py.__version__).base_version) >= parse("3.1.0") and \ hasattr(gpuarray.GPUArray, '__cuda_array_interface__') and \ not ('PTYPY_USE_MPI' in os.environ) @@ -97,7 +97,7 @@ def allReduceSum(self, arr): if parallel.MPIenabled: comm = parallel.comm comm.Allreduce(parallel.MPI.IN_PLACE, arr) - + # pick the appropriate communicator depending on installed packages def get_multi_gpu_communicator(use_cuda_mpi=True): diff --git a/ptypy/resources/__init__.py b/ptypy/resources/__init__.py index 50bab0b5e..becdece26 100644 --- a/ptypy/resources/__init__.py +++ b/ptypy/resources/__init__.py @@ -1,15 +1,17 @@ -import pkg_resources +from importlib.resources import files import numpy as np -flowerfile = pkg_resources.resource_filename(__name__,'flowers.png') -moonfile = pkg_resources.resource_filename(__name__,'moon.png') -treefile = pkg_resources.resource_filename(__name__,'tree.png') +_res_dir = files('ptypy.resources') +flowerfile = _res_dir / 'flowers.png' +moonfile = _res_dir / 'moon.png' +treefile = _res_dir / 'tree.png' + def flower_obj(shape=None): from ptypy import utils as u import numpy as np from matplotlib.image import imread - + im = u.rgb2complex(imread(flowerfile)) if shape is not None: sh = u.expect2(shape) @@ -37,7 +39,7 @@ def moon_pr(shape=None): from ptypy import utils as u import numpy as np from matplotlib.image import imread - + im = u.rgb2complex(imread(moonfile)) if shape is not None: sh = u.expect2(shape) diff --git a/pyproject.toml b/pyproject.toml index 4b35cf554..d3493bce4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] description = "Ptychography Reconstruction for Python" readme = "README.rst" -requires-python = ">=3.7" +requires-python = ">=3.9" classifiers = [ "Programming Language :: Python :: 3", "Development Status :: 3 - Alpha", @@ -31,7 +31,7 @@ dependencies = [ [project.optional-dependencies] full = ["mpi4py","matplotlib","pyzmq","pillow", "pyfftw", "pyyaml"] cupy-cuda11x = ["mpi4py","matplotlib","pyzmq","pillow", "pyfftw", "cupy-cuda11x"] -cupy-cuda12x = ["mpi4py","matplotlib","pyzmq","pillow", "pyfftw", "cupy-cuda12x"] +cupy-cuda12x = ["mpi4py","matplotlib","pyzmq","pillow", "pyfftw", "cupy-cuda12x"] [project.scripts] "ptypy.plot" = "ptypy.cli.plotter:ptypy_plot" @@ -69,4 +69,4 @@ ptypy = "ptypy" [tool.setuptools.package-data] ptypy = ["resources/*",] -"ptypy.accelerate.cuda_common" = ["*.cu", "*.cuh"] \ No newline at end of file +"ptypy.accelerate.cuda_common" = ["*.cu", "*.cuh"] diff --git a/test/accelerate_tests/cuda_cupy_tests/multi_gpu_test.py b/test/accelerate_tests/cuda_cupy_tests/multi_gpu_test.py index 0c234d878..424c6696b 100644 --- a/test/accelerate_tests/cuda_cupy_tests/multi_gpu_test.py +++ b/test/accelerate_tests/cuda_cupy_tests/multi_gpu_test.py @@ -11,7 +11,7 @@ from ptypy.accelerate.cuda_cupy import multi_gpu as mgpu from ptypy.utils import parallel -from pkg_resources import parse_version +from packaging.version import parse class GpuDataTest(CupyCudaTest): """ @@ -36,8 +36,8 @@ def setUp(self): @unittest.skipIf(parallel.rank != 0, "Only in MPI rank 0") def test_version(self): - v1 = parse_version("3.1.0") - v2 = parse_version(parse_version("3.1.0a").base_version) + v1 = parse("3.1.0") + v2 = parse(parse("3.1.0a").base_version) self.assertGreaterEqual(v2, v1) @@ -61,7 +61,7 @@ def multigpu_tester(self, com): def test_multigpu_auto(self): self.multigpu_tester(mgpu.get_multi_gpu_communicator()) - + def test_multigpu_mpi(self): self.multigpu_tester(mgpu.MultiGpuCommunicatorMpi()) @@ -71,4 +71,4 @@ def test_multigpu_cudampi(self): @unittest.skipIf(not mgpu.have_nccl, "NCCL not available") def test_multigpu_nccl(self): - self.multigpu_tester(mgpu.MultiGpuCommunicatorNccl()) \ No newline at end of file + self.multigpu_tester(mgpu.MultiGpuCommunicatorNccl()) diff --git a/test/accelerate_tests/cuda_pycuda_tests/multi_gpu_test.py b/test/accelerate_tests/cuda_pycuda_tests/multi_gpu_test.py index be96aed54..6aa3ac537 100644 --- a/test/accelerate_tests/cuda_pycuda_tests/multi_gpu_test.py +++ b/test/accelerate_tests/cuda_pycuda_tests/multi_gpu_test.py @@ -13,7 +13,7 @@ from ptypy.accelerate.cuda_pycuda import multi_gpu as mgpu from ptypy.utils import parallel -from pkg_resources import parse_version +from packaging.version import parse class GpuDataTest(unittest.TestCase): """ @@ -51,8 +51,8 @@ def tearDown(self): @unittest.skipIf(parallel.rank != 0, "Only in MPI rank 0") def test_version(self): - v1 = parse_version("3.1.0") - v2 = parse_version(parse_version("3.1.0a").base_version) + v1 = parse("3.1.0") + v2 = parse(parse("3.1.0a").base_version) self.assertGreaterEqual(v2, v1) From aa34bba16febec841dd449fe0f4ef605d954a696 Mon Sep 17 00:00:00 2001 From: "Benedikt J. Daurer" Date: Fri, 13 Dec 2024 18:08:14 +0000 Subject: [PATCH 6/7] Fix in compilation of filtered_fft, import from setuptools instead of distutils (#591) --- cufft/extensions.py | 5 +++-- cufft/setup.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cufft/extensions.py b/cufft/extensions.py index 4d8b7f598..468d8d1cb 100644 --- a/cufft/extensions.py +++ b/cufft/extensions.py @@ -4,8 +4,8 @@ import os, re import subprocess import sysconfig -from distutils.unixccompiler import UnixCCompiler -from distutils.command.build_ext import build_ext +from setuptools._distutils.unixccompiler import UnixCCompiler +from setuptools.command.build_ext import build_ext def find_in_path(name, path): @@ -116,6 +116,7 @@ def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): compiler_command = [self.CUDA["nvcc"]] + self.NVCC_FLAGS + self.OPTFLAGS + ["-Xcompiler"] + self.CXXFLAGS + CPPFLAGS compiler_exec = " ".join(compiler_command) self.set_executable('compiler_so', compiler_exec) + self.set_executable('compiler_so_cxx', compiler_exec) postargs = [] # we don't actually have any postargs super(NvccCompiler, self)._compile(obj, src, ext, cc_args, postargs, pp_opts) # the _compile method # reset the default compiler_so, which we might have changed for cuda diff --git a/cufft/setup.py b/cufft/setup.py index a8ef7c61e..b02262bdf 100644 --- a/cufft/setup.py +++ b/cufft/setup.py @@ -2,7 +2,7 @@ # we should aim to remove the distutils dependency import setuptools -from distutils.core import setup, Extension +from setuptools import setup, Extension import os ext_modules = [] From e3f812c394d4a434d8ea21b9bc9bc7e3c65eef31 Mon Sep 17 00:00:00 2001 From: "Benedikt J. Daurer" Date: Fri, 13 Dec 2024 20:53:04 +0000 Subject: [PATCH 7/7] FFT filter changes with Numpy v2.0 (#592) * Include accelerate base tests by default * Change expected output in FFT filter tests to reflect support for single precision in Numpy > 2.0 * Adjust tolerance in pycuda fft filter tests --- pyproject.toml | 1 + .../base_tests/array_utils_test.py | 142 +++++++++--------- .../cuda_pycuda_tests/array_utils_test.py | 12 +- 3 files changed, 78 insertions(+), 77 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d3493bce4..3f4b0c9b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ testpaths = [ "test/ptyscan_tests", "test/template_tests", "test/util_tests", + "test/accelerate_tests/base_tests" ] # this is all BETA according to setuptools diff --git a/test/accelerate_tests/base_tests/array_utils_test.py b/test/accelerate_tests/base_tests/array_utils_test.py index 364352a5c..78540f978 100644 --- a/test/accelerate_tests/base_tests/array_utils_test.py +++ b/test/accelerate_tests/base_tests/array_utils_test.py @@ -305,42 +305,42 @@ def test_fft_filter(self): kernel = np.fft.fftn(rk) output = au.fft_filter(data, kernel, prefactor, postfactor) + + known_test_output = np.array([-0.0000000e+00+0.00000000e+00j, -0.0000000e+00+0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + 0.0000000e+00-0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + -0.0000000e+00+0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + -0.0000000e+00+0.00000000e+00j, -0.0000000e+00+0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00-0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00+0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 0.0000000e+00-0.00000000e+00j, + 0.0000000e+00+0.00000000e+00j, 6.1097220e-05+2.92563982e-05j, + 4.0044695e-05+2.52102855e-05j, 8.9999994e+02+9.00000000e+02j, + 8.9999988e+02+8.99999939e+02j, 5.0999994e+02+5.09999939e+02j, + 1.9365043e-05+5.84280206e-05j, 3.0681291e-05+2.31116355e-05j, + 1.2552022e-05-1.01537153e-05j, -1.4034913e-05-1.17988075e-05j, + -1.9193330e-05+2.72889110e-07j, 1.3895768e-05+1.64778357e-05j, + 6.5228807e-05+2.45708943e-05j, 3.8999994e+02+3.89999939e+02j, + 8.9999988e+02+8.99999878e+02j, 8.9999982e+02+8.99999878e+02j, + 3.0000015e+01+3.00000248e+01j, 3.8863189e-05+3.26705631e-05j, + 2.8768281e-06-1.62116921e-05j, -3.2418033e-05-1.97073969e-05j, + -6.6843757e-05+7.19546824e-06j, 6.5036993e-06+3.95851657e-06j, + -2.4053887e-05+9.88548163e-06j, 1.5231475e-05+1.31202614e-06j, + 8.7000000e+01+8.70000305e+01j, 6.1035156e-05+0.00000000e+00j, + 6.1035156e-05+0.00000000e+00j, -2.4943074e-07+6.62429193e-06j, + 1.6712515e-06-2.97475322e-06j, 1.9025241e-05+2.97752194e-07j, + -9.2436176e-07-3.86252796e-05j, -8.8145862e-06-9.89961700e-06j, + -1.5782407e-06+1.01533060e-05j, -4.7593076e-06+2.96332291e-05j]) - known_test_output = np.array([-0.00000000e+00+0.00000000e+00j, 0.00000000e+00-0.00000000e+00j, - -0.00000000e+00 + 0.00000000e+00j, 0.00000000e+00 - 0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, -0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - -0.00000000e+00+0.00000000e+00j, -0.00000000e+00+0.00000000e+00j, - -0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, 0.00000000e+00-0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00-0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, 0.00000000e+00-0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, 8.66422277e-14+4.86768828e-14j, - 7.23113320e-14+2.82331542e-14j, 9.00000000e+02+9.00000000e+02j, - 9.00000000e+02+9.00000000e+02j, 5.10000000e+02+5.10000000e+02j, - 1.41172830e-14+3.62223425e-14j, 2.61684238e-14-4.13866575e-14j, - 2.16691314e-14-1.95102733e-14j, -1.36536942e-13-9.94589021e-14j, - -1.42905371e-13-5.77964697e-14j, -5.00005072e-14+4.08620637e-14j, - 6.38160272e-14+7.61753583e-14j, 3.90000000e+02+3.90000000e+02j, - 9.00000000e+02+9.00000000e+02j, 9.00000000e+02+9.00000000e+02j, - 3.00000000e+01+3.00000000e+01j, 8.63255773e-14+7.08532924e-14j, - 1.80941313e-14-3.85517154e-14j, 7.84277340e-14-1.32008745e-14j, - -6.57025196e-14-1.72739350e-14j, -6.69570857e-15+6.49622898e-14j, - 6.27436466e-15+7.57162569e-14j, 2.01150157e-15+3.65538558e-14j, - 8.70000000e+01+8.70000000e+01j, -1.13686838e-13-1.70530257e-13j, - 0.00000000e+00-2.27373675e-13j, -1.84492121e-14-9.21502853e-14j, - 2.12418687e-14-8.62209232e-14j, 1.20880692e-13+3.86522371e-14j, - 1.03754734e-13+9.19851759e-14j, 5.50926123e-14+1.17150422e-13j, - -5.47869215e-14+5.87176511e-14j, -3.52652980e-14+8.44455504e-15j]) - - np.testing.assert_array_almost_equal(output.flat[::2000], known_test_output) + np.testing.assert_array_almost_equal(output.flat[::2000], known_test_output, decimal=5) def test_fft_filter_batched(self): data = np.zeros((2,256, 512), dtype=COMPLEX_TYPE) @@ -356,42 +356,42 @@ def test_fft_filter_batched(self): kernel = np.fft.fftn(rk) output = au.fft_filter(data, kernel, prefactor, postfactor) - - known_test_output = np.array([-0.00000000e+00+0.00000000e+00j, 0.00000000e+00-0.00000000e+00j, - -0.00000000e+00 + 0.00000000e+00j, 0.00000000e+00 - 0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, -0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - -0.00000000e+00+0.00000000e+00j, -0.00000000e+00+0.00000000e+00j, - -0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, 0.00000000e+00-0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00+0.00000000e+00j, - 0.00000000e+00+0.00000000e+00j, 0.00000000e+00-0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, 0.00000000e+00-0.00000000e+00j, - 0.00000000e+00-0.00000000e+00j, 8.66422277e-14+4.86768828e-14j, - 7.23113320e-14+2.82331542e-14j, 9.00000000e+02+9.00000000e+02j, - 9.00000000e+02+9.00000000e+02j, 5.10000000e+02+5.10000000e+02j, - 1.41172830e-14+3.62223425e-14j, 2.61684238e-14-4.13866575e-14j, - 2.16691314e-14-1.95102733e-14j, -1.36536942e-13-9.94589021e-14j, - -1.42905371e-13-5.77964697e-14j, -5.00005072e-14+4.08620637e-14j, - 6.38160272e-14+7.61753583e-14j, 3.90000000e+02+3.90000000e+02j, - 9.00000000e+02+9.00000000e+02j, 9.00000000e+02+9.00000000e+02j, - 3.00000000e+01+3.00000000e+01j, 8.63255773e-14+7.08532924e-14j, - 1.80941313e-14-3.85517154e-14j, 7.84277340e-14-1.32008745e-14j, - -6.57025196e-14-1.72739350e-14j, -6.69570857e-15+6.49622898e-14j, - 6.27436466e-15+7.57162569e-14j, 2.01150157e-15+3.65538558e-14j, - 8.70000000e+01+8.70000000e+01j, -1.13686838e-13-1.70530257e-13j, - 0.00000000e+00-2.27373675e-13j, -1.84492121e-14-9.21502853e-14j, - 2.12418687e-14-8.62209232e-14j, 1.20880692e-13+3.86522371e-14j, - 1.03754734e-13+9.19851759e-14j, 5.50926123e-14+1.17150422e-13j, - -5.47869215e-14+5.87176511e-14j, -3.52652980e-14+8.44455504e-15j]) - - np.testing.assert_array_almost_equal(output[1].flat[::2000], known_test_output) + + known_test_output = np.array([ 0.00000000e+00-0.0000000e+00j, 0.00000000e+00-0.0000000e+00j, + 0.00000000e+00+0.0000000e+00j, 0.00000000e+00+0.0000000e+00j, + 0.00000000e+00+0.0000000e+00j, 0.00000000e+00+0.0000000e+00j, + 0.00000000e+00+0.0000000e+00j, 0.00000000e+00+0.0000000e+00j, + -0.00000000e+00+0.0000000e+00j, 0.00000000e+00-0.0000000e+00j, + 0.00000000e+00-0.0000000e+00j, 0.00000000e+00-0.0000000e+00j, + 0.00000000e+00+0.0000000e+00j, 0.00000000e+00+0.0000000e+00j, + 0.00000000e+00+0.0000000e+00j, 0.00000000e+00+0.0000000e+00j, + 0.00000000e+00+0.0000000e+00j, 0.00000000e+00+0.0000000e+00j, + 0.00000000e+00+0.0000000e+00j, 0.00000000e+00-0.0000000e+00j, + 0.00000000e+00-0.0000000e+00j, 0.00000000e+00-0.0000000e+00j, + 0.00000000e+00+0.0000000e+00j, 0.00000000e+00+0.0000000e+00j, + 0.00000000e+00+0.0000000e+00j, 0.00000000e+00+0.0000000e+00j, + 0.00000000e+00+0.0000000e+00j, 0.00000000e+00+0.0000000e+00j, + 0.00000000e+00+0.0000000e+00j, 0.00000000e+00-0.0000000e+00j, + 0.00000000e+00-0.0000000e+00j, 0.00000000e+00-0.0000000e+00j, + 0.00000000e+00-0.0000000e+00j, 4.86995195e-05-9.1511911e-06j, + 5.89395277e-05+3.6706428e-05j, 8.99999817e+02+9.0000000e+02j, + 8.99999817e+02+9.0000000e+02j, 5.09999969e+02+5.0999997e+02j, + 6.86399580e-05+5.5245564e-05j, -2.15578075e-06-8.0761157e-07j, + -5.99612467e-05-3.7489859e-05j, -2.08058154e-05-1.7001423e-05j, + -3.15661709e-05-2.0192698e-05j, -1.17410173e-05-2.3929812e-05j, + 8.41844594e-05+4.9635066e-05j, 3.90000031e+02+3.9000003e+02j, + 8.99999817e+02+8.9999994e+02j, 8.99999817e+02+8.9999994e+02j, + 3.00000153e+01+3.0000000e+01j, 4.75842753e-05+1.7961407e-05j, + -1.28229876e-05-3.3492659e-05j, -1.50405585e-05+3.0159079e-05j, + -1.00799960e-04-6.6932058e-05j, -4.90295024e-05-3.6601130e-05j, + -4.48861247e-05-1.4717044e-05j, 2.60417364e-05-8.3221821e-06j, + 8.69999847e+01+8.7000046e+01j, 4.31583721e-05+4.3158372e-05j, + 4.31583721e-05+4.3158372e-05j, 4.04649109e-06-1.6836095e-05j, + 1.37377283e-05+5.2577798e-06j, -2.30404657e-05-3.4596611e-05j, + -1.33214944e-05-3.2517899e-05j, 2.45428764e-05-3.5186855e-07j, + -1.85950885e-05-2.1921931e-05j, -1.65030433e-05-8.0249208e-07j]) + + np.testing.assert_array_almost_equal(output[1].flat[::2000], known_test_output, decimal=5) def test_complex_gaussian_filter_fft(self): diff --git a/test/accelerate_tests/cuda_pycuda_tests/array_utils_test.py b/test/accelerate_tests/cuda_pycuda_tests/array_utils_test.py index be82cc292..d11584b01 100644 --- a/test/accelerate_tests/cuda_pycuda_tests/array_utils_test.py +++ b/test/accelerate_tests/cuda_pycuda_tests/array_utils_test.py @@ -580,7 +580,7 @@ def test_fft_filter_UNITY(self): output = au.fft_filter(data, kernel, prefactor, postfactor) - np.testing.assert_allclose(output, data_dev.get(), rtol=1e-5, atol=1e-6) + np.testing.assert_allclose(output, data_dev.get(), rtol=1e-5, atol=1e-5) def test_fft_filter_batched_UNITY(self): sh = (2,16, 35) @@ -607,7 +607,7 @@ def test_fft_filter_batched_UNITY(self): output = au.fft_filter(data, kernel, prefactor, postfactor) print(data_dev.get()) - np.testing.assert_allclose(output, data_dev.get(), rtol=1e-5, atol=1e-6) + np.testing.assert_allclose(output, data_dev.get(), rtol=1e-5, atol=1e-5) def test_complex_gaussian_filter_fft_little_blurring_UNITY(self): # Arrange @@ -624,7 +624,7 @@ def test_complex_gaussian_filter_fft_little_blurring_UNITY(self): out_exp = au.complex_gaussian_filter_fft(data, mfs) out = data_dev.get() - np.testing.assert_allclose(out_exp, out, atol=1e-6) + np.testing.assert_allclose(out_exp, out, atol=1e-5) def test_complex_gaussian_filter_fft_more_blurring_UNITY(self): # Arrange @@ -641,7 +641,7 @@ def test_complex_gaussian_filter_fft_more_blurring_UNITY(self): out_exp = au.complex_gaussian_filter_fft(data, mfs) out = data_dev.get() - np.testing.assert_allclose(out_exp, out, atol=1e-6) + np.testing.assert_allclose(out_exp, out, atol=1e-5) def test_complex_gaussian_filter_fft_nonsquare_UNITY(self): # Arrange @@ -660,7 +660,7 @@ def test_complex_gaussian_filter_fft_nonsquare_UNITY(self): out_exp = au.complex_gaussian_filter_fft(data, mfs) out = data_dev.get() - np.testing.assert_allclose(out_exp, out, atol=1e-6) + np.testing.assert_allclose(out_exp, out, atol=1e-5) def test_complex_gaussian_filter_fft_batched(self): # Arrange @@ -680,4 +680,4 @@ def test_complex_gaussian_filter_fft_batched(self): out_exp = au.complex_gaussian_filter_fft(data, mfs) out = data_dev.get() - np.testing.assert_allclose(out_exp, out, atol=1e-6) \ No newline at end of file + np.testing.assert_allclose(out_exp, out, atol=1e-5) \ No newline at end of file