Merge branch 'main' into ianna/3214-cuda-test-failure-tests-cudatest_…

…3149_complex_reducerspytest_block_boundary_prod_complex13-but-only-the-_first-time_-it-is-run

.all-contributorsrc

@@ -465,6 +465,24 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "ariostas",
+      "name": "Andres Rios Tascon",
+      "avatar_url": "https://avatars.githubusercontent.com/u/7596837?v=4",
+      "profile": "http://www.ariostas.com",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "maxymnaumchyk",
+      "name": "maxymnaumchyk",
+      "avatar_url": "https://avatars.githubusercontent.com/u/70752300?v=4",
+      "profile": "https://github.com/maxymnaumchyk",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.github/workflows/build-wheels.yml

@@ -105,7 +105,7 @@ jobs:
     - name: Prepare build files
       run: pipx run nox -s prepare
 
-    - uses: pypa/cibuildwheel@v2.19
+    - uses: pypa/cibuildwheel@v2.20
       env:
         CIBW_BUILD: "${{ matrix.build }}*"
         CIBW_ARCHS: ${{ matrix.arch }}
@@ -136,7 +136,7 @@ jobs:
       SOURCE_DATE_EPOCH: ${{ needs.determine-source-date-epoch.outputs.source-date-epoch }}
     strategy:
       matrix:
-        python: [38, 39, 310, 311, 312]
+        python: [38, 39, 310, 311, 312, 313]
         arch: [aarch64]
     steps:
 
@@ -157,7 +157,7 @@ jobs:
 
     - uses: docker/[email protected]
 
-    - uses: pypa/cibuildwheel@v2.19
+    - uses: pypa/cibuildwheel@v2.20
       env:
         CIBW_BUILD: cp${{ matrix.python }}-*
         CIBW_ARCHS: ${{ matrix.arch }}

.github/workflows/deploy-cpp.yml

@@ -35,8 +35,8 @@ jobs:
       run: ls -l dist/
 
     - name: Generate artifact attestation for sdist and wheel
-      uses: actions/attest-build-provenance@5e9cb68e95676991667494a6a4e59b8a2f13e1d0 # v1.3.3
+      uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3
       with:
         subject-path: "dist/awkward*cpp-*"
 
-    - uses: pypa/gh-action-pypi-publish@v1.9.0
+    - uses: pypa/gh-action-pypi-publish@v1.10.1

.github/workflows/deploy.yml

@@ -88,7 +88,7 @@ jobs:
       run: pipx run twine check dist/*
 
     - name: Generate artifact attestation for sdist and wheel
-      uses: actions/attest-build-provenance@5e9cb68e95676991667494a6a4e59b8a2f13e1d0 # v1.3.3
+      uses: actions/attest-build-provenance@1c608d11d69870c2092266b3f9a6f3abbf17002c # v1.4.3
       with:
         subject-path: "dist/awkward-*"
 
@@ -135,7 +135,7 @@ jobs:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: gh attestation verify dist/awkward-*.whl --repo ${{ github.repository }}
 
-    - uses: pypa/gh-action-pypi-publish@v1.9.0
+    - uses: pypa/gh-action-pypi-publish@v1.10.1
 
   publish-headers:
     name: "Publish header-only libraries alongside release"

.github/workflows/packaging-test.yml

@@ -68,15 +68,15 @@ jobs:
     - name: Prepare build files
       run: pipx run nox -s prepare
 
-    - uses: pypa/cibuildwheel@v2.19
+    - uses: pypa/cibuildwheel@v2.20
       env:
         CIBW_ARCHS_MACOS: universal2
         CIBW_BUILD: cp39-win_amd64 cp310-manylinux_x86_64 cp38-macosx_universal2
       with:
         config-file: cibuildwheel.toml
         package-dir: awkward-cpp
 
-    - uses: pypa/cibuildwheel@v2.19
+    - uses: pypa/cibuildwheel@v2.20
       if: matrix.os == 'ubuntu-latest'
       env:
         CIBW_BUILD: cp312-manylinux_x86_64

.github/workflows/test.yml

@@ -39,6 +39,7 @@ jobs:
           - ubuntu-latest
           - macos-13
         python-version:
+          - '3.13'
           - '3.12'
           - '3.11'
           - '3.10'
@@ -110,6 +111,11 @@ jobs:
           files: |
             awkward-cpp/dist/*.whl
 
+      - name: Add workaround for 3.13 + cramjam
+        if: matrix.python-version == '3.13'
+        run: echo 'PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1' >> $GITHUB_ENV
+        shell: bash
+
       - name: Install awkward, awkward-cpp, and dependencies
         run: >-
           python -m pip install -v . ${{ steps.find-wheel.outputs.paths }} pytest-github-actions-annotate-failures

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
     additional_dependencies: [pyyaml]
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.5
+  rev: v0.6.4
   hooks:
   - id: ruff
     args: ["--fix", "--show-fixes"]
@@ -62,20 +62,20 @@ repos:
     files: ^tests/
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.1
+  rev: 0.29.2
   hooks:
     - id: check-github-workflows
       args: ["--verbose"]
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.11.0
+  rev: v1.11.2
   hooks:
     - id: mypy
       files: src
       additional_dependencies:
         - numpy>=1.24
 
 - repo: https://github.com/abravalheri/validate-pyproject
-  rev: v0.18
+  rev: v0.19
   hooks:
     - id: validate-pyproject

README.md

@@ -228,6 +228,8 @@ Thanks especially to the gracious help of Awkward Array contributors (including
     </tr>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/pfackeldey"><img src="https://avatars.githubusercontent.com/u/18463582?v=4?s=100" width="100px;" alt="Peter Fackeldey"/><br /><sub><b>Peter Fackeldey</b></sub></a><br /><a href="https://github.com/scikit-hep/awkward/commits?author=pfackeldey" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://www.ariostas.com"><img src="https://avatars.githubusercontent.com/u/7596837?v=4?s=100" width="100px;" alt="Andres Rios Tascon"/><br /><sub><b>Andres Rios Tascon</b></sub></a><br /><a href="https://github.com/scikit-hep/awkward/commits?author=ariostas" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/maxymnaumchyk"><img src="https://avatars.githubusercontent.com/u/70752300?v=4?s=100" width="100px;" alt="maxymnaumchyk"/><br /><sub><b>maxymnaumchyk</b></sub></a><br /><a href="https://github.com/scikit-hep/awkward/commits?author=maxymnaumchyk" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

awkward-cpp/pyproject.toml

@@ -1,13 +1,13 @@
 [build-system]
 requires = [
-    "scikit-build-core>=0.9",
+    "scikit-build-core>=0.10",
     "pybind11",
 ]
 build-backend = "scikit_build_core.build"
 
 [project]
 name = "awkward_cpp"
-version = "37"
+version = "38"
 dependencies = [
     "numpy>=1.18.0",
     "importlib_resources;python_version < \"3.9\""
@@ -59,7 +59,7 @@ Releases = "https://github.com/scikit-hep/awkward-1.0/releases"
 
 
 [tool.scikit-build]
-minimum-version = "0.9"
+minimum-version = "build-system.requires"
 build-dir = "build/{cache_tag}"
 sdist.reproducible = true
 sdist.include = [

cibuildwheel.toml

@@ -23,5 +23,5 @@ build-verbosity = 1
 PIP_ONLY_BINARY = "cmake,numpy"
 
 [[tool.cibuildwheel.overrides]]
-select = "cp312-*"
-environment.PIP_PRE = "1"
+select = "cp313*"
+environment.PYO3_USE_ABI3_FORWARD_COMPATIBILITY = "1"

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "awkward"
-version = "2.6.7"
+version = "2.6.8"
 description = "Manipulate JSON-like data with NumPy-like idioms."
 license = { text = "BSD-3-Clause" }
 requires-python = ">=3.8"
@@ -32,6 +32,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Topic :: Scientific/Engineering",
     "Topic :: Scientific/Engineering :: Information Analysis",
     "Topic :: Scientific/Engineering :: Mathematics",
@@ -40,7 +41,7 @@ classifiers = [
     "Topic :: Utilities",
 ]
 dependencies = [
-    "awkward_cpp==37",
+    "awkward_cpp==38",
     "importlib_metadata>=4.13.0;python_version < \"3.12\"",
     "numpy>=1.18.0",
     "packaging",
@@ -136,7 +137,7 @@ filterwarnings = [
 ]
 log_cli_level = "info"
 
-[tool.pylint.master]
+[tool.pylint]
 py-version = "3.8"
 jobs = "0"
 ignore-paths = [
@@ -251,7 +252,6 @@ ignore_errors = true
 ignore_missing_imports = true
 
 [tool.ruff]
-src = ["src"]
 extend-exclude = [
     "studies",
     "pybind11",
@@ -289,10 +289,6 @@ ignore = [
     "PLC1901", # x == "" can be simplified to not x (empty string is falsey)
     "ISC001",  # Conflicts with the formatter in 0.1.2
 ]
-unfixable = [
-    "T20",  # Removes print statements
-    "F841", # Removes unused variables
-]
 typing-modules = ["awkward._typing"]
 external = []
 mccabe.max-complexity = 100

requirements-test-full.txt

@@ -1,9 +1,9 @@
 fsspec>=2022.11.0;sys_platform != "win32"
-jax[cpu]>=0.2.15;sys_platform != "win32" and python_version < "3.12"
-numba>=0.50.0;sys_platform != "win32" and python_version < "3.12"
-numexpr>=2.7; python_version < "3.12"
-pandas>=0.24.0;sys_platform != "win32" and python_version < "3.12"
-pyarrow==16.0.0;sys_platform != "win32" and python_version < "3.12"
+jax[cpu]>=0.2.15;sys_platform != "win32" and python_version < "3.13"
+numba>=0.50.0;sys_platform != "win32" and python_version < "3.13"
+numexpr>=2.7; python_version < "3.13"
+pandas>=0.24.0;sys_platform != "win32" and python_version < "3.13"
+pyarrow==16.0.0;sys_platform != "win32" and python_version < "3.13"
 pytest>=6
 pytest-cov
 pytest-xdist

src/awkward/_connect/cuda/cuda_kernels/awkward_reduce_prod_complex.cu

@@ -30,8 +30,8 @@ awkward_reduce_prod_complex_a(
     int64_t thread_id = blockIdx.x * blockDim.x + threadIdx.x;
 
     if (thread_id < outlength) {
-      toptr[thread_id * 2] = (T)1.0f;
-      toptr[thread_id * 2 + 1] = (T)0.0f;
+      toptr[thread_id * 2] = (T)1;
+      toptr[thread_id * 2 + 1] = (T)0;
     }
   }
 }
@@ -59,8 +59,8 @@ awkward_reduce_prod_complex_b(
 
     if (thread_id < lenparents) {
       for (int64_t stride = 1; stride < blockDim.x; stride *= 2) {
-        T real = (T)1.0f;
-        T imag = (T)0.0f;
+        T real = (T)1;
+        T imag = (T)0;
         if (idx >= stride && thread_id < lenparents && parents[thread_id] == parents[thread_id - stride]) {
           real = temp[(idx - stride) * 2];
           imag = temp[(idx - stride) * 2 + 1];

src/awkward/_connect/pyarrow/table_conv.py

@@ -221,7 +221,7 @@ def replace_schema(table: pyarrow.Table, new_schema: pyarrow.Schema) -> pyarrow.
         new_batches.append(
             pyarrow.RecordBatch.from_arrays(arrays=columns, schema=new_schema)
         )
-    return pyarrow.Table.from_batches(new_batches)
+    return pyarrow.Table.from_batches(new_batches, schema=new_schema)
 
 
 def array_with_replacement_type(

src/awkward/contents/bitmaskedarray.py

@@ -11,6 +11,7 @@
 from awkward._backends.backend import Backend
 from awkward._meta.bitmaskedmeta import BitMaskedMeta
 from awkward._nplikes.array_like import ArrayLike
+from awkward._nplikes.cupy import Cupy
 from awkward._nplikes.numpy import Numpy
 from awkward._nplikes.numpy_like import IndexType, NumpyMetadata
 from awkward._nplikes.placeholder import PlaceholderArray
@@ -687,6 +688,24 @@ def _to_arrow(
             pyarrow, mask_node, validbytes, length, options
         )
 
+    def _to_cudf(self, cudf: Any, mask: Content | None, length: int):
+        cp = Cupy.instance()._module
+
+        assert mask is None  # this class has its own mask
+        if not self.lsb_order:
+            m = cp.flip(
+                cp.packbits(cp.flip(cp.unpackbits(cp.asarray(self._mask.data))))
+            )
+        else:
+            m = self._mask.data
+
+        if m.nbytes % 64:
+            m = cp.resize(m, ((m.nbytes // 64) + 1) * 64)
+        m = cudf.core.buffer.as_buffer(m)
+        inner = self._content._to_cudf(cudf, mask=None, length=length)
+        inner.set_base_mask(m)
+        return inner
+
     def _to_backend_array(self, allow_missing, backend):
         return self.to_ByteMaskedArray()._to_backend_array(allow_missing, backend)
 

src/awkward/contents/bytemaskedarray.py

@@ -12,6 +12,7 @@
 from awkward._layout import maybe_posaxis
 from awkward._meta.bytemaskedmeta import ByteMaskedMeta
 from awkward._nplikes.array_like import ArrayLike
+from awkward._nplikes.cupy import Cupy
 from awkward._nplikes.numpy import Numpy
 from awkward._nplikes.numpy_like import IndexType, NumpyMetadata
 from awkward._nplikes.placeholder import PlaceholderArray
@@ -1051,6 +1052,18 @@ def _to_arrow(
             options,
         )
 
+    def _to_cudf(self, cudf: Any, mask: Content | None, length: int):
+        cp = Cupy.instance()._module
+
+        assert mask is None  # this class has its own mask
+        m = cp.packbits(cp.asarray(self._mask), bitorder="little")
+        if m.nbytes % 64:
+            m = cp.resize(m, ((m.nbytes // 64) + 1) * 64)
+        m = cudf.core.buffer.as_buffer(m)
+        inner = self._content._to_cudf(cudf, mask=None, length=length)
+        inner.set_base_mask(m)
+        return inner
+
     def _to_backend_array(self, allow_missing, backend):
         return self.to_IndexedOptionArray64()._to_backend_array(allow_missing, backend)
 

src/awkward/contents/content.py

@@ -1010,6 +1010,10 @@ def _to_arrow(
     ):
         raise NotImplementedError
 
+    def _to_cudf(self, cudf: Any, mask: Content | None, length: int):
+        # prototype abstract signature
+        raise NotImplementedError
+
     def to_backend_array(
         self, allow_missing: bool = True, *, backend: Backend | str | None = None
     ):

src/awkward/contents/emptyarray.py

@@ -387,6 +387,15 @@ def _to_arrow(
             )
             return next._to_arrow(pyarrow, mask_node, validbytes, length, options)
 
+    def _to_cudf(self, cudf: Any, mask: Content | None, length: int):
+        dtype = np.dtype("float64")
+        next = ak.contents.NumpyArray(
+            numpy.empty(length, dtype=dtype),
+            parameters=self._parameters,
+            backend=self._backend,
+        )
+        return next._to_cudf(cudf, None, 0)
+
     @classmethod
     def _arrow_needs_option_type(cls):
         return True  # This overrides Content._arrow_needs_option_type