fix: protect ak.to_parquet against memory explosion when args are swapped. #2881

Workflow file for this run

	name: Docs
	on:
	push:
	branches:
	- main
	pull_request:
	release:
	types: [released]
	workflow_dispatch:

	concurrency:
	group: 'docs-${{ github.head_ref \|\| github.run_id }}'
	cancel-in-progress: true

	env:
	PYTHON_VERSION: "3.10.6"
	SOURCE_DATE_EPOCH: "1668811211"

	jobs:
	awkward-cpp-wasm:
	name: Build C++ WASM
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/checkout@v3
	with:
	submodules: true

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "${{ env.PYTHON_VERSION }}"

	- name: Generate build files
	run: pipx run nox -s prepare -- --headers --signatures

	- name: Cache wheel
	id: cache-awkward-cpp-wasm-wheel
	uses: actions/cache@v3
	with:
	path: ./awkward-cpp/dist
	key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-awkward-cpp-wasm-${{ hashFiles('awkward-cpp/**') }}

	- name: Install EMSDK
	uses: mymindstorm/setup-emsdk@v12
	if: steps.cache-awkward-cpp-wasm-wheel.outputs.cache-hit != 'true'
	with:
	version: 3.1.14

	- name: Build wheel
	if: steps.cache-awkward-cpp-wasm-wheel.outputs.cache-hit != 'true'
	run: \|
	# pyodide-build doesn't work out of the box with pipx
	python3 -m pip install pyyaml pyodide-build==0.21.0
	export CMAKE_ARGS="-DEMSCRIPTEN=1"
	pyodide build --exports whole_archive
	working-directory: awkward-cpp

	- name: Upload wheel
	uses: actions/upload-artifact@v3
	with:
	name: awkward-cpp-wasm
	path: awkward-cpp/dist/awkward*wasm32.whl

	awkward-cpp-x86-64:
	runs-on: ubuntu-22.04
	name: Build C++ x86
	steps:
	- uses: actions/checkout@v3
	with:
	submodules: true

	# TODO: remove this once mamba-org/mamba#1726 is fixed
	# and replace with `-r requirements.txt` in a
	# non-template `environment.yml`
	- name: Template environment.yml
	run: pipx run --spec cogapp cog -o environment.yml environment.yml.cog
	working-directory: docs

	# Technically this give us an environment that is incompatible with
	# the wheel built in the awkward-x86-64 job if the environments
	# solve with different external library versions. By default,
	# ROOT uses cxx-compiler too, so hopefully this won't be an issue
	- name: Setup Python via Conda
	uses: mamba-org/setup-micromamba@v1
	with:
	# Cache invalidates daily by default
	cache-environment: true
	cache-downloads: true
	environment-file: docs/environment.yml
	create-args: >-
	python=${{ env.PYTHON_VERSION }}
	doxygen

	- name: Generate build files
	run: pipx run nox -s prepare -- --headers --signatures

	- name: Cache wheel
	id: cache-awkward-cpp-x86-64-wheel
	uses: actions/cache@v3
	with:
	path: ./awkward-cpp/dist
	key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-awkward-x86-64-wasm-${{ hashFiles('awkward-cpp/**') }}

	- name: Build wheel
	if: steps.cache-awkward-cpp-x86-64-wheel.outputs.cache-hit != 'true'
	run: pipx run build -w ./awkward-cpp

	- name: Upload wheel
	uses: actions/upload-artifact@v3
	with:
	name: awkward-cpp-x86-64
	path: awkward-cpp/dist/awkward*.whl

	awkward:
	runs-on: ubuntu-22.04
	name: Build Python
	steps:
	- uses: actions/checkout@v3
	with:
	submodules: true

	- name: Generate build files
	run: pipx run nox -s prepare -- --headers --signatures

	- name: Build wheel
	run: pipx run build -w

	- name: Upload wheel
	uses: actions/upload-artifact@v3
	with:
	name: awkward
	path: dist/awkward*.whl


	build-docs:
	runs-on: ubuntu-22.04
	needs: [awkward-cpp-wasm, awkward-cpp-x86-64, awkward]
	name: Build Docs
	defaults:
	run:
	# Ensure conda env is activated
	shell: "bash -l {0}"
	env:
	DOCS_CANONICAL_VERSION: main
	steps:
	- uses: actions/checkout@v3

	# TODO: remove this once mamba-org/mamba#1726 is fixed
	# and replace with `-r requirements.txt` in a
	# non-template `environment.yml`
	- name: Template environment.yml
	run: pipx run --spec cogapp cog -o environment.yml environment.yml.cog
	working-directory: docs

	# Technically this give us an environment that is incompatible with
	# the wheel built in the awkward-x86-64 job if the environments
	# solve with different external library versions. By default,
	# ROOT uses cxx-compiler too, so hopefully this won't be an issue
	- name: Setup Python via Conda
	uses: mamba-org/setup-micromamba@v1
	with:
	# Cache invalidates daily by default
	cache-environment: true
	cache-downloads: true
	environment-file: docs/environment.yml
	create-args: >-
	python=${{ env.PYTHON_VERSION }}
	doxygen

	- name: Download awkward wheel
	uses: actions/download-artifact@v3
	with:
	name: awkward
	path: dist

	- name: Copy awkward wheel to JupyterLite
	run: \|
	mkdir -p docs/lite/pypi/
	cp dist/awkward*.whl docs/lite/pypi/

	- name: Download & copy awkward-cpp WASM wheel to JupyterLite
	uses: actions/download-artifact@v3
	with:
	name: awkward-cpp-wasm
	path: docs/lite/pypi

	- name: Download awkward-cpp x86_64 wheel
	uses: actions/download-artifact@v3
	with:
	name: awkward-cpp-x86-64
	path: dist

	- name: Install awkward and awkward-cpp wheels
	run: python -m pip install dist/awkward*.whl --force-reinstall --no-deps

	- name: Generate build files
	run: pipx run nox -s prepare -- --docs --headers

	- name: Generate C++ documentation
	run: doxygen
	working-directory: awkward-cpp/docs

	- name: Copy C++ documentation
	run: cp -r awkward-cpp/docs/html/ docs/_static/doxygen

	- name: Enable analytics & version selector
	if: github.event_name == 'push' \|\| github.event_name == 'release'
	run: \|
	echo "DOCS_REPORT_ANALYTICS=1" >> $GITHUB_ENV
	echo "DOCS_SHOW_VERSION=1" >> $GITHUB_ENV

	- name: Set version to main
	if: github.event_name == 'push'
	run: \|
	echo "DOCS_VERSION=main" >> $GITHUB_ENV

	- name: Generate Python documentation
	run: sphinx-build -M html . _build/ -T
	working-directory: docs

	- name: Upload docs artefact
	uses: actions/upload-artifact@v3
	with:
	name: docs
	path: docs/_build/html

	- name: Upload Jupyter Book cache
	uses: actions/upload-artifact@v3
	with:
	name: doctrees
	path: docs/_build/doctrees

	- name: Upload Jupyter Book cache
	uses: actions/upload-artifact@v3
	with:
	name: jupyter-cache
	path: docs/_build/.jupyter_cache

	branch-preview:
	runs-on: ubuntu-22.04
	needs: [build-docs]
	name: Deploy Branch Preview
	# We can only deploy for PRs on host repo
	if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
	permissions:
	id-token: write
	contents: read
	env:
	S3_BUCKET: "preview.awkward-array.org"
	DEPLOY_URL: "http://preview.awkward-array.org.s3-website.us-east-1.amazonaws.com"
	environment:
	name: docs-preview
	url: ${{ env.DEPLOY_URL }}/${{ github.head_ref }}
	steps:
	- uses: actions/checkout@v3
	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v2
	with:
	aws-region: eu-west-2
	role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE }}
	- name: Download rendered docs
	uses: actions/download-artifact@v3
	with:
	name: docs
	path: built-docs
	- name: Sync artefacts
	run: \|
	aws s3 sync built-docs/ "s3://${S3_BUCKET}/${{ github.head_ref }}"

	deploy:
	runs-on: ubuntu-22.04
	needs: [ build-docs ]
	# We can only deploy for PRs on host repo
	if: github.event_name == 'push' \|\| github.event_name == 'release'
	name: Deploy
	permissions:
	id-token: write
	contents: read
	env:
	S3_BUCKET: "awkward-array.org"
	PRODUCTION_URL: "http://awkward-array.org"
	CLOUDFRONT_ID: "EFM4QVENUIXHS"
	environment:
	name: docs-prod
	url: ${{ env.PRODUCTION_URL }}${{ steps.sync-main.outputs.path \|\| steps.sync-stable.outputs.path }}
	steps:
	- uses: actions/checkout@v3
	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v2
	with:
	aws-region: eu-west-2
	role-to-assume: ${{ secrets.AWS_DEPLOY_ROLE }}
	- name: Download rendered docs
	uses: actions/download-artifact@v3
	with:
	name: docs
	path: built-docs
	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.11"
	# Pushes to main trigger latest
	- name: Sync `main`
	if: github.event_name == 'push'
	id: sync-main
	run: \|
	aws s3 sync built-docs/ "s3://${S3_BUCKET}/doc/main/"
	aws cloudfront create-invalidation --distribution-id "${CLOUDFRONT_ID}" \
	--paths "/doc/main*"
	echo "path=/doc/main" >> $GITHUB_OUTPUT
	# Releases trigger versions
	- name: Sync `stable`
	if: github.event_name == 'release'
	id: sync-stable
	run: \|
	# Take only leading version
	version=$(echo "${GITHUB_REF_NAME}" \| sed -n -E "s/v?([0-9]+\.[0-9]+)\.[0-9]+/\1/p")
	aws s3 cp docs/switcher.json "s3://${S3_BUCKET}/doc/"
	aws s3 sync built-docs/ "s3://${S3_BUCKET}/doc/$version/"
	aws s3 sync built-docs/ "s3://${S3_BUCKET}/doc/stable/"
	aws cloudfront create-invalidation --distribution-id "${CLOUDFRONT_ID}" \
	--paths "/doc/$version" "/doc/stable" "/doc/switcher.json"
	echo "path=/doc/stable" >> $GITHUB_OUTPUT

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

fix: protect ak.to_parquet against memory explosion when args are swapped. #2881

Workflow file

fix: protect ak.to_parquet against memory explosion when args are swapped. #2881

Jobs

Run details

Workflow file for this run