Add partitioned output trace replayer #11470

Workflow file for this run

.github/workflows/scheduled.yml at fd1cbd7

	# Copyright (c) Facebook, Inc. and its affiliates.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	name: "Fuzzer Jobs"

	on:
	pull_request:
	paths:
	- "velox/**"
	- "!velox/docs/**"
	- "CMakeLists.txt"
	- "CMake/**"
	- "third_party/**"
	- "scripts/setup-ubuntu.sh"
	- "scripts/setup-helper-functions.sh"
	- ".github/workflows/scheduled.yml"

	push:
	branches:
	- "main"
	paths:
	- "velox/**"
	- "!velox/docs/**"
	- "CMakeLists.txt"
	- "CMake/**"
	- "third_party/**"
	- "scripts/setup-ubuntu.sh"
	- "scripts/setup-helper-functions.sh"
	- ".github/workflows/scheduled.yml"

	schedule:
	- cron: '0 3 * * *'

	workflow_dispatch:
	inputs:
	ref:
	description: 'Ref to checkout out'
	default: 'main'
	numThreads:
	description: 'Number of threads'
	default: 16
	maxHighMemJobs:
	description: 'Number of high memory jobs'
	default: 8
	maxLinkJobs:
	description: 'Maximum number of link jobs'
	default: 4
	extraCMakeFlags:
	description: 'Additional CMake flags'
	default: ''
	duration:
	description: 'Duration of fuzzer run in seconds'
	default: 1800

	defaults:
	run:
	shell: bash

	permissions:
	contents: read

	concurrency:
	# This will not cancel fuzzer runs on main (regardless of which trigger)
	# by making the commit sha part of the group but will use the branch
	# name in PRs to cancel on going runs on a new commit.
	group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.head_ref \|\| github.sha }}
	cancel-in-progress: true

	env:
	# Run for 15 minute on PRs
	DURATION: "${{ inputs.duration \|\| ( github.event_name != 'schedule' && 900 \|\| 1800 )}}"
	# minimize artifact duration for PRs, keep them a bit longer for nightly runs
	RETENTION: "${{ github.event_name == 'pull_request' && 1 \|\| 3 }}"

	jobs:
	compile:
	name: Build
	# prevent errors when forks ff their main branch
	if: ${{ github.repository == 'facebookincubator/velox' }}
	runs-on: 16-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:centos9
	timeout-minutes: 120
	env:
	CCACHE_DIR: "${{ github.workspace }}/.ccache"
	LINUX_DISTRO: "ubuntu"
	MAKEFLAGS: "NUM_THREADS=${{ inputs.numThreads \|\| 16 }} MAX_HIGH_MEM_JOBS=${{ inputs.maxHighMemJobs \|\| 8 }} MAX_LINK_JOBS=${{ inputs.maxLinkJobs \|\| 4 }}"

	defaults:
	run:
	shell: bash
	working-directory: velox
	outputs:
	presto_bias: ${{ steps.sig-check.outputs.presto_functions }}
	presto_error: ${{ steps.sig-check.outputs.presto_error }}
	spark_bias: ${{ steps.sig-check.outputs.spark_functions }}
	spark_error: ${{ steps.sig-check.outputs.spark_error }}
	presto_aggregate_bias: ${{ steps.sig-check.outputs.presto_aggregate_functions }}
	presto_aggregate_error: ${{ steps.sig-check.outputs.presto_aggregate_error }}

	steps:

	- name: Get latest commit from main
	if: ${{ github.event_name != 'schedule' }}
	working-directory: ${{ github.workspace }}
	env:
	GH_TOKEN: ${{ github.token }}
	id: get-head
	run: \|
	if [ '${{ github.event_name == 'push' }}' == "true" ]; then
	# get the parent commit of the current one to get the relevant function signatures
	head_main=$(gh api -q '.parents.[0].sha' '/repos/facebookincubator/velox/commits/${{ github.sha }}')
	else
	head_main=$(gh api -H "Accept: application/vnd.github.sha" /repos/facebookincubator/velox/commits/heads/main)
	fi
	echo "head_main=$head_main" >> $GITHUB_OUTPUT

	- name: Get Function Signature Stash
	uses: assignUser/stash/restore@v1
	id: get-sig
	with:
	path: /tmp/signatures
	key: function-signatures-${{ steps.get-head.outputs.head_main \|\| github.sha }}

	- name: Restore ccache
	uses: assignUser/stash/restore@v1
	with:
	path: "${{ env.CCACHE_DIR }}"
	key: ccache-fuzzer-centos

	- name: Fix git permissions
	working-directory: ${{ github.workspace }}
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: \|
	git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox
	git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox_main

	- name: Ensure Stash Dirs Exists
	working-directory: ${{ github.workspace }}
	run: \|
	mkdir -p '${{ env.CCACHE_DIR }}'
	mkdir -p /tmp/signatures

	- name: Checkout Main
	if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }}
	uses: actions/checkout@v4
	with:
	ref: ${{ steps.get-head.outputs.head_main \|\| 'main' }}
	path: velox_main

	- name: Build PyVelox
	if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }}
	working-directory: velox_main
	run: \|
	python3 -m venv .venv
	source .venv/bin/activate

	make python-build

	- name: Create Baseline Signatures
	if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' }}
	working-directory: velox_main
	run: \|
	source .venv/bin/activate
	python3 -m pip install deepdiff
	python3 scripts/signature.py export --spark /tmp/signatures/spark_signatures_main.json
	python3 scripts/signature.py export --presto /tmp/signatures/presto_signatures_main.json
	python3 scripts/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_main.json

	- name: Save Function Signature Stash
	if: ${{ github.event_name == 'pull_request' && steps.get-sig.outputs.stash-hit != 'true' }}
	uses: assignUser/stash/save@v1
	with:
	path: /tmp/signatures
	key: function-signatures-${{ steps.get-head.outputs.head_main }}

	- name: Checkout Contender
	uses: actions/checkout@v4
	with:
	path: velox
	submodules: 'recursive'
	ref: "${{ inputs.ref }}"

	- name: Zero Ccache Statistics
	run: \|
	ccache -sz

	- name: Build
	env:
	EXTRA_CMAKE_FLAGS: "-DVELOX_ENABLE_ARROW=ON -DVELOX_BUILD_PYTHON_PACKAGE=ON ${{ inputs.extraCMakeFlags }}"
	run: \|
	EXTRA_CMAKE_FLAGS="-DPYTHON_EXECUTABLE=$(which python3) $EXTRA_CMAKE_FLAGS"
	make debug

	- name: Ccache after
	run: ccache -s

	- name: Save ccache
	# see https://github.com/actions/upload-artifact/issues/543
	continue-on-error: true
	if: ${{ github.event_name != 'schedule' }}
	uses: assignUser/stash/save@v1
	with:
	path: "${{ env.CCACHE_DIR }}"
	key: ccache-fuzzer-centos

	- name: Build PyVelox
	if: ${{ github.event_name != 'schedule' }}
	env:
	VELOX_BUILD_DIR: "_build/debug"
	run: \|
	python3 -m venv .venv
	source .venv/bin/activate
	python3 -m pip install -e .

	- name: Create and test new function signatures
	if: ${{ github.event_name != 'schedule' }}
	id: sig-check
	run: \|
	source .venv/bin/activate
	python3 -m pip install deepdiff
	python3 scripts/signature.py gh_bias_check presto spark
	python3 scripts/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_contender.json
	python3 scripts/signature.py bias_aggregates /tmp/signatures/presto_aggregate_signatures_main.json \
	/tmp/signatures/presto_aggregate_signatures_contender.json /tmp/signatures/presto_aggregate_bias_functions \
	/tmp/signatures/presto_aggregate_errors

	- name: Upload Signature Artifacts
	if: ${{ github.event_name != 'schedule' }}
	uses: actions/upload-artifact@v4
	with:
	name: signatures
	path: /tmp/signatures
	retention-days: "${{ env.RETENTION }}"

	- name: Prepare signatures
	working-directory: /tmp/signatures
	if: ${{ github.event_name == 'push' }}
	run: \|
	# Remove irrelevant artifacts
	rm -f *_bias_functions
	rm -f *_signatures_main.json
	# Rename signature files as 'main' files
	for f in *_signatures_contender.json; do
	mv "$f" "${f/_contender.json/_main.json}"
	done

	- name: Save Function Signature Stash
	if: ${{ github.event_name == 'push' }}
	uses: assignUser/stash/save@v1
	with:
	path: /tmp/signatures
	key: function-signatures-${{ github.sha }}

	- name: Upload presto fuzzer
	uses: actions/upload-artifact@v4
	with:
	name: presto
	path: velox/_build/debug/velox/expression/fuzzer/velox_expression_fuzzer_test
	retention-days: "${{ env.RETENTION }}"

	- name: Upload spark expression fuzzer
	uses: actions/upload-artifact@v4
	with:
	name: spark_expression_fuzzer
	path: velox/_build/debug/velox/expression/fuzzer/spark_expression_fuzzer_test
	retention-days: "${{ env.RETENTION }}"

	- name: Upload spark aggregation fuzzer
	uses: actions/upload-artifact@v4
	with:
	name: spark_aggregation_fuzzer
	path: velox/_build/debug/velox/functions/sparksql/fuzzer/spark_aggregation_fuzzer_test
	retention-days: "${{ env.RETENTION }}"

	- name: Upload aggregation fuzzer
	uses: actions/upload-artifact@v4
	with:
	name: aggregation
	path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test
	retention-days: "${{ env.RETENTION }}"

	- name: Upload join fuzzer
	uses: actions/upload-artifact@v4
	with:
	name: join
	path: velox/_build/debug/velox/exec/tests/velox_join_fuzzer_test
	retention-days: "${{ env.RETENTION }}"

	- name: Upload exchange fuzzer
	uses: actions/upload-artifact@v4
	with:
	name: exchange
	path: velox/_build/debug//velox/exec/tests/velox_exchange_fuzzer_test
	retention-days: "${{ env.RETENTION }}"

	- name: Upload window fuzzer
	uses: actions/upload-artifact@v4
	with:
	name: window
	path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_window_fuzzer_test
	retention-days: "${{ env.RETENTION }}"

	- name: Upload row number fuzzer
	uses: actions/upload-artifact@v4
	with:
	name: row_number
	path: velox/_build/debug//velox/exec/tests/velox_row_number_fuzzer_test
	retention-days: "${{ env.RETENTION }}"

	- name: Upload writer fuzzer
	uses: actions/upload-artifact@v4
	with:
	name: writer
	path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_writer_fuzzer_test
	retention-days: "${{ env.RETENTION }}"

	presto-fuzzer-run:
	name: Presto Fuzzer
	if: ${{ needs.compile.outputs.presto_bias != 'true' }}
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 120
	steps:

	- uses: dorny/paths-filter@v3
	if: github.event_name == 'pull_request'
	id: changes
	with:
	filters: \|
	presto:
	- 'velox/expression/!(test)**'
	- 'velox/exec/!(test)**'
	- 'velox/common/!(test)**'
	- 'velox/core/!(test)**'
	- 'velox/vector/!(test)**'

	- name: Set presto specific fuzzer duration
	env:
	# Run for 30 minutes instead of 15, when files relevant to presto are touched
	pr_duration: "${{ steps.changes.outputs.presto == 'true' && 1800 \|\| 900 }}"
	# Run for 60 minutes if its a scheduled run
	other_duration: "${{ inputs.duration \|\| (github.event_name == 'push' && 1800 \|\| 3600) }}"
	is_pr: "${{ github.event_name == 'pull_request' }}"
	run: \|

	if [ "$is_pr" == "true" ]; then
	duration=$pr_duration
	else
	duration=$other_duration
	fi

	echo "DURATION=$duration" >> $GITHUB_ENV

	- name: Download presto fuzzer
	uses: actions/download-artifact@v4
	with:
	name: presto

	- name: Run Presto Fuzzer
	run: \|
	mkdir -p /tmp/fuzzer_repro/logs/
	chmod -R 777 /tmp/fuzzer_repro
	chmod +x velox_expression_fuzzer_test
	./velox_expression_fuzzer_test \
	--seed ${RANDOM} \
	--enable_variadic_signatures \
	--velox_fuzzer_enable_complex_types \
	--lazy_vector_generation_ratio 0.2 \
	--velox_fuzzer_enable_column_reuse \
	--velox_fuzzer_enable_expression_reuse \
	--max_expression_trees_per_step 2 \
	--retry_with_try \
	--enable_dereference \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/fuzzer_repro/logs \
	--repro_persist_path=/tmp/fuzzer_repro \
	&& echo -e "\n\nFuzzer run finished successfully."

	- name: Archive production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: presto-fuzzer-failure-artifacts
	path: \|
	/tmp/fuzzer_repro

	presto-bias-fuzzer:
	name: Presto Bias Fuzzer
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	if: ${{ needs.compile.outputs.presto_bias == 'true' }}
	timeout-minutes: 120
	steps:

	- name: Download presto expression fuzzer
	uses: actions/download-artifact@v4
	with:
	name: presto

	- name: Download Signatures
	uses: actions/download-artifact@v4
	with:
	name: signatures
	path: /tmp/signatures

	- name: Run Presto Expression Fuzzer
	run: \|
	ls /tmp/signatures
	mkdir -p /tmp/presto_bias_fuzzer_repro/logs/
	chmod -R 777 /tmp/presto_bias_fuzzer_repro
	chmod +x velox_expression_fuzzer_test
	./velox_expression_fuzzer_test \
	--seed ${RANDOM} \
	--lazy_vector_generation_ratio 0.2 \
	--assign_function_tickets $(cat /tmp/signatures/presto_bias_functions) \
	--duration_sec 3600 \
	--enable_variadic_signatures \
	--velox_fuzzer_enable_complex_types \
	--velox_fuzzer_enable_column_reuse \
	--velox_fuzzer_enable_expression_reuse \
	--max_expression_trees_per_step 2 \
	--retry_with_try \
	--enable_dereference \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/presto_bias_fuzzer_repro/logs \
	--repro_persist_path=/tmp/presto_bias_fuzzer_repro \
	&& echo -e "\n\nPresto Fuzzer run finished successfully."

	- name: Archive Spark expression production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: presto-bias-fuzzer-failure-artifacts
	path: \|
	/tmp/presto_bias_fuzzer_repro

	spark-aggregate-fuzzer-run:
	name: Spark Aggregate Fuzzer
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 60
	steps:

	- name: Download spark aggregation fuzzer
	uses: actions/download-artifact@v4
	with:
	name: spark_aggregation_fuzzer

	- name: Run Spark Aggregate Fuzzer
	run: \|
	mkdir -p /tmp/spark_aggregate_fuzzer_repro/logs/
	chmod -R 777 /tmp/spark_aggregate_fuzzer_repro
	chmod +x spark_aggregation_fuzzer_test
	./spark_aggregation_fuzzer_test \
	--seed ${RANDOM} \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/spark_aggregate_fuzzer_repro/logs \
	--repro_persist_path=/tmp/spark_aggregate_fuzzer_repro \
	&& echo -e "\n\nSpark Aggregation Fuzzer run finished successfully."

	- name: Archive Spark aggregate production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: spark-agg-fuzzer-failure-artifacts
	path: \|
	/tmp/spark_aggregate_fuzzer_repro

	spark-bias-fuzzer:
	name: Spark Bias Fuzzer
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	if: ${{ needs.compile.outputs.spark_bias == 'true' }}
	timeout-minutes: 120
	steps:

	- name: Download spark expression fuzzer
	uses: actions/download-artifact@v4
	with:
	name: spark_expression_fuzzer

	- name: Download Signatures
	uses: actions/download-artifact@v4
	with:
	name: signatures
	path: /tmp/signatures

	- name: Run Spark Expression Fuzzer
	run: \|
	ls /tmp/signatures
	mkdir -p /tmp/spark_bias_fuzzer_repro/logs/
	chmod -R 777 /tmp/spark_bias_fuzzer_repro
	chmod +x spark_expression_fuzzer_test
	./spark_expression_fuzzer_test \
	--seed ${RANDOM} \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/spark_bias_fuzzer_repro/logs \
	--assign_function_tickets $(cat /tmp/signatures/spark_bias_functions) \
	--repro_persist_path=/tmp/spark_bias_fuzzer_repro \
	&& echo -e "\n\nSpark Fuzzer run finished successfully."

	- name: Archive Spark expression production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: spark-fuzzer-failure-artifacts
	path: \|
	/tmp/spark_bias_fuzzer_repro

	spark-fuzzer:
	name: Spark Fuzzer
	if: ${{ needs.compile.outputs.spark_bias != 'true' }}
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 120
	steps:

	- name: Download spark expression fuzzer
	uses: actions/download-artifact@v4
	with:
	name: spark_expression_fuzzer

	- name: Run Spark Expression Fuzzer
	run: \|
	mkdir -p /tmp/spark_fuzzer_repro/logs/
	chmod -R 777 /tmp/spark_fuzzer_repro
	chmod +x spark_expression_fuzzer_test
	./spark_expression_fuzzer_test \
	--seed ${RANDOM} \
	--enable_variadic_signatures \
	--lazy_vector_generation_ratio 0.2 \
	--velox_fuzzer_enable_column_reuse \
	--velox_fuzzer_enable_expression_reuse \
	--max_expression_trees_per_step 2 \
	--retry_with_try \
	--enable_dereference \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/spark_fuzzer_repro/logs \
	--repro_persist_path=/tmp/spark_fuzzer_repro \
	&& echo -e "\n\nSpark Fuzzer run finished successfully."

	- name: Archive Spark expression production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: spark-fuzzer-failure-artifacts
	path: \|
	/tmp/spark_fuzzer_repro

	presto-java-join-fuzzer-run:
	name: Join Fuzzer
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	needs: compile
	timeout-minutes: 120
	env:
	CCACHE_DIR: "${{ github.workspace }}/.ccache/"
	LINUX_DISTRO: "centos"
	steps:

	- name: Download join fuzzer
	uses: actions/download-artifact@v4
	with:
	name: join

	- name: "Checkout Repo"
	uses: actions/checkout@v4
	with:
	path: velox
	submodules: 'recursive'
	ref: "${{ inputs.ref }}"

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: Run Join Fuzzer
	run: \|
	cd velox
	cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;'
	cd -
	mkdir -p /tmp/join_fuzzer_repro/logs/
	chmod -R 777 /tmp/join_fuzzer_repro
	chmod +x velox_join_fuzzer_test
	./velox_join_fuzzer_test \
	--seed ${RANDOM} \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/join_fuzzer_repro/logs \
	--presto_url=http://127.0.0.1:8080 \
	--req_timeout_ms=2000 \
	&& echo -e "\n\nJoin fuzzer run finished successfully."

	- name: Archive join production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: presto-sot-join-fuzzer-failure-artifacts
	path: \|
	/tmp/join_fuzzer_repro
	/tmp/server.log

	exchange-fuzzer-run:
	name: Exchange Fuzzer
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 120
	steps:

	- name: Download exchange fuzzer
	uses: actions/download-artifact@v4
	with:
	name: exchange

	- name: Run exchange Fuzzer
	run: \|
	cat /proc/sys/vm/max_map_count
	mkdir -p /tmp/exchange_fuzzer_repro/logs/
	chmod -R 777 /tmp/exchange_fuzzer_repro
	chmod +x velox_exchange_fuzzer_test
	./velox_exchange_fuzzer_test \
	--seed ${RANDOM} \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/exchange_fuzzer_repro/logs \
	--repro_path=/tmp/exchange_fuzzer_repro \
	&& echo -e "\n\Exchange fuzzer run finished successfully."

	- name: Archive Exchange production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: exchange-fuzzer-failure-artifacts
	path: \|
	/tmp/exchange_fuzzer_repro

	row-number-fuzzer-run:
	name: RowNumber Fuzzer
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 120
	steps:

	- name: Download row number fuzzer
	uses: actions/download-artifact@v4
	with:
	name: row_number

	- name: Run RowNumber Fuzzer
	run: \|
	cat /proc/sys/vm/max_map_count
	mkdir -p /tmp/row_fuzzer_repro/logs/
	chmod -R 777 /tmp/row_fuzzer_repro
	chmod +x velox_row_number_fuzzer_test
	./velox_row_number_fuzzer_test \
	--seed ${RANDOM} \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/row_fuzzer_repro/logs \
	&& echo -e "\n\Row number fuzzer run finished successfully."

	- name: Archive row number production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: row-fuzzer-failure-artifacts
	path: \|
	/tmp/row_fuzzer_repro
	presto-java-aggregation-fuzzer-run:
	name: Aggregation Fuzzer with Presto as source of truth
	needs: compile
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	timeout-minutes: 120
	env:
	CCACHE_DIR: "${{ github.workspace }}/.ccache/"
	LINUX_DISTRO: "centos"
	steps:

	- name: Download aggregation fuzzer
	uses: actions/download-artifact@v4
	with:
	name: aggregation

	- name: "Checkout Repo"
	uses: actions/checkout@v4
	with:
	path: velox
	submodules: 'recursive'
	ref: "${{ inputs.ref }}"

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox


	- name: "Run Aggregate Fuzzer"
	run: \|
	cd velox
	cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;'
	cd -
	mkdir -p /tmp/aggregate_fuzzer_repro/logs/
	chmod -R 777 /tmp/aggregate_fuzzer_repro
	chmod +x velox_aggregation_fuzzer_test
	./velox_aggregation_fuzzer_test \
	--seed ${RANDOM} \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/aggregate_fuzzer_repro/logs \
	--repro_persist_path=/tmp/aggregate_fuzzer_repro \
	--enable_sorted_aggregations=true \
	--presto_url=http://127.0.0.1:8080 \
	&& echo -e "\n\nAggregation fuzzer run finished successfully."

	- name: Archive aggregate production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: presto-sot-aggregate-fuzzer-failure-artifacts
	path: \|
	/tmp/aggregate_fuzzer_repro
	/tmp/server.log


	presto-bias-java-aggregation-fuzzer-run:
	name: Biased Aggregation Fuzzer with Presto as source of truth
	needs: compile
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	timeout-minutes: 120
	if: ${{ needs.compile.outputs.presto_aggregate_bias == 'true' }}
	env:
	CCACHE_DIR: "${{ github.workspace }}/.ccache/"
	LINUX_DISTRO: "centos"
	steps:

	- name: Download aggregation fuzzer
	uses: actions/download-artifact@v4
	with:
	name: aggregation

	- name: "Checkout Repo"
	uses: actions/checkout@v4
	with:
	path: velox
	submodules: 'recursive'
	ref: "${{ inputs.ref }}"

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: Download Signatures
	uses: actions/download-artifact@v4
	with:
	name: signatures
	path: /tmp/signatures

	- name: "Run Bias Aggregate Fuzzer"
	run: \|
	cd velox
	cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;'
	cd -
	mkdir -p /tmp/aggregate_fuzzer_repro/logs/
	chmod -R 777 /tmp/aggregate_fuzzer_repro
	chmod +x velox_aggregation_fuzzer_test
	echo "signatures folder"
	ls /tmp/signatures/
	echo "Biased functions:"
	cat /tmp/signatures/presto_aggregate_bias_functions
	echo "Running Fuzzer for $DURATION"
	./velox_aggregation_fuzzer_test \
	--seed ${RANDOM} \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/aggregate_fuzzer_repro/logs \
	--repro_persist_path=/tmp/aggregate_fuzzer_repro \
	--enable_sorted_aggregations=true \
	--only=$(cat /tmp/signatures/presto_aggregate_bias_functions) \
	--presto_url=http://127.0.0.1:8080 \
	&& echo -e "\n\nAggregation fuzzer run finished successfully."

	- name: Archive bias aggregate production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: presto-bias-sot-aggregate-fuzzer-failure-artifacts
	path: \|
	/tmp/aggregate_fuzzer_repro
	/tmp/server.log

	surface-signature-errors:
	name: Signature Changes
	if: ${{ github.event_name != 'schedule' }}
	needs: compile
	runs-on: ubuntu-latest
	steps:
	- name: Download Signatures
	uses: actions/download-artifact@v4
	with:
	name: signatures
	path: /tmp/signatures

	- name: Surface Presto function signature errors
	if: ${{ needs.compile.outputs.presto_error == 'true' }}
	run: \|
	cat /tmp/signatures/presto_errors
	exit 1

	- name: Surface Aggregate function signature errors
	if: ${{ needs.compile.outputs.presto_aggregate_error == 'true' }}
	run: \|
	cat /tmp/signatures/presto_aggregate_errors
	exit 1

	presto-java-window-fuzzer-run:
	name: Window Fuzzer with Presto as source of truth
	needs: compile
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	timeout-minutes: 120
	env:
	CCACHE_DIR: "${{ github.workspace }}/.ccache/"
	LINUX_DISTRO: "centos"
	steps:

	- name: Download window fuzzer
	uses: actions/download-artifact@v4
	with:
	name: window

	- name: "Checkout Repo"
	uses: actions/checkout@v4
	with:
	path: velox
	submodules: 'recursive'
	ref: "${{ inputs.ref }}"

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox


	- name: "Run Window Fuzzer"
	run: \|
	cd velox
	cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;'
	cd -
	mkdir -p /tmp/window_fuzzer_repro/logs/
	chmod -R 777 /tmp/window_fuzzer_repro
	chmod +x velox_window_fuzzer_test
	./velox_window_fuzzer_test \
	--seed ${RANDOM} \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/window_fuzzer_repro/logs \
	--repro_persist_path=/tmp/window_fuzzer_repro \
	--enable_window_reference_verification \
	--presto_url=http://127.0.0.1:8080 \
	&& echo -e "\n\nWindow fuzzer run finished successfully."

	- name: Archive window production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: presto-sot-window-fuzzer-failure-artifacts
	path: \|
	/tmp/window_fuzzer_repro
	/tmp/server.log

	presto-java-writer-fuzzer-run:
	name: Writer Fuzzer with Presto as source of truth
	needs: compile
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	timeout-minutes: 120
	env:
	CCACHE_DIR: "${{ github.workspace }}/.ccache/"
	LINUX_DISTRO: "centos"
	steps:

	- name: Download writer fuzzer
	uses: actions/download-artifact@v4
	with:
	name: writer

	- name: "Checkout Repo"
	uses: actions/checkout@v4
	with:
	path: velox
	submodules: 'recursive'
	ref: "${{ inputs.ref }}"

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: "Run Writer Fuzzer"
	run: \|
	cd velox
	cp ./scripts/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	echo "jvm config content:"
	cat $PRESTO_HOME/etc/jvm.config
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	ls -lR /var/log
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --version
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA hive.tpch;'
	cd -
	mkdir -p /tmp/writer_fuzzer_repro/logs/
	chmod -R 777 /tmp/writer_fuzzer_repro
	chmod +x velox_writer_fuzzer_test
	./velox_writer_fuzzer_test \
	--seed ${RANDOM} \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--req_timeout_ms 60000 \
	--log_dir=/tmp/writer_fuzzer_repro/logs \
	--presto_url=http://127.0.0.1:8080 \
	&& echo -e "\n\Writer fuzzer run finished successfully."

	- name: Archive writer production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@v4
	with:
	name: presto-sot-writer-fuzzer-failure-artifacts
	path: \|
	/tmp/writer_fuzzer_repro
	/tmp/server.log
	/var/log

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add partitioned output trace replayer #11470

Workflow file

Add partitioned output trace replayer #11470

Jobs

Run details

Workflow file for this run