Kubernetes Operator: add support for GC #11021

Workflow file for this run

	# Copyright (C) 2020 Dremio
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	# Unifies main + PR workflow.
	#
	# The unified CI workflow consists of 2 "stages":
	# - Checks - test, intTest, NesQuEIT, etc
	# - Finalize - a "success" dummy job for PRs + a "save to github-cache" job for push-to-main
	#
	# Utilizes the Gradle build cache for all stages. The updated build cache
	# of the jobs in the checks stage are saved as artifacts (with the minimum
	# retention period). The updated build cache is pushed back to GigHub's
	# cache when the checks have successfully finished.

	name: CI build

	on:
	push:
	branches:
	- main
	- release-*
	paths-ignore:
	- 'LICENSE'
	- 'NOTICE'
	- '**.md'
	- '!site/**'
	- '.github/renovate.json5'
	- '.github/workflows/release*.yml'
	- '.github/workflows/check*.yml'
	- '.idea/**'
	- '.editorconfig'
	pull_request:
	types: [labeled, opened, synchronize, reopened]

	# For the main branch: let all CI runs complete, one after the other. This has a couple advantages:
	# * Site deployments happen in commit-order
	# * Saved Gradle cache are persisted in commit-order
	# * (Potentially) more GH runners available for PRs
	concurrency:
	# PRs: 1 CI run concurrently / older ones are cancelled
	# main branch: 1 CI run concurrently / no cancellation
	group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
	cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

	jobs:
	code-checks:
	name: CI Code Checks et al
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java

	# Needed for the Quarkus plugin - can likely go away once we use Quarkus 3 or newer
	- name: Bump Gradle daemon heap
	run: sed -i 's/-Xms.*/-Xms6G -Xmx6G -XX:MaxMetaspaceSize=1g \\/' gradle.properties

	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare

	- name: Gradle / Compile
	run: \|
	./gradlew \
	spotlessCheck \
	compileAll \
	-x :nessie-quarkus:compileAll \
	-x :nessie-server-admin-tool:compileAll \
	-x :nessie-events-quarkus:compileAll \
	-x :nessie-operator:compileAll \
	--scan

	- name: Gradle / Compile Quarkus
	run: \|
	# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
	./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll :nessie-operator:compileAll --scan \|\| \
	./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll :nessie-operator:compileAll --scan \|\| \
	./gradlew :nessie-quarkus:compileAll :nessie-server-admin-tool:compileAll :nessie-events-quarkus:compileAll :nessie-operator:compileAll --scan

	- name: Gradle / Code checks
	run: ./gradlew codeChecks --scan

	- name: Gradle / Assemble
	run: ./gradlew assemble --scan

	- name: Gradle / Publish to Maven local
	run: ./gradlew publishToMavenLocal --scan

	# This is a rather quick one and uses the output of 'publishToMavenLocal', which uses the
	# outputs of 'assemble'
	- name: Gradle / build tools integration tests
	run: ./gradlew buildToolsIntegrationTest

	- name: Save partial Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-save
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	with:
	job-name: 'code-checks'

	test:
	name: CI Test
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java

	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare

	- name: Gradle / test
	run: ./gradlew test :nessie-client:check -x :nessie-client:intTest -x :nessie-quarkus:test -x :nessie-server-admin-tool:test -x :nessie-events-quarkus:test -x :nessie-operator:test --scan

	- name: Capture Test Reports
	uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4
	if: ${{ failure() }}
	with:
	name: ci-test-reports
	path: \|
	*/build/reports/
	*/build/test-results/
	retention-days: 7

	- name: Save partial Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-save
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	with:
	job-name: 'test'

	test-quarkus:
	name: CI Test Quarkus
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java

	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare

	- name: Gradle / Test Quarkus Server
	run: \|
	# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
	./gradlew :nessie-quarkus:test --scan \|\| \
	./gradlew :nessie-quarkus:test --scan \|\| \
	./gradlew :nessie-quarkus:test --scan

	- name: Gradle / Test Quarkus Events
	run: \|
	# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
	./gradlew :nessie-events-quarkus:test --scan \|\| \
	./gradlew :nessie-events-quarkus:test --scan \|\| \
	./gradlew :nessie-events-quarkus:test --scan

	- name: Gradle / Test Quarkus Operator
	run: \|
	# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
	./gradlew :nessie-operator:test --scan \|\| \
	./gradlew :nessie-operator:test --scan \|\| \
	./gradlew :nessie-operator:test --scan

	- name: Dump quarkus.log
	if: ${{ failure() }}
	run: \|
	find . -path "**/build/quarkus.log" \| while read ql ; do
	echo "::group::Quarkus build log $ql"
	cat $ql
	echo "::endgroup::"
	done

	- name: Capture Test Reports
	uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4
	if: ${{ failure() }}
	with:
	name: ci-test-quarkus-reports
	path: \|
	**/build/quarkus.log
	*/build/reports/
	*/build/test-results/
	retention-days: 7

	- name: Save partial Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-save
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	with:
	job-name: 'test-quarkus'

	int-test:
	name: CI intTest
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	env:
	SPARK_LOCAL_IP: localhost
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java

	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare

	- name: Gradle / intTest
	run: \|
	echo "::group::Collect :nessie-versioned-storage projects"
	./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-persist- --output ../persist-prjs.txt --exclude
	echo "::endgroup::"

	echo "::group::Collect :nessie-versioned-persist projects"
	./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-storage- --output ../storage-prjs.txt --exclude
	echo "::endgroup::"

	echo "::group::Collect :nessie-spark-extensions projects"
	./gradlew :listProjectsWithPrefix --prefix :nessie-spark-ext --output ../spark-prjs.txt --exclude
	echo "::endgroup::"

	./gradlew intTest \
	-x :nessie-quarkus:intTest \
	-x :nessie-server-admin-tool:intTest \
	-x :nessie-events-quarkus:intTest \
	-x :nessie-operator:intTest \
	$(cat ../persist-prjs.txt) \
	$(cat ../storage-prjs.txt) \
	$(cat ../spark-prjs.txt) \
	--scan

	- name: Capture Test Reports
	uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4
	if: ${{ failure() }}
	with:
	name: ci-inttest-reports
	path: \|
	*/build/reports/
	*/build/test-results/
	retention-days: 7

	- name: Save partial Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-save
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	with:
	job-name: 'int-test'

	int-test-stores:
	name: CI intTest versioned/stores
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java

	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare

	- name: Gradle / intTest versioned/stores
	run: \|
	echo "::group::Collect :nessie-versioned-storage projects"
	./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-storage- --output ../storage-prjs.txt
	echo "::endgroup::"

	echo "::group::Collect :nessie-versioned-persist projects"
	./gradlew :listProjectsWithPrefix --prefix :nessie-versioned-persist- --output ../persist-prjs.txt
	echo "::endgroup::"

	./gradlew $(cat ../persist-prjs.txt) $(cat ../storage-prjs.txt) --scan

	- name: Capture Test Reports
	uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4
	if: ${{ failure() }}
	with:
	name: ci-inttest-stores-reports
	path: \|
	*/build/reports/
	*/build/test-results/
	retention-days: 7

	- name: Save partial Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-save
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	with:
	job-name: 'int-test-stores'

	int-test-integrations:
	name: CI intTest integrations
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	env:
	SPARK_LOCAL_IP: localhost
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java
	with:
	# Need Java 17 in addition to the default Java 21
	additional-java-version: 17

	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare

	- name: Gradle / intTest integrations
	run: \|
	echo "::group::Collect :nessie-spark-extensions projects"
	./gradlew :listProjectsWithPrefix --prefix :nessie-spark-ext --output ../spark-prjs.txt
	echo "::endgroup::"

	./gradlew $(cat ../spark-prjs.txt) --scan

	- name: Capture Test Reports
	uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4
	if: ${{ failure() }}
	with:
	name: ci-inttest-integrations-reports
	path: \|
	*/build/reports/
	*/build/test-results/
	retention-days: 7

	- name: Save partial Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-save
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	with:
	job-name: 'int-test-integrations'

	int-test-quarkus-server:
	name: CI intTest Quarkus Server
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java

	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare

	- name: Gradle / intTest Quarkus Server
	run:
	# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
	./gradlew :nessie-quarkus:intTest --scan \|\| \
	./gradlew :nessie-quarkus:intTest --scan \|\| \
	./gradlew :nessie-quarkus:intTest --scan

	- name: Dump quarkus.log
	if: ${{ failure() }}
	run: \|
	find . -path "**/build/quarkus.log" \| while read ql ; do
	echo "::group::Quarkus build log $ql"
	cat $ql
	echo "::endgroup::"
	done

	- name: Capture Test Reports
	uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4
	if: ${{ failure() }}
	with:
	name: ci-inttest-quarkus-server-reports
	path: \|
	**/build/quarkus.log
	*/build/reports/
	*/build/test-results/
	retention-days: 7

	- name: Save partial Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-save
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	with:
	job-name: 'int-test-quarkus-server'

	int-test-quarkus-tool:
	name: CI intTest Admin Tool
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java

	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare

	- name: Gradle / intTest Admin Tool
	run:
	# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
	./gradlew :nessie-server-admin-tool:intTest --scan \|\| \
	./gradlew :nessie-server-admin-tool:intTest --scan \|\| \
	./gradlew :nessie-server-admin-tool:intTest --scan

	- name: Dump quarkus.log
	if: ${{ failure() }}
	run: \|
	find . -path "**/build/quarkus.log" \| while read ql ; do
	echo "::group::Quarkus build log $ql"
	cat $ql
	echo "::endgroup::"
	done

	- name: Capture Test Reports
	uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4
	if: ${{ failure() }}
	with:
	name: ci-inttest-quarkus-tool-reports
	path: \|
	**/build/quarkus.log
	*/build/reports/
	*/build/test-results/
	retention-days: 7

	- name: Save partial Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-save
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	with:
	job-name: 'int-test-quarkus-tool'

	int-test-quarkus-events:
	name: CI intTest Quarkus Events
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java

	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare

	- name: Gradle / intTest Quarkus Events
	run:
	# 2 Retries - to mitigate https://github.com/gradle/gradle/issues/25751
	./gradlew :nessie-events-quarkus:intTest --scan \|\| \
	./gradlew :nessie-events-quarkus:intTest --scan \|\| \
	./gradlew :nessie-events-quarkus:intTest --scan

	- name: Dump quarkus.log
	if: ${{ failure() }}
	run: \|
	find . -path "**/build/quarkus.log" \| while read ql ; do
	echo "::group::Quarkus build log $ql"
	cat $ql
	echo "::endgroup::"
	done

	- name: Capture Test Reports
	uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4
	if: ${{ failure() }}
	with:
	name: ci-inttest-quarkus-events-reports
	path: \|
	**/build/quarkus.log
	*/build/reports/
	*/build/test-results/
	retention-days: 7

	- name: Save partial Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-save
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	with:
	job-name: 'int-test-quarkus-events'

	int-test-quarkus-operator:
	name: CI intTest Quarkus Operator
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java

	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare

	- name: Gradle / intTest Operator
	uses: gradle/actions/setup-gradle@v3
	with:
	arguments: \|
	:nessie-operator:intTest
	--scan

	- name: Dump quarkus.log
	if: ${{ failure() }}
	run: \|
	find . -path "**/build/quarkus.log" \| while read ql ; do
	echo "::group::Quarkus build log $ql"
	cat $ql
	echo "::endgroup::"
	done

	- name: Capture Test Reports
	uses: actions/upload-artifact@v4
	if: ${{ failure() }}
	with:
	name: ci-inttest-operator-reports
	path: \|
	**/build/quarkus.log
	*/build/reports/
	*/build/test-results/
	retention-days: 7

	- name: Save partial Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-save
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	with:
	job-name: 'int-test-quarkus-operator'

	determine-jobs:
	name: CI Determine jobs
	runs-on: ubuntu-22.04
	timeout-minutes: 5
	outputs:
	# Each "conditional" job has a mapped output here, also a "non-PR" case and a "PR" case
	# with label and globs (at the end of the script).
	docker: ${{ steps.determine.outputs.docker }}
	nesqueit: ${{ steps.determine.outputs.nesqueit }}
	steps:
	- name: install minimatch
	shell: bash
	run: npm install minimatch@"^7.4.3"
	- name: Determine conditional jobs to run
	id: determine
	uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
	with:
	script: \|
	const {Minimatch} = require("minimatch");

	async function getChangedFiles(pull_request) {
	const listFilesOptions = github.rest.pulls.listFiles.endpoint.merge({
	owner: pull_request.base.repo.owner.login,
	repo: pull_request.base.repo.name,
	pull_number: pull_request.number,
	});
	return github.paginate(listFilesOptions).then(resp => resp.map(f => f.filename))
	}

	function isMatch(changedFile, matchers) {
	for (const matcher of matchers) {
	if (matcher.match(changedFile)) {
	core.info("Match found for changed file " + changedFile);
	return true;
	}
	}
	core.info("No changed file matched the globs");
	return false;
	}

	function filesMatchAnyGlob(changedFiles, globs) {
	const matchers = globs.map(g => new Minimatch(g));
	for (const changedFile of changedFiles) {
	if (isMatch(changedFile, matchers)) {
	return true;
	}
	}
	return false;
	}

	function checkLabelOrGlobs(pr, changedFiles, labelName, globs) {
	core.info("Checking for " + labelName);

	// If the PR has the given label, then let the conditional job run.
	for (const label of pr.labels) {
	if (label.name === labelName) {
	core.info("PR has label " + labelName);
	return true;
	}
	}

	// For pull request, check the list of changed files against the given globs.
	// Let the conditional job run, if any of the globs matches and of the changed files.
	core.info("Checking changed files against globs " + globs);
	return filesMatchAnyGlob(changedFiles, globs);
	}

	///////////////////////////////////////////////////////////////////////////////////////
	// "Non PR" case:
	// Behavior of the conditional jobs when the event that triggered the workflow is not
	// for a pull request (usually a push-to-main).
	//
	if (!context.payload \|\| !context.payload.pull_request) {
	core.info("Not a pull-request, enabling all jobs.");

	// Yield default values for all event payload types, except pull_request
	core.setOutput('docker', true);
	core.setOutput('nesqueit', false); // handled in a scheduled job
	return true;
	}

	///////////////////////////////////////////////////////////////////////////////////////
	// "PR" case:
	// Determine the whether the conditional jobs shall run, based on a label name, which
	// has been manually added, or, if the label is not present, based on some globs to
	// check.

	const pr = context.payload.pull_request
	core.info("pull-request #" + pr.number);

	// Retrieves the full list of changed files for the PR.
	const changedFiles = await getChangedFiles(pr);

	core.startGroup('PR job check / Docker');
	core.setOutput('docker', checkLabelOrGlobs(pr, changedFiles,
	'pr-docker',
	['gradle/wrapper/', 'tools/dockerbuild/', 'helm/', '.github/']));
	core.endGroup();

	core.startGroup('PR job check / NesQuEIT');
	core.setOutput('nesqueit', checkLabelOrGlobs(pr, changedFiles,
	'pr-integrations',
	['gradle/wrapper/', 'integrations/', 'api/', '.github/']));
	core.endGroup();

	docker-testing:
	name: CI Docker and Helm checks
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	needs:
	- determine-jobs
	if: needs.determine-jobs.outputs.docker == 'true'
	steps:
	- name: Checkout
	uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	fetch-depth: 0
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java

	- name: Extract version
	run: \|
	VERSION="$(cat version.txt)"
	DOCKER_VERSION="${VERSION%-SNAPSHOT}"
	echo ${DOCKER_VERSION}
	echo "DOCKER_VERSION=${DOCKER_VERSION}" >> ${GITHUB_ENV}

	# Free disk space (minikube warning: "Docker is nearly out of disk space, which may cause
	# deployments to fail! (85% of capacity)")
	- name: Free disk space
	uses: ./.github/actions/free-disk-space

	- name: Setup Helm
	uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4
	with:
	# https://helm.sh/docs/topics/version_skew/
	version: 'v3.11.3'

	- name: Setup chart-testing
	uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1

	- name: Setup Python
	uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5
	with:
	python-version: '3.11'

	- name: Setup & Start Minikube
	uses: medyagh/setup-minikube@d8c0eb871f6f455542491d86a574477bd3894533 # v0.0.18
	with:
	# If required, use the matrix strategy against this option to test against multiple Kubernetes versions:
	kubernetes-version: stable
	cache: false
	# This _should_ work, but doesn't somehow (the image push fails). It errors out looking for a coredns pod
	# with these settings...
	#addons: 'registry'
	#insecure-registry: '192.168.0.0/16'

	- name: Setup Docker registry
	run: \|
	echo "::group::Get registry IP"
	DOCKER_REGISTRY="$(minikube ip)"
	echo "Registry IP is ${DOCKER_REGISTRY}"
	echo "::endgroup::"

	echo "::group::Update buildkitd_conf for 'docker buildx build'"
	# Use 'http' instead of 'https' during 'docker buildx build' in 'tools/dockerbuild/build-push-images.sh'
	cat <<EOF > ../buildkitd.toml
	[registry."${DOCKER_REGISTRY}:5000"]
	http = true
	EOF
	buildkitd_conf="$(pwd)/../buildkitd.toml"
	echo "::endgroup::"

	echo "::group::Update /etc/docker/daemon.json for 'docker pull'"
	cp /etc/docker/daemon.json ..
	cat ../daemon.json \| \
	jq ". + {\"insecure-registries\": [\"${DOCKER_REGISTRY}:5000\"]}" \| \
	sudo tee /etc/docker/daemon.json
	echo "::endgroup::"

	# minikube restart, because:
	# 1. required after docker daemon restart
	# 2. tweak the "registry addon" into the start command
	# 3. tweak the "insecure-registry" setting into the start command
	# Must delete the minikube cluster to let the insecure-registry setting take effect.
	# See 'Enabling Insecure Registries' in https://minikube.sigs.k8s.io/docs/handbook/registry/
	echo "::group::Stop minikube"
	minikube stop
	echo "::endgroup::"
	echo "::group::Delete minikube"
	minikube delete
	echo "::endgroup::"
	echo "::group::Restart docker daemon"
	sudo systemctl restart docker
	echo "::endgroup::"
	echo "::group::Start minikube"
	minikube start --insecure-registry="${DOCKER_REGISTRY}:5000" --addons=registry
	echo "::endgroup::"

	echo "BUILDX_CONFIG=--config ${buildkitd_conf}" >> ${GITHUB_ENV}
	echo "DOCKER_IMAGE=${DOCKER_REGISTRY}:5000/nessie-testing" >> ${GITHUB_ENV}
	echo "DOCKER_GC_IMAGE=${DOCKER_REGISTRY}:5000/nessie-gc-testing" >> ${GITHUB_ENV}
	echo "DOCKER_SERVER_ADMIN_IMAGE=${DOCKER_REGISTRY}:5000/nessie-server-admin-testing" >> ${GITHUB_ENV}
	echo "DOCKER_CLI_IMAGE=${DOCKER_REGISTRY}:5000/nessie-cli-testing" >> ${GITHUB_ENV}

	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare

	- name: Docker images publishing
	run: \|
	tools/dockerbuild/build-push-images.sh \
	-g ":nessie-quarkus" \
	-p "servers/quarkus-server" \
	-d "Dockerfile-server" \
	${DOCKER_IMAGE}

	tools/dockerbuild/build-push-images.sh \
	-g ":nessie-gc-tool" \
	-p "gc/gc-tool" \
	-d "Dockerfile-gctool" \
	${DOCKER_GC_IMAGE}

	tools/dockerbuild/build-push-images.sh \
	-g ":nessie-server-admin-tool" \
	-p "tools/server-admin" \
	-d "Dockerfile-admintool" \
	${DOCKER_SERVER_ADMIN_IMAGE}

	tools/dockerbuild/build-push-images.sh \
	-g ":nessie-cli" \
	-p "cli/cli" \
	-d "Dockerfile-cli" \
	${DOCKER_CLI_IMAGE}

	- name: Cleanup buildx
	run: \|
	docker buildx use default
	docker buildx rm nessiebuild

	- name: Check if expected Docker images exist
	run: \|
	docker pull ${DOCKER_IMAGE}:latest
	docker pull ${DOCKER_IMAGE}:latest-java
	docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION}
	docker pull ${DOCKER_IMAGE}:${DOCKER_VERSION}-java
	docker pull ${DOCKER_GC_IMAGE}:latest
	docker pull ${DOCKER_GC_IMAGE}:latest-java
	docker pull ${DOCKER_GC_IMAGE}:${DOCKER_VERSION}
	docker pull ${DOCKER_GC_IMAGE}:${DOCKER_VERSION}-java
	docker pull ${DOCKER_SERVER_ADMIN_IMAGE}:latest
	docker pull ${DOCKER_SERVER_ADMIN_IMAGE}:latest-java
	docker pull ${DOCKER_SERVER_ADMIN_IMAGE}:${DOCKER_VERSION}
	docker pull ${DOCKER_SERVER_ADMIN_IMAGE}:${DOCKER_VERSION}-java
	docker pull ${DOCKER_CLI_IMAGE}:latest
	docker pull ${DOCKER_CLI_IMAGE}:latest-java
	docker pull ${DOCKER_CLI_IMAGE}:${DOCKER_VERSION}
	docker pull ${DOCKER_CLI_IMAGE}:${DOCKER_VERSION}-java
	cat <<! >> $GITHUB_STEP_SUMMARY
	## Docker images

	\`\`\`
	$(docker images)
	\`\`\`
	!

	- name: Check if Server Docker Java image works
	run: \|
	docker run --rm --detach --name nessie ${DOCKER_IMAGE}:latest-java
	echo "Let Nessie Java Docker image run for one minute (to make sure it starts up fine)..."
	for i in {1..60}; do
	STATUS="$(docker container inspect nessie \| jq -r '.[0].State.Status')"
	if [[ ${STATUS} != "running" ]] ; then
	echo "Nessie Java Docker image stopped on its own ... a bug?" > /dev/stderr
	docker logs nessie
	cat <<! >> $GITHUB_STEP_SUMMARY
	## Nessie Java Docker image FAILED

	\`\`\`
	$(docker logs nessie)
	\`\`\`
	!
	exit 1
	fi
	sleep 1
	done
	echo "## Nessie Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY
	echo "Nessie Java Docker image smoke test: PASSED"
	docker stop nessie

	- name: Check if GC Tool Docker Java image works
	run: \|
	if docker run --rm --name nessie-gc ${DOCKER_GC_IMAGE}:latest-java --help \| grep -q "Usage: nessie-gc.jar"; then
	echo "## GC Tool Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY
	echo "GC Tool Java Docker image smoke test: PASSED"
	else
	echo "GC Tool Java Docker image smoke test: FAILED" > /dev/stderr
	cat <<! >> $GITHUB_STEP_SUMMARY
	## GC Tool Java Docker image FAILED

	\`\`\`
	$(docker logs nessie-gc)
	\`\`\`
	!
	exit 1
	fi

	- name: Check if Server Admin Tool Docker Java image works
	run: \|
	if docker run --rm --name nessie-server-admin ${DOCKER_SERVER_ADMIN_IMAGE}:latest-java --help \| grep -q "Usage: nessie-server-admin"; then
	echo "## Server Admin Tool Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY
	echo "Server Admin Tool Java Docker image smoke test: PASSED"
	else
	echo "Server Admin Tool Java Docker image smoke test: FAILED" > /dev/stderr
	cat <<! >> $GITHUB_STEP_SUMMARY
	## Server Admin Tool Java Docker image FAILED

	\`\`\`
	$(docker logs nessie-server-admin)
	\`\`\`
	!
	exit 1
	fi

	- name: Check if CLI Docker Java image works
	run: \|
	if docker run --rm --name nessie-cli ${DOCKER_CLI_IMAGE}:latest-java --help \| grep -q "Usage: nessie-cli.jar"; then
	echo "## CLI Java Docker image smoke test: PASSED" >> $GITHUB_STEP_SUMMARY
	echo "CLI Java Docker image smoke test: PASSED"
	else
	echo "CLI Java Docker image smoke test: FAILED" > /dev/stderr
	cat <<! >> $GITHUB_STEP_SUMMARY
	## CLI Java Docker image FAILED

	\`\`\`
	$(docker logs nessie-cli)
	\`\`\`
	!
	exit 1
	fi

	- name: Run chart-testing (list-changed)
	run: \|
	ct list-changed --target-branch ${{ github.event.repository.default_branch }}

	- name: Run chart-testing (lint)
	run: ct lint --debug --charts ./helm/nessie

	- name: Show pods
	run: kubectl get pods -A

	- name: Run chart-testing (install)
	run: \|
	echo "Using image: ${DOCKER_IMAGE}"
	echo " tag: ${DOCKER_VERSION}"

	ct install \
	--helm-extra-set-args "--set=image.repository=${DOCKER_IMAGE} --set=image.tag=${DOCKER_VERSION}" \
	--debug --charts ./helm/nessie

	nesqueit:
	name: CI NesQuEIT
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	needs:
	- determine-jobs
	# Only run NesQuEIT tests for PRs, if requested. This job can easily run for 30+ minutes.
	if: github.event_name == 'pull_request' && needs.determine-jobs.outputs.nesqueit == 'true'
	env:
	NESSIE_DIR: included-builds/nessie
	NESSIE_PATCH_REPOSITORY: ''
	NESSIE_PATCH_BRANCH: ''
	NESQUEIT_REPOSITORY: projectnessie/query-engine-integration-tests
	NESQUEIT_BRANCH: main
	ICEBERG_DIR: included-builds/iceberg
	ICEBERG_MAIN_REPOSITORY: apache/iceberg
	ICEBERG_MAIN_BRANCH: main
	ICEBERG_PATCH_REPOSITORY: snazy/iceberg
	ICEBERG_PATCH_BRANCH: iceberg-nesqueit
	SPARK_LOCAL_IP: localhost
	steps:
	- name: Prepare Git
	run: \|
	git config --global user.email "[email protected]"
	git config --global user.name "Integrations Testing [Bot]"

	- name: Checkout NeQuEIT repo
	uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	with:
	repository: ${{env.NESQUEIT_REPOSITORY}}
	ref: ${{env.NESQUEIT_BRANCH}}

	- name: Free disk space
	uses: ./.github/actions/free-disk-space

	- name: Setup runner
	uses: ./.github/actions/setup-runner
	with:
	more-memory: 'true'

	- name: Checkout and patch Nessie PR
	uses: ./.github/actions/patch-git
	with:
	name: Nessie
	local-dir: ${{env.NESSIE_DIR}}
	main-repository: ${{ env.GITHUB_REPOSITORY }}
	patch-repository: ${{env.NESSIE_PATCH_REPOSITORY}}
	patch-branch: ${{env.NESSIE_PATCH_BRANCH}}
	work-branch: nessie-integration-patched

	- name: Checkout and patch Iceberg
	uses: ./.github/actions/patch-git
	with:
	name: Nessie
	local-dir: ${{env.ICEBERG_DIR}}
	main-repository: ${{env.ICEBERG_MAIN_REPOSITORY}}
	main-branch: ${{env.ICEBERG_MAIN_BRANCH}}
	patch-repository: ${{env.ICEBERG_PATCH_REPOSITORY}}
	patch-branch: ${{env.ICEBERG_PATCH_BRANCH}}
	work-branch: iceberg-integration-patched

	- name: Setup JDK
	uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018 # v4
	with:
	distribution: 'temurin'
	# Java 17 or 21 required for Nessie build, Java 17 required for Iceberg build, Java 11 required for Flink & Presto
	java-version: \|
	11
	17
	21

	- name: Setup Gradle
	uses: gradle/actions/setup-gradle@16bf8bc8fe830fa669c3c9f914d3eb147c629707 # v4
	with:
	cache-read-only: true
	validate-wrappers: false

	- name: Iceberg Nessie test
	run: ./gradlew :iceberg:iceberg-nessie:test --scan

	- name: Nessie Spark 3.3 / 2.12 Extensions test
	run: ./gradlew :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.3_2.12:intTest --scan

	- name: Nessie Spark 3.4 / 2.13 Extensions test
	run: ./gradlew :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.13:intTest --scan

	- name: Nessie Spark 3.5 / 2.13 Extensions test
	run: ./gradlew :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:intTest --scan

	#- name: Publish Nessie + Iceberg to local Maven repo
	# run: ./gradlew publishLocal --scan
	#
	#- name: Gather locally published versions
	# run: \|
	# NESSIE_VERSION="$(cat included-builds/nessie/version.txt)"
	# ICEBERG_VERSION="$(cat included-builds/iceberg/build/iceberg-build.properties \| grep '^git.build.version=' \| cut -d= -f2)"
	# echo "NESSIE_VERSION=${NESSIE_VERSION}" >> ${GITHUB_ENV}
	# echo "ICEBERG_VERSION=${ICEBERG_VERSION}" >> ${GITHUB_ENV}
	# cat <<! >> $GITHUB_STEP_SUMMARY
	# ## Published versions
	# \| Published Nessie version \| Published Iceberg version \|
	# \| ------------------------ \| ------------------------- \|
	# \| ${NESSIE_VERSION} \| ${ICEBERG_VERSION} \|
	# !

	- name: Tools & Integrations tests
	run: ./gradlew intTest --scan

	site:
	name: CI Website
	runs-on: ubuntu-22.04
	timeout-minutes: 20
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java
	- name: Prepare Gradle build cache
	uses: ./.github/actions/ci-incr-build-cache-prepare
	- name: Setup Python
	uses: ./.github/actions/dev-tool-python
	with:
	python-version: '3.11'

	- name: Generate Static Site
	run: make build
	working-directory: ./site
	- name: Check code block indentation
	run: python3 check_code_indent.py
	working-directory: ./site
	- name: Deploy Static Site to GitHub
	if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'projectnessie/nessie'
	uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4
	with:
	external_repository: projectnessie/projectnessie.github.io
	publish_branch: main
	deploy_key: ${{ secrets.NESSIE_SITE_DEPLOY_KEY }}
	publish_dir: ./site/site
	cname: projectnessie.org

	store-cache:
	# Store the Gradle cache to GH cache as soon as all relevant Nessie/Gradle jobs have finished.
	name: CI Store Cache
	runs-on: ubuntu-22.04
	timeout-minutes: 30
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	needs:
	# Only include jobs that use Nessie's Gradle cache, especially excluding NesQuEIT, which
	# is a "very special" citizen and also not run for "main" CI, which does
	# not add anything to the Gradle cache that's not already produced by other jobs.
	- code-checks
	- test
	- test-quarkus
	- int-test
	- int-test-stores
	- int-test-integrations
	- int-test-quarkus-server
	- int-test-quarkus-tool
	- int-test-quarkus-events
	- int-test-quarkus-operator
	steps:
	- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
	- name: Setup runner
	uses: ./.github/actions/setup-runner
	- name: Setup Java, Gradle
	uses: ./.github/actions/dev-tool-java

	- name: Collect partial Gradle build caches
	uses: ./.github/actions/ci-incr-build-cache-prepare
	with:
	cache-read-only: false

	- name: Trigger Gradle home cleanup
	run: ./gradlew --no-daemon :showVersion

	# Note: the "Post Gradle invocation" archives the updated build cache.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Kubernetes Operator: add support for GC #11021

Workflow file

Kubernetes Operator: add support for GC #11021

Jobs

Run details

Workflow file for this run