Add support for Python3.13, fix running test on localhost, and move regression tests to pytest

README.md

@@ -6,9 +6,9 @@
   to you under the Apache License, Version 2.0 (the
   "License"); you may not use this file except in compliance
   with the License.  You may obtain a copy of the License at
- 
+
    http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing,
   software distributed under the License is distributed on an
   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,7 +19,7 @@
 
 # Apache Polaris (incubating)
 
-Apache Polaris™ is an open-source, fully-featured catalog for Apache Iceberg™. It implements Iceberg's 
+Apache Polaris™ is an open-source, fully-featured catalog for Apache Iceberg™. It implements Iceberg's
 [REST API](https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml),
 enabling seamless multi-engine interoperability across a wide range of platforms, including Apache Doris™, Apache Flink®,
 Apache Spark™, StarRocks, and Trino.
@@ -36,18 +36,18 @@ for contribution guidelines.
 
 [dev-list-subscribe]: mailto:[email protected]
 
-## Building and Running 
+## Building and Running
 
 Apache Polaris is organized into the following modules:
 - `polaris-core` - The main Polaris entity definitions and core business logic
 - `polaris-server` - The Polaris REST API server
 - `polaris-eclipselink` - The Eclipselink implementation of the MetaStoreManager interface
- 
+
 Apache Polaris is built using Gradle with Java 21+ and Docker 27+.
 - `./gradlew build` - To build and run tests. Make sure Docker is running, as the integration tests depend on it.
 - `./gradlew assemble` - To skip tests.
 - `./gradlew test` - To run unit tests and integration tests.
-- `./gradlew runApp` - To run the Polaris server locally on localhost:8181. 
+- `./gradlew runApp` - To run the Polaris server locally on localhost:8181.
 - `./regtests/run_spark_sql.sh` - To connect from Spark SQL. Here are some example commands to run in the Spark SQL shell:
 ```sql
 create database db1;
@@ -83,7 +83,7 @@ Running in Kubernetes
 - `kubectl describe deployment polaris-deployment -n polaris` - To troubleshoot if things aren't working as expected.
 
 Running regression tests
-- `./regtests/run.sh` - To run regression tests in another terminal.
+- `./regtests/run.sh` - To execute regression tests locally, ensure that the Polaris server is started on localhost in a separate terminal window before initiating the tests.
 - `docker compose up --build --exit-code-from regtest` - To run regression tests in a Docker environment.
 
 Building docs
@@ -101,5 +101,5 @@ Apache Polaris is under the Apache License Version 2.0. See the [LICENSE](LICENS
 ## ASF Incubator disclaimer
 
 Apache Polaris™ is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
- 
+
 <sub>Apache&reg;, Apache Polaris&trade;, Apache Iceberg&trade;, Apache Spark&trade; are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries.</sub>

docker-compose.yml

@@ -52,7 +52,6 @@ services:
       AWS_TEST_ENABLED: $AWS_TEST_ENABLED
       AWS_STORAGE_BUCKET: $AWS_STORAGE_BUCKET
       AWS_ROLE_ARN: $AWS_ROLE_ARN
-      AWS_TEST_BASE: $AWS_TEST_BASE
       GCS_TEST_ENABLED: $GCS_TEST_ENABLED
       GCS_TEST_BASE: $GCS_TEST_BASE
       GOOGLE_APPLICATION_CREDENTIALS: $GOOGLE_APPLICATION_CREDENTIALS

regtests/Dockerfile

@@ -17,41 +17,25 @@
 # under the License.
 #
 
-FROM docker.io/apache/spark:3.5.4-python3
+FROM docker.io/python:3.13-slim
 ARG POLARIS_HOST=polaris
 ENV POLARIS_HOST=$POLARIS_HOST
-ENV SPARK_HOME=/opt/spark
 
-USER root
-RUN apt update
-RUN apt-get install -y diffutils wget curl python3.8-venv
-RUN mkdir -p /home/spark &&  \
-    chown -R spark /home/spark && \
-    mkdir -p /tmp/polaris-regtests && \
-    chown -R spark /tmp/polaris-regtests
-RUN mkdir /opt/spark/conf && chmod -R 777 /opt/spark/conf
+RUN groupadd polaris && \
+    useradd -m -g polaris -s /bin/bash polaris
 
-USER spark
-ENV PYTHONPATH="${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH"
+WORKDIR /home/polaris/regtests
 
-# Copy and run setup.sh separately so that test sources can change, but the setup script run is still cached
-WORKDIR /home/spark/regtests
-COPY ./setup.sh /home/spark/regtests/setup.sh
-COPY ./pyspark-setup.sh /home/spark/regtests/pyspark-setup.sh
-COPY ./client/python /home/spark/regtests/client/python
-COPY ./polaris-reg-test /home/spark/polaris
+COPY ./polaris-reg-test /home/polaris/polaris
+COPY . /home/polaris/regtests
 
-RUN python3 -m venv /home/spark/polaris-venv && \
-    . /home/spark/polaris-venv/bin/activate && \
-    pip install poetry==1.5.0 && \
-    deactivate \
-RUN ./setup.sh
+RUN apt-get update && \
+    apt-get install -y procps openjdk-17-jdk && \
+    rm -rf /var/lib/apt/lists/* && \
+    chown -R polaris:polaris /home/polaris && \
+    mkdir /tmp/polaris-regtests && \
+    chown -R polaris:polaris /tmp/polaris-regtests
 
-COPY --chown=spark . /home/spark/regtests
+USER polaris
 
-# /home/spark/regtests might not be writable in all situations, see https://github.com/apache/polaris/pull/205
-USER root
-RUN chmod -R go+rwx /home/spark/regtests
-USER spark
-
-ENTRYPOINT ["./run.sh"]
+ENTRYPOINT ["/home/polaris/regtests/run.sh"]

regtests/README.md

@@ -60,7 +60,6 @@ AWS_ACCESS_KEY_ID=<your_access_key>
 AWS_SECRET_ACCESS_KEY=<your_secret_key>
 AWS_STORAGE_BUCKET=<your_s3_bucket>
 AWS_ROLE_ARN=<iam_role_with_access_to_bucket>
-AWS_TEST_BASE=s3://<your_s3_bucket>/<any_path>
 
 # GCP variables
 GCS_TEST_ENABLED=true
@@ -69,6 +68,8 @@ GOOGLE_APPLICATION_CREDENTIALS=/tmp/credentials/<your_credentials.json>
 
 # Azure variables
 AZURE_TEST_ENABLED=true
+AZURE_CLIENT_ID=<your_access_key>
+AZURE_CLIENT_SECRET=<your_secret_key>
 AZURE_TENANT_ID=<your_tenant_id>
 AZURE_DFS_TEST_BASE=abfss://<container-name>@<storage-account-name>.dfs.core.windows.net/<any_path>
 AZURE_BLOB_TEST_BASE=abfss://<container-name>@<storage-account-name>.blob.core.windows.net/<any_path>