From 57f5ff6c11269b55d4f39c9c05a09580af3ccaa2 Mon Sep 17 00:00:00 2001 From: Casey Artner Date: Fri, 23 Aug 2024 12:06:35 -0400 Subject: [PATCH] fix(pyspark): suppress errors for unsupported Databricks serverless compute properties (#9830) --- ibis/backends/pyspark/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ibis/backends/pyspark/__init__.py b/ibis/backends/pyspark/__init__.py index 0ea49d20b2bc..bcdde50a2810 100644 --- a/ibis/backends/pyspark/__init__.py +++ b/ibis/backends/pyspark/__init__.py @@ -181,7 +181,19 @@ def do_connect( # local time to UTC with microsecond resolution. # https://spark.apache.org/docs/latest/sql-pyspark-pandas-with-arrow.html#timestamp-with-time-zone-semantics self._session.conf.set("spark.sql.session.timeZone", "UTC") - self._session.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") + + # Databricks Serverless compute only supports limited properties + # and any attempt to set unsupported properties will result in an error. + # https://docs.databricks.com/en/spark/conf.html + try: + from pyspark.errors.exceptions.connect import SparkConnectGrpcException + except ImportError: + # Use a dummy class for when spark connect is not available + class SparkConnectGrpcException(Exception): + pass + + with contextlib.suppress(SparkConnectGrpcException): + self._session.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") for key, value in kwargs.items(): self._session.conf.set(key, value)