diff --git a/.github/workflows/build_main.yml b/.github/workflows/build_main.yml
index f1088a5b6..8a0660111 100644
--- a/.github/workflows/build_main.yml
+++ b/.github/workflows/build_main.yml
@@ -19,7 +19,7 @@ jobs:
python: [ 3.10.12 ]
numpy: [ 1.22.4 ]
gdal: [ 3.4.1 ]
- spark: [ 3.4.0 ]
+ spark: [ 3.4.1 ]
R: [ 4.2.2 ]
steps:
- name: checkout code
diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml
index 30d62cb3c..1b766a2e3 100644
--- a/.github/workflows/build_python.yml
+++ b/.github/workflows/build_python.yml
@@ -15,7 +15,7 @@ jobs:
python: [ 3.10.12 ]
numpy: [ 1.22.4 ]
gdal: [ 3.4.1 ]
- spark: [ 3.4.0 ]
+ spark: [ 3.4.1 ]
R: [ 4.2.2 ]
steps:
- name: checkout code
diff --git a/.github/workflows/build_r.yml b/.github/workflows/build_r.yml
index 986ca744d..4c6ad732f 100644
--- a/.github/workflows/build_r.yml
+++ b/.github/workflows/build_r.yml
@@ -16,7 +16,7 @@ jobs:
python: [ 3.10.12 ]
numpy: [ 1.22.4 ]
gdal: [ 3.4.1 ]
- spark: [ 3.4.0 ]
+ spark: [ 3.4.1 ]
R: [ 4.2.2 ]
steps:
- name: checkout code
diff --git a/.github/workflows/build_scala.yml b/.github/workflows/build_scala.yml
index ba464022e..b804b491e 100644
--- a/.github/workflows/build_scala.yml
+++ b/.github/workflows/build_scala.yml
@@ -14,7 +14,7 @@ jobs:
python: [ 3.10.12 ]
numpy: [ 1.22.4 ]
gdal: [ 3.4.1 ]
- spark: [ 3.4.0 ]
+ spark: [ 3.4.1 ]
R: [ 4.2.2 ]
steps:
- name: checkout code
diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml
index 251ead879..30396423e 100644
--- a/.github/workflows/pypi-release.yml
+++ b/.github/workflows/pypi-release.yml
@@ -12,7 +12,7 @@ jobs:
python: [ 3.10.12 ]
numpy: [ 1.22.4 ]
gdal: [ 3.4.1 ]
- spark: [ 3.4.0 ]
+ spark: [ 3.4.1 ]
R: [ 4.2.2 ]
steps:
- name: checkout code
diff --git a/.gitignore b/.gitignore
index 975675c69..7d56ac217 100644
--- a/.gitignore
+++ b/.gitignore
@@ -159,3 +159,9 @@ spark-warehouse
.DS_Store
.Rproj.user
docker/.m2/
+/python/notebooks/
+/scripts/m2/
+/python/mosaic_test/
+/python/checkpoint/
+/python/checkpoint-new/
+/scripts/docker/docker-build/ubuntu-22-spark-3.4/Dockerfile
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 56e7bff22..3a9ff6687 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,12 @@
+## v0.4.3 [DBR 13.3 LTS]
+- Pyspark requirement removed from python setup.cfg as it is supplied by DBR
+- Python version limited to "<3.11,>=3.10" for DBR
+- iPython dependency limited to "<8.11,>=7.4.2" for both DBR and keplergl-jupyter
+- Expanded support for fuse-based checkpointing (persisted raster storage), managed through:
+ - spark config 'spark.databricks.labs.mosaic.raster.use.checkpoint' in addition to 'spark.databricks.labs.mosaic.raster.checkpoint'.
+ - python: `mos.enable_gdal(spark, with_checkpoint_path=path)`.
+ - scala: `MosaicGDAL.enableGDALWithCheckpoint(spark, path)`.
+
## v0.4.2 [DBR 13.3 LTS]
- Geopandas now fixed to "<0.14.4,>=0.14" due to conflict with minimum numpy version in geopandas 0.14.4.
- H3 python changed from "==3.7.0" to "<4.0,>=3.7" to pick up patches.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0684cca0b..ea82645e9 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -84,7 +84,7 @@ The repository is structured as follows:
## Test & build Mosaic
Given that DBR 13.3 is Ubuntu 22.04, we recommend using docker,
-see [mosaic-docker.sh](https://github.com/databrickslabs/mosaic/blob/main/scripts/mosaic-docker.sh).
+see [mosaic-docker.sh](https://github.com/databrickslabs/mosaic/blob/main/scripts/docker/mosaic-docker.sh).
### Scala JAR
diff --git a/R/sparkR-mosaic/sparkrMosaic/DESCRIPTION b/R/sparkR-mosaic/sparkrMosaic/DESCRIPTION
index e5e1de0f1..2ea7718f9 100644
--- a/R/sparkR-mosaic/sparkrMosaic/DESCRIPTION
+++ b/R/sparkR-mosaic/sparkrMosaic/DESCRIPTION
@@ -1,6 +1,6 @@
Package: sparkrMosaic
Title: SparkR bindings for Databricks Mosaic
-Version: 0.4.2
+Version: 0.4.3
Authors@R:
person("Robert", "Whiffin", , "robert.whiffin@databricks.com", role = c("aut", "cre")
)
diff --git a/R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION b/R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION
index 69619e72a..4ce2f2be1 100644
--- a/R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION
+++ b/R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION
@@ -1,6 +1,6 @@
Package: sparklyrMosaic
Title: sparklyr bindings for Databricks Mosaic
-Version: 0.4.2
+Version: 0.4.3
Authors@R:
person("Robert", "Whiffin", , "robert.whiffin@databricks.com", role = c("aut", "cre")
)
diff --git a/R/sparklyr-mosaic/tests.R b/R/sparklyr-mosaic/tests.R
index 9bad8f6fa..2cf3b8fca 100644
--- a/R/sparklyr-mosaic/tests.R
+++ b/R/sparklyr-mosaic/tests.R
@@ -9,7 +9,7 @@ library(sparklyr.nested)
spark_home <- Sys.getenv("SPARK_HOME")
spark_home_set(spark_home)
-install.packages("sparklyrMosaic_0.4.2.tar.gz", repos = NULL)
+install.packages("sparklyrMosaic_0.4.3.tar.gz", repos = NULL)
library(sparklyrMosaic)
# find the mosaic jar in staging
diff --git a/docs/source/api/rasterio-udfs.rst b/docs/source/api/rasterio-udfs.rst
index 339223c58..46cf46f77 100644
--- a/docs/source/api/rasterio-udfs.rst
+++ b/docs/source/api/rasterio-udfs.rst
@@ -248,7 +248,6 @@ depending on your needs.
def write_raster(raster, driver, file_id, fuse_dir):
from io import BytesIO
from pathlib import Path
- from pyspark.sql.functions import udf
from rasterio.io import MemoryFile
import numpy as np
import rasterio
diff --git a/docs/source/usage/install-gdal.rst b/docs/source/usage/install-gdal.rst
index cc1801a27..12d1217d0 100644
--- a/docs/source/usage/install-gdal.rst
+++ b/docs/source/usage/install-gdal.rst
@@ -112,7 +112,7 @@ Here are spark session configs available for raster, e.g. :code:`spark.conf.set(
- Checkpoint location, e.g. :ref:`rst_maketiles`
* - spark.databricks.labs.mosaic.raster.use.checkpoint
- "false"
- - Checkpoint for session, in 0.4.2+
+ - Checkpoint for session, in 0.4.3+
* - spark.databricks.labs.mosaic.raster.tmp.prefix
- "" (will use "/tmp")
- Local directory for workers
diff --git a/pom.xml b/pom.xml
index 8d4a8abf7..d41e60a04 100644
--- a/pom.xml
+++ b/pom.xml
@@ -146,27 +146,6 @@
-
- org.scoverage
- scoverage-maven-plugin
- 2.0.2
-
-
- scoverage-report
- package
-
- check
- report-only
-
-
-
-
- ${minimum.coverage}
- true
- ${scala.version}
- skipTests=false
-
-
net.alchim31.maven
@@ -277,8 +256,45 @@
2.12.10
2.12
- 3.4.0
- 0.4.2
+ 3.4.1
+ 0.4.3
+
+
+
+
+ org.scoverage
+ scoverage-maven-plugin
+ 2.0.2
+
+
+ scoverage-report
+ package
+
+ check
+ report-only
+
+
+
+
+ ${minimum.coverage}
+ true
+ ${scala.version}
+ skipTests=false
+
+
+
+
+
+
+
+ skipScoverage
+
+ 2.12.10
+ 2.12
+ 3.4.1
+ 0.4.3
+ 1.8
+ 1.8
diff --git a/python/mosaic/__init__.py b/python/mosaic/__init__.py
index 22fcceb1f..a8cd7e73a 100644
--- a/python/mosaic/__init__.py
+++ b/python/mosaic/__init__.py
@@ -4,4 +4,4 @@
from .models import SpatialKNN
from .readers import read
-__version__ = "0.4.2"
+__version__ = "0.4.3"
diff --git a/python/mosaic/api/__init__.py b/python/mosaic/api/__init__.py
index 6377554d3..9c8ce7a81 100644
--- a/python/mosaic/api/__init__.py
+++ b/python/mosaic/api/__init__.py
@@ -1,7 +1,7 @@
from .accessors import *
from .aggregators import *
from .constructors import *
-from .enable import enable_mosaic
+from .enable import enable_mosaic, get_install_version, get_install_lib_dir
from .functions import *
from .fuse import *
from .predicates import *
diff --git a/python/mosaic/api/enable.py b/python/mosaic/api/enable.py
index 5a96d226e..e7e66f974 100644
--- a/python/mosaic/api/enable.py
+++ b/python/mosaic/api/enable.py
@@ -1,3 +1,5 @@
+import importlib.metadata
+import importlib.resources
import warnings
from IPython.core.getipython import get_ipython
@@ -72,24 +74,25 @@ def enable_mosaic(
if not jar_autoattach:
spark.conf.set("spark.databricks.labs.mosaic.jar.autoattach", "false")
print("...set 'spark.databricks.labs.mosaic.jar.autoattach' to false")
+ config.jar_autoattach=False
if jar_path is not None:
spark.conf.set("spark.databricks.labs.mosaic.jar.path", jar_path)
print(f"...set 'spark.databricks.labs.mosaic.jar.path' to '{jar_path}'")
+ config.jar_path=jar_path
if log_info:
spark.sparkContext.setLogLevel("info")
+ config.log_info=True
+
+ # Config global objects
+ # - add MosaicContext after MosaicLibraryHandler
config.mosaic_spark = spark
- _ = MosaicLibraryHandler(config.mosaic_spark, log_info=log_info)
- config.mosaic_context = MosaicContext(config.mosaic_spark)
-
- # Register SQL functions
- optionClass = getattr(spark._sc._jvm.scala, "Option$")
- optionModule = getattr(optionClass, "MODULE$")
- config.mosaic_context._context.register(
- spark._jsparkSession, optionModule.apply(None)
- )
-
- isSupported = config.mosaic_context._context.checkDBR(spark._jsparkSession)
- if not isSupported:
+ _ = MosaicLibraryHandler(spark, log_info=log_info)
+ config.mosaic_context = MosaicContext(spark)
+ config.mosaic_context.jRegister(spark)
+
+ _jcontext = config.mosaic_context.jContext()
+ is_supported = _jcontext.checkDBR(spark._jsparkSession)
+ if not is_supported:
# unexpected - checkDBR returns true or throws exception
print("""WARNING: checkDBR returned False.""")
@@ -104,3 +107,31 @@ def enable_mosaic(
from mosaic.utils.kepler_magic import MosaicKepler
config.ipython_hook.register_magics(MosaicKepler)
+
+
+def get_install_version() -> str:
+ """
+ :return: mosaic version installed
+ """
+ return importlib.metadata.version("databricks-mosaic")
+
+
+def get_install_lib_dir(override_jar_filename=None) -> str:
+ """
+ This is looking for the library dir under site packages using the jar name.
+ :return: located library dir.
+ """
+ v = get_install_version()
+ jar_filename = f"mosaic-{v}-jar-with-dependencies.jar"
+ if override_jar_filename:
+ jar_filename = override_jar_filename
+ with importlib.resources.path("mosaic.lib", jar_filename) as p:
+ return p.parent.as_posix()
+
+
+def refresh_context():
+ """
+ Refresh mosaic context, using previously configured information.
+ - This is needed when spark configs change, such as for checkpointing.
+ """
+ config.mosaic_context.jContextReset()
diff --git a/python/mosaic/api/gdal.py b/python/mosaic/api/gdal.py
index 57c0e10f6..d7e88a47e 100644
--- a/python/mosaic/api/gdal.py
+++ b/python/mosaic/api/gdal.py
@@ -1,9 +1,16 @@
+from .enable import refresh_context
from .fuse import SetupMgr
+from mosaic.config import config
from pyspark.sql import SparkSession
import subprocess
-__all__ = ["setup_gdal", "enable_gdal"]
+__all__ = [
+ "setup_gdal", "enable_gdal",
+ "update_checkpoint_path", "set_checkpoint_on", "set_checkpoint_off",
+ "has_context", "is_use_checkpoint", "get_checkpoint_path", "reset_checkpoint",
+ "get_checkpoint_path_default"
+]
def setup_gdal(
@@ -50,7 +57,7 @@ def setup_gdal(
return setup_mgr.configure(test_mode=test_mode)
-def enable_gdal(spark: SparkSession) -> None:
+def enable_gdal(spark: SparkSession, with_checkpoint_path: str = None) -> None:
"""
Enable GDAL at runtime on a cluster with GDAL installed using init script,
e.g. generated by setup_gdal() or setup_fuse_install() call.
@@ -59,18 +66,25 @@ def enable_gdal(spark: SparkSession) -> None:
----------
spark : pyspark.sql.SparkSession
The active SparkSession.
+ with_checkpoint_path : str
+ Optional, enable checkpointing;
+ default is None.
Returns
-------
-
"""
try:
- sc = spark.sparkContext
- mosaicGDALObject = getattr(
- sc._jvm.com.databricks.labs.mosaic.gdal, "MosaicGDAL"
- )
- mosaicGDALObject.enableGDAL(spark._jsparkSession)
+ if with_checkpoint_path is not None:
+ spark.conf.set("spark.databricks.labs.mosaic.raster.use.checkpoint", "true")
+ spark.conf.set("spark.databricks.labs.mosaic.raster.checkpoint", with_checkpoint_path)
+ refresh_context()
+ config.mosaic_context.jEnableGDAL(spark, with_checkpoint_path=with_checkpoint_path)
+ else:
+ config.mosaic_context.jEnableGDAL(spark)
+
print("GDAL enabled.\n")
+ if with_checkpoint_path:
+ print(f"checkpoint path '{with_checkpoint_path}' configured for this session.")
result = subprocess.run(["gdalinfo", "--version"], stdout=subprocess.PIPE)
print(result.stdout.decode() + "\n")
except Exception as e:
@@ -78,9 +92,94 @@ def enable_gdal(spark: SparkSession) -> None:
"GDAL not enabled. Mosaic with GDAL requires that GDAL be installed on the cluster.\n"
)
print(
- "You can run setup_gdal() or setup_fuse_install() to generate the init script for install GDAL install.\n"
+ "You can run `setup_gdal()` or `setup_fuse_install()` to generate the init script for GDAL install.\n"
)
print(
- "After the init script is generated, you need to add the init script to your cluster and restart to complete the setup.\n"
+ "After the init script is generated, you need to add it to your cluster and restart.\n"
)
print("Error: " + str(e))
+
+
+def update_checkpoint_path(spark: SparkSession, path: str):
+ """
+ Change the checkpoint location; does not adjust checkpoint on/off (stays as-is).
+ :param spark: session to use.
+ :param path: new path.
+ """
+ spark.conf.set("spark.databricks.labs.mosaic.raster.checkpoint", path)
+ refresh_context()
+ config.mosaic_context.jUpdateCheckpointPath(spark,path)
+
+
+def set_checkpoint_off(spark: SparkSession):
+ """
+ Turn off checkpointing.
+ :param spark: session to use.
+ """
+ spark.conf.set("spark.databricks.labs.mosaic.raster.use.checkpoint", "false")
+ refresh_context()
+ config.mosaic_context.jSetCheckpointOff(spark)
+
+
+def set_checkpoint_on(spark: SparkSession):
+ """
+ Turn on checkpointing, will use the configured path.
+ :param spark: session to use.
+ """
+ spark.conf.set("spark.databricks.labs.mosaic.raster.use.checkpoint", "true")
+ refresh_context()
+ config.mosaic_context.jSetCheckpointOn(spark)
+
+
+def reset_checkpoint(spark: SparkSession):
+ """
+ Go back to defaults.
+ - spark conf unset for use checkpoint (off)
+ - spark conf unset for checkpoint path
+ :param spark: session to use.
+ """
+ spark.conf.set("spark.databricks.labs.mosaic.raster.use.checkpoint", "false")
+ spark.conf.set("spark.databricks.labs.mosaic.raster.checkpoint", get_checkpoint_path_default())
+ refresh_context()
+ config.mosaic_context.jResetCheckpoint(spark)
+
+
+#################################################################
+# GETTERS
+#################################################################
+
+
+def has_context() -> bool:
+ """
+ This is run on the driver, assumes enable.py already invoked.
+ :return: True if the mosaic context has been initialized;
+ otherwise False.
+ """
+ try:
+ return config.mosaic_context.has_context()
+ except Exception as e:
+ return False
+
+
+def is_use_checkpoint() -> bool:
+ """
+ This is run on the driver, assumes enable.py already invoked.
+ :return: checkpoint on/off; otherwise exception.
+ """
+ return config.mosaic_context.is_use_checkpoint()
+
+
+def get_checkpoint_path() -> str:
+ """
+ This is run on the driver, assumes enable.py already invoked.
+ :return: checkpoint path or exception.
+ """
+ return config.mosaic_context.get_checkpoint_path()
+
+
+def get_checkpoint_path_default() -> str:
+ """
+ This is run on the driver, assumes enable.py already invoked.
+ :return: default checkpoint path.
+ """
+ return config.mosaic_context.get_checkpoint_path_default()
diff --git a/python/mosaic/config/config.py b/python/mosaic/config/config.py
index a59979be6..16048e4d8 100644
--- a/python/mosaic/config/config.py
+++ b/python/mosaic/config/config.py
@@ -10,3 +10,6 @@
display_handler: DisplayHandler
ipython_hook: InteractiveShell
notebook_utils = None
+jar_path: str = None
+jar_autoattach: bool = True
+log_info: bool = False
diff --git a/python/mosaic/core/mosaic_context.py b/python/mosaic/core/mosaic_context.py
index 85b15ab6b..7edf25cae 100644
--- a/python/mosaic/core/mosaic_context.py
+++ b/python/mosaic/core/mosaic_context.py
@@ -1,5 +1,4 @@
from typing import Any
-
from py4j.java_gateway import JavaClass, JavaObject
from py4j.protocol import Py4JJavaError
from pyspark.sql import SparkSession
@@ -17,29 +16,20 @@ class MosaicContext:
def __init__(self, spark: SparkSession):
sc = spark.sparkContext
- self._mosaicContextClass = getattr(
- sc._jvm.com.databricks.labs.mosaic.functions, "MosaicContext"
- )
+
+ self._mosaicContextClass = getattr(sc._jvm.com.databricks.labs.mosaic.functions, "MosaicContext")
self._mosaicPackageRef = getattr(sc._jvm.com.databricks.labs.mosaic, "package$")
self._mosaicPackageObject = getattr(self._mosaicPackageRef, "MODULE$")
- self._mosaicGDALObject = getattr(
- sc._jvm.com.databricks.labs.mosaic.gdal, "MosaicGDAL"
- )
- self._indexSystemFactory = getattr(
- sc._jvm.com.databricks.labs.mosaic.core.index, "IndexSystemFactory"
- )
+ self._mosaicGDALObject = getattr(sc._jvm.com.databricks.labs.mosaic.gdal, "MosaicGDAL")
+ self._indexSystemFactory = getattr(sc._jvm.com.databricks.labs.mosaic.core.index, "IndexSystemFactory")
try:
- self._geometry_api = spark.conf.get(
- "spark.databricks.labs.mosaic.geometry.api"
- )
+ self._geometry_api = spark.conf.get("spark.databricks.labs.mosaic.geometry.api")
except Py4JJavaError as e:
self._geometry_api = "JTS"
try:
- self._index_system = spark.conf.get(
- "spark.databricks.labs.mosaic.index.system"
- )
+ self._index_system = spark.conf.get("spark.databricks.labs.mosaic.index.system")
except Py4JJavaError as e:
self._index_system = "H3"
@@ -48,15 +38,97 @@ def __init__(self, spark: SparkSession):
except Py4JJavaError as e:
self._raster_api = "GDAL"
+ # singleton on the java side
+ # - access dynamically
IndexSystem = self._indexSystemFactory.getIndexSystem(self._index_system)
GeometryAPIClass = getattr(self._mosaicPackageObject, self._geometry_api)
+ self._context = self._mosaicContextClass.build(IndexSystem, GeometryAPIClass())
+ def jContext(self):
+ """
+ :return: dynamic getter for jvm MosaicContext object
+ """
+ return self._context
+
+ def jContextReset(self):
+ """
+ Reset the MosaicContext jContext().
+ - This requires a re-init essentially.
+ - Needed sometimes for checkpointing.
+ """
+ self._mosaicContextClass.reset()
+ IndexSystem = self._indexSystemFactory.getIndexSystem(self._index_system)
+ GeometryAPIClass = getattr(self._mosaicPackageObject, self._geometry_api)
self._context = self._mosaicContextClass.build(IndexSystem, GeometryAPIClass())
def invoke_function(self, name: str, *args: Any) -> MosaicColumn:
+ """
+ use jvm context to invoke function.
+ :param name: name of function.
+ :param args: any passed args.
+ :return: MosaicColumn.
+ """
func = getattr(self._context.functions(), name)
return MosaicColumn(func(*args))
+ def jRegister(self, spark: SparkSession):
+ """
+ Register SQL expressions.
+ - the jvm functions for checkpointing handle after initial invoke
+ by enable.py.
+ :param spark: session to use.
+ """
+ optionClass = getattr(spark._sc._jvm.scala, "Option$")
+ optionModule = getattr(optionClass, "MODULE$")
+ self._context.register(spark._jsparkSession, optionModule.apply(None))
+
+ def jResetCheckpoint(self, spark: SparkSession):
+ """
+ Go back to defaults.
+ - spark conf unset for use checkpoint (off)
+ - spark conf unset for checkpoint path
+ :param spark: session to use.
+ """
+ self._mosaicGDALObject.resetCheckpoint(spark._jsparkSession)
+
+ def jEnableGDAL(self, spark: SparkSession, with_checkpoint_path: str = None):
+ """
+ Enable GDAL, assumes regular enable already called.
+ :param spark: session to use.
+ :param with_checkpoint_path: optional checkpoint path, default is None.
+ """
+ if with_checkpoint_path:
+ self._mosaicGDALObject.enableGDALWithCheckpoint(spark._jsparkSession, with_checkpoint_path)
+ else:
+ self._mosaicGDALObject.enableGDAL(spark._jsparkSession)
+
+
+ def jUpdateCheckpointPath(self, spark: SparkSession, path: str):
+ """
+ Change the checkpoint location; does not adjust checkpoint on/off (stays as-is).
+ :param spark: session to use.
+ :param path: new path.
+ """
+ self._mosaicGDALObject.updateCheckpointPath(spark._jsparkSession, path)
+
+ def jSetCheckpointOff(self, spark: SparkSession):
+ """
+ Turn off checkpointing.
+ :param spark: session to use.
+ """
+ self._mosaicGDALObject.setCheckpointOff(spark._jsparkSession)
+
+ def jSetCheckpointOn(self, spark: SparkSession):
+ """
+ Turn on checkpointing, will use the configured path.
+ :param spark: session to use.
+ """
+ self._mosaicGDALObject.setCheckpointOn(spark._jsparkSession)
+
+ #################################################################
+ # PROPERTY ACCESSORS + GETTERS
+ #################################################################
+
@property
def geometry_api(self):
return self._geometry_api
@@ -65,5 +137,14 @@ def geometry_api(self):
def index_system(self):
return self._index_system
- def enable_gdal(self, spark: SparkSession):
- return self._mosaicGDALObject.enableGDAL(spark._jsparkSession)
+ def is_use_checkpoint(self) -> bool:
+ return self._mosaicGDALObject.isUseCheckpoint()
+
+ def get_checkpoint_path(self) -> str:
+ return self._mosaicGDALObject.getCheckpointPath()
+
+ def get_checkpoint_path_default(self) -> str:
+ return self._mosaicGDALObject.getCheckpointPathDefault()
+
+ def has_context(self) -> bool:
+ return self._context is not None
diff --git a/python/setup.cfg b/python/setup.cfg
index 298663a38..4ffd3dc94 100644
--- a/python/setup.cfg
+++ b/python/setup.cfg
@@ -16,13 +16,12 @@ classifiers =
[options]
packages = find:
-python_requires = >=3.10.0
+python_requires = <3.11,>=3.10
install_requires =
geopandas<0.14.4,>=0.14
h3<4.0,>=3.7
- ipython>=7.22.0
+ ipython<8.11,>=7.4.2
keplergl==0.3.2
- pyspark<3.5,>=3.4
[options.package_data]
mosaic =
diff --git a/python/test/context.py b/python/test/context.py
index d118955b3..e910bf629 100644
--- a/python/test/context.py
+++ b/python/test/context.py
@@ -6,4 +6,5 @@
import mosaic.api as api
import mosaic.readers as readers
import mosaic.api.raster as rst
+from mosaic.config import config
from mosaic.core import MosaicContext, MosaicLibraryHandler
diff --git a/python/test/test_checkpoint.py b/python/test/test_checkpoint.py
new file mode 100644
index 000000000..b756e46ad
--- /dev/null
+++ b/python/test/test_checkpoint.py
@@ -0,0 +1,88 @@
+from .context import api
+from .utils import MosaicTestCaseWithGDAL
+import os
+
+class TestCheckpoint(MosaicTestCaseWithGDAL):
+ def setUp(self) -> None:
+ return super().setUp()
+
+ def test_all(self):
+ self.assertEqual(
+ self.spark.conf.get("spark.databricks.labs.mosaic.test.mode"), "true",
+ "spark should have TEST_MODE set.")
+
+ # - context
+ self.assertIsNotNone(self.get_context(), "python context should exist.")
+ self.assertTrue(self.get_context().has_context(), "jvm context should be initialized.")
+
+ # - path
+ self.assertEqual(
+ self.get_context().get_checkpoint_path(), self.check_dir,
+ "checkpoint path should equal dir.")
+ self.assertEqual(
+ self.get_context().get_checkpoint_path(),
+ self.spark.conf.get("spark.databricks.labs.mosaic.raster.checkpoint"),
+ "checkpoint path should equal spark conf.")
+
+ # - checkpoint on
+ api.gdal.set_checkpoint_on(self.spark) # <- important to call from api.gdal
+ self.assertTrue(self.get_context().is_use_checkpoint(), "context should be configured on.")
+ result = (
+ self.generate_singleband_raster_df()
+ .withColumn("rst_boundingbox", api.rst_boundingbox("tile"))
+ .withColumn("tile", api.rst_clip("tile", "rst_boundingbox"))
+ )
+ result.write.format("noop").mode("overwrite").save()
+ self.assertEqual(result.count(), 1)
+ tile = result.select("tile").first()[0]
+ raster = tile['raster']
+ self.assertIsInstance(raster, str, "raster type should be string.")
+
+ # - update path
+ api.gdal.update_checkpoint_path(self.spark, self.new_check_dir) # <- important to call from api.gdal
+ self.assertEqual(
+ self.get_context().get_checkpoint_path(), self.new_check_dir,
+ "context should be configured on.")
+ self.assertTrue(os.path.exists(self.new_check_dir), "new check dir should exist.")
+ result = (
+ self.generate_singleband_raster_df()
+ .withColumn("rst_boundingbox", api.rst_boundingbox("tile"))
+ .withColumn("tile", api.rst_clip("tile", "rst_boundingbox"))
+ )
+ result.write.format("noop").mode("overwrite").save()
+ self.assertEqual(result.count(), 1)
+ tile = result.select("tile").first()[0]
+ raster = tile['raster']
+ self.assertIsInstance(raster, str, "raster type should be string.")
+
+ # - checkpoint off
+ api.gdal.set_checkpoint_off(self.spark) # <- important to call from api.gdal
+ self.assertFalse(self.get_context().is_use_checkpoint(), "context should be configured off.")
+ result = (
+ self.generate_singleband_raster_df()
+ .withColumn("rst_boundingbox", api.rst_boundingbox("tile"))
+ .withColumn("tile", api.rst_clip("tile", "rst_boundingbox"))
+ )
+ result.write.format("noop").mode("overwrite").save()
+ self.assertEqual(result.count(), 1)
+ tile = result.select("tile").first()[0]
+ raster = tile['raster']
+ self.assertNotIsInstance(raster, str, "raster type should be binary (not string).")
+
+ # - reset
+ api.gdal.reset_checkpoint(self.spark)
+ self.assertFalse(self.get_context().is_use_checkpoint(), "context should be configured off.")
+ self.assertEqual(
+ self.get_context().get_checkpoint_path(), api.gdal.get_checkpoint_path_default(),
+ f"checkpoint path should equal default '{api.gdal.get_checkpoint_path_default()}'."
+ )
+ result = (
+ self.generate_singleband_raster_df()
+ .withColumn("rst_boundingbox", api.rst_boundingbox("tile"))
+ .withColumn("tile", api.rst_clip("tile", "rst_boundingbox"))
+ )
+ result.write.format("noop").mode("overwrite").save()
+ self.assertEqual(result.count(), 1)
+ tile = result.select("tile").first()[0]
+ raster = tile['raster']
+ self.assertNotIsInstance(raster, str, "raster type should be binary (not string).")
diff --git a/python/test/test_fuse_install.py b/python/test/test_fuse_install.py
index ec0d82745..b5b69fa5a 100644
--- a/python/test/test_fuse_install.py
+++ b/python/test/test_fuse_install.py
@@ -14,24 +14,6 @@ def test_setup_script_only(self):
self.assertEqual(len(installer.list_files()),1) # <- script generated
- def test_setup_jar(self):
- installer = FuseInstaller(jar_copy=True, jni_so_copy=False)
- try:
- self.assertTrue(installer.do_op())
- except Exception:
- self.fail("Executing `setup_fuse_install()` raised an exception.")
-
- self.assertEqual(len(installer.list_files()), 2) # <- init script and jar
-
- def test_setup_jni(self):
- installer = FuseInstaller(jar_copy=False, jni_so_copy=True)
- try:
- self.assertTrue(installer.do_op())
- except Exception:
- self.fail("Executing `setup_fuse_install()` raised an exception.")
-
- self.assertEqual(len(installer.list_files()), 4) # <- init script and so files
-
def test_setup_all(self):
installer = FuseInstaller(jar_copy=True, jni_so_copy=True)
try:
diff --git a/python/test/test_mosaic.py b/python/test/test_mosaic.py
index 149afd06c..f185189b3 100644
--- a/python/test/test_mosaic.py
+++ b/python/test/test_mosaic.py
@@ -1,5 +1,4 @@
from pyspark.sql.functions import _to_java_column, col
-
from .context import MosaicContext, MosaicLibraryHandler
from .utils import SparkTestCase
@@ -8,6 +7,11 @@ class TestMosaicContext(SparkTestCase):
def setUp(self) -> None:
return super().setUp()
+ def test_has_context(self):
+ _ = MosaicLibraryHandler(self.spark)
+ context = MosaicContext(self.spark)
+ self.assertTrue(context.has_context(), "JVM context should be available after python init.")
+
def test_invoke_function(self):
_ = MosaicLibraryHandler(self.spark)
context = MosaicContext(self.spark)
diff --git a/python/test/utils/mosaic_test_case.py b/python/test/utils/mosaic_test_case.py
index 30437c265..c3ecd9929 100644
--- a/python/test/utils/mosaic_test_case.py
+++ b/python/test/utils/mosaic_test_case.py
@@ -1,24 +1,27 @@
from test.context import api
-
+from test.context import config
from pyspark.sql import DataFrame
from pyspark.sql.functions import col, to_json
-
from mosaic import st_geomfromgeojson, st_point
-
from .spark_test_case import SparkTestCase
class MosaicTestCase(SparkTestCase):
+
def setUp(self) -> None:
- return super.setUp()
+ return super().setUp()
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
api.enable_mosaic(cls.spark)
- def setUp(self) -> None:
- return super().setUp()
+ @classmethod
+ def tearDownClass(cls) -> None:
+ super().tearDownClass()
+
+ def get_context(self):
+ return config.mosaic_context
def generate_input_single_linestring(self) -> DataFrame:
return self.spark.createDataFrame(
diff --git a/python/test/utils/mosaic_test_case_with_gdal.py b/python/test/utils/mosaic_test_case_with_gdal.py
index 046e71674..bf47f8f60 100644
--- a/python/test/utils/mosaic_test_case_with_gdal.py
+++ b/python/test/utils/mosaic_test_case_with_gdal.py
@@ -1,20 +1,42 @@
from test.context import api
-
from .mosaic_test_case import MosaicTestCase
-
from pyspark.sql.dataframe import DataFrame
+import os
+import shutil
+
class MosaicTestCaseWithGDAL(MosaicTestCase):
+ check_dir = None
+ new_check_dir = None
+
def setUp(self) -> None:
return super().setUp()
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
+
+ pwd_dir = os.getcwd()
+ cls.check_dir = f"{pwd_dir}/checkpoint"
+ cls.new_check_dir = f"{pwd_dir}/checkpoint-new"
+ if not os.path.exists(cls.check_dir):
+ os.makedirs(cls.check_dir)
+ if not os.path.exists(cls.new_check_dir):
+ os.makedirs(cls.new_check_dir)
+ cls.spark.conf.set("spark.databricks.labs.mosaic.raster.checkpoint", cls.check_dir)
+
api.enable_mosaic(cls.spark)
api.enable_gdal(cls.spark)
+ @classmethod
+ def tearDownClass(cls) -> None:
+ super().tearDownClass()
+ if cls.check_dir is not None and os.path.exists(cls.check_dir):
+ shutil.rmtree(cls.check_dir)
+ if cls.new_check_dir is not None and os.path.exists(cls.new_check_dir):
+ shutil.rmtree(cls.new_check_dir)
+
def generate_singleband_raster_df(self) -> DataFrame:
return (
self.spark.read.format("gdal")
diff --git a/python/test/utils/spark_test_case.py b/python/test/utils/spark_test_case.py
index 640713ba7..af7a60f6a 100644
--- a/python/test/utils/spark_test_case.py
+++ b/python/test/utils/spark_test_case.py
@@ -1,5 +1,6 @@
import unittest
import os
+import shutil
from importlib.metadata import version
from pyspark.sql import SparkSession
@@ -11,6 +12,7 @@ class SparkTestCase(unittest.TestCase):
spark = None
library_location = None
log4jref = None
+ tmp_dir = None
@classmethod
def setUpClass(cls) -> None:
@@ -19,12 +21,9 @@ def setUpClass(cls) -> None:
cls.library_location = f"{mosaic.__path__[0]}/lib/mosaic-{version('databricks-mosaic')}-SNAPSHOT-jar-with-dependencies.jar"
pwd_dir = os.getcwd()
- tmp_dir = f"{pwd_dir}/mosaic_test/"
- check_dir = f"{pwd_dir}/checkpoint"
- if not os.path.exists(tmp_dir):
- os.makedirs(tmp_dir)
- if not os.path.exists(check_dir):
- os.makedirs(check_dir)
+ cls.tmp_dir = f"{pwd_dir}/mosaic_test/"
+ if not os.path.exists(cls.tmp_dir):
+ os.makedirs(cls.tmp_dir)
cls.spark = (
SparkSession.builder.master("local[*]")
@@ -32,22 +31,24 @@ def setUpClass(cls) -> None:
.config("spark.driver.memory", "4g")
.config(
"spark.driver.extraJavaOptions",
- "-Dorg.apache.logging.log4j.level=FATAL",
+ "-Dorg.apache.logging.log4j.level=ERROR",
)
.config(
"spark.executor.extraJavaOptions",
- "-Dorg.apache.logging.log4j.level=FATAL",
+ "-Dorg.apache.logging.log4j.level=ERROR",
)
.getOrCreate()
)
+ cls.spark.conf.set("spark.databricks.labs.mosaic.test.mode", "true")
cls.spark.conf.set("spark.databricks.labs.mosaic.jar.autoattach", "false")
- cls.spark.conf.set("spark.databricks.labs.mosaic.raster.tmp.prefix", tmp_dir)
- cls.spark.conf.set("spark.databricks.labs.mosaic.raster.checkpoint", check_dir)
- cls.spark.sparkContext.setLogLevel("FATAL")
+ cls.spark.conf.set("spark.databricks.labs.mosaic.raster.tmp.prefix", cls.tmp_dir)
+ cls.spark.sparkContext.setLogLevel("ERROR")
@classmethod
def tearDownClass(cls) -> None:
cls.spark.stop()
+ if cls.tmp_dir is not None and os.path.exists(cls.tmp_dir):
+ shutil.rmtree(cls.tmp_dir)
def setUp(self) -> None:
- self.spark.sparkContext.setLogLevel("FATAL")
+ self.spark.sparkContext.setLogLevel("ERROR")
diff --git a/scripts/docker/README.md b/scripts/docker/README.md
new file mode 100644
index 000000000..4e6a37c32
--- /dev/null
+++ b/scripts/docker/README.md
@@ -0,0 +1,28 @@
+# Docker Build
+
+> This is adapted from [Mosaic-Docker](https://github.com/r3stl355/mosaic-docker) repo, focused on DBR 13.3 LTS which is Ubuntu 22.04.
+> It is needed when you want to build and run tests on non Ubuntu Jammy machines, e.g. MacOS.
+
+## Steps
+
+1. Cmd `GDAL_VERSION=3.4.1 LIBPROJ_VERSION=7.1.0 SPARK_VERSION=3.4.1 CORES=4 ./build`
+ builds the docker image for DBR 13.3 LTS. Name will be 'mosaic-dev:ubuntu22-gdal3.4.1-spark3.4.1'.
+2. Cmd `sh scripts/docker/mosaic-docker.sh` to run. That script launches a container and further (optionally) configures.
+
+## Additional Notes
+
+* Image is configured to JDK 8 to match DBR 13; python 3.10 as well
+* Support IDE driven or Jupyter notebook testing in addition to straight shell,
+ see more at [Mosaic-Docker](https://github.com/r3stl355/mosaic-docker). Recommend placing any test notebooks
+ in '/python/notebooks' which is already added to .gitignore
+* If you want to run tests within a container shell:
+ - `unset JAVA_TOOL_OPTIONS` is needed to execute JVM tests
+ - then can test e.g. `mvn -X test -DskipTests=false -Dsuites=com.databricks.labs.mosaic.core.raster.TestRasterGDAL`
+ and `python3 -m unittest mosaic test/test_fuse_install.py` from ./python dir
+ - you may need to run `mvn clean` occasionally, especially around initial setup as intellij is JDK 11 (pom.xml)
+ and docker is JDK 8
+ - you don't need to specify -PskipCoverage (see 'm2/settings.xml' and pom.xml)
+* Get shell with `docker exec -it mosaic-dev /bin/bash -c "unset JAVA_TOOL_OPTIONS && cd /root/mosaic && /bin/bash"`,
+ can have multiple shells going; call `sh scripts/docker/exec-shell.sh` also
+* `docker stop mosaic-dev` whenever done to terminate the container
+* NOTE: Ignore 'ERRO[0000] error waiting for container: context canceled' if you get this on MacOS
\ No newline at end of file
diff --git a/scripts/docker/docker-build/ubuntu-22-spark-3.4/Dockerfile.template b/scripts/docker/docker-build/ubuntu-22-spark-3.4/Dockerfile.template
new file mode 100755
index 000000000..173452a6c
--- /dev/null
+++ b/scripts/docker/docker-build/ubuntu-22-spark-3.4/Dockerfile.template
@@ -0,0 +1,115 @@
+FROM --platform=linux/amd64 ubuntu:22.04
+
+# refresh package info
+RUN apt-get update -y
+
+# Install OpenJDK 8
+RUN apt-get install -y openjdk-8-jdk --no-install-recommends
+
+# Install native dependencies
+RUN apt-get install -y python3-numpy unixodbc libcurl3-gnutls libsnappy-dev libopenjp2-7
+
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+
+# Install dependencies
+RUN set -ex \
+ && deps=" \
+ python3-dev \
+ python3-numpy \
+ python3-pip \
+ python3-venv \
+ bash-completion \
+ libspatialite-dev \
+ libpq-dev \
+ libcurl4-gnutls-dev \
+ libxml2-dev \
+ libgeos-dev \
+ libnetcdf-dev \
+ libpoppler-dev \
+ libhdf4-alt-dev \
+ libhdf5-serial-dev \
+ libpoppler-private-dev \
+ sqlite3 \
+ libsqlite3-dev \
+ libtiff-dev \
+ wget \
+ curl \
+ " \
+ && buildDeps=" \
+ build-essential \
+ cmake \
+ swig \
+ ant \
+ pkg-config \
+ "\
+ && apt-get update -y && apt-get install -y $buildDeps $deps --no-install-recommends
+
+# Install the remaining components
+ENV ROOTDIR /usr/local
+ENV LD_LIBRARY_PATH /usr/local/lib
+ENV SPARK_VERSION %%SPARK_VERSION%%
+ENV GDAL_VERSION %%GDAL_VERSION%%
+ENV LIBPROJ_VERSION %%LIBPROJ_VERSION%%
+ENV CORES %%CORES%%
+
+WORKDIR $ROOTDIR/
+RUN mkdir -p $ROOTDIR/src
+
+# Install PROJ
+RUN wget -qO- https://download.osgeo.org/proj/proj-${LIBPROJ_VERSION}.tar.gz | \
+ tar -xzC $ROOTDIR/src/
+
+RUN cd src/proj-${LIBPROJ_VERSION} && ./configure && make -j${CORES} && make install \
+ && cd $ROOTDIR && rm -Rf src/proj*
+
+# Install GDAL
+RUN wget -qO- https://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz | \
+ tar -xzC $ROOTDIR/src/
+
+RUN cd src/gdal-${GDAL_VERSION} \
+ && ./configure --with-java=$JAVA_HOME \
+ && make -j${CORES} && make -j${CORES} install && ldconfig
+
+# Install Java bindings for GDAL
+RUN cd $ROOTDIR/src/gdal-${GDAL_VERSION}/swig/java && make -j${CORES} && make -j${CORES} install
+
+# Copy binaries to the location expected to be by Mosaic
+RUN ln -s $ROOTDIR/lib/libgdal.so /usr/lib/libgdal.so
+RUN ln -s $ROOTDIR/lib/libgdal.so.30 /usr/lib/libgdal.so.30
+RUN ln -s $ROOTDIR/lib/libgdal.so.30.0.3 /usr/lib/libgdal.so.30.0.3
+RUN mkdir -p /usr/lib/jni && ln -s $ROOTDIR/lib/libgdalalljni.so /usr/lib/jni/libgdalalljni.so.30
+RUN mkdir -p /usr/lib/ogdi && ln -s $ROOTDIR/lib/libgdal.so /usr/lib/ogdi/libgdal.so
+
+# Add Maven
+ARG MAVEN_VERSION=3.9.6
+ARG USER_HOME_DIR="/root"
+ARG BASE_URL=https://dlcdn.apache.org/maven/maven-3/${MAVEN_VERSION}/binaries
+ARG ARG SHA=706f01b20dec0305a822ab614d51f32b07ee11d0218175e55450242e49d2156386483b506b3a4e8a03ac8611bae96395fd5eec15f50d3013d5deed6d1ee18224
+
+RUN mkdir -p $ROOTDIR/share/maven $ROOTDIR/share/maven/ref \
+ && echo "Downlaoding maven" \
+ && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \
+ \
+ && echo "Checking download hash" \
+ && echo "${SHA} /tmp/apache-maven.tar.gz" | sha512sum -c - \
+ \
+ && echo "Unziping maven" \
+ && tar -xzf /tmp/apache-maven.tar.gz -C $ROOTDIR/share/maven --strip-components=1 \
+ \
+ && echo "Cleaning and setting links" \
+ && rm -f /tmp/apache-maven.tar.gz \
+ && ln -s $ROOTDIR/share/maven/bin/mvn $ROOTDIR/bin/mvn
+
+ENV MAVEN_HOME $ROOTDIR/share/maven
+ENV MAVEN_CONFIG "$USER_HOME_DIR/.m2"
+
+# Python packages
+# - Adds additional needed packages
+RUN pip3 install pip --upgrade
+RUN pip3 install build wheel keplergl ipython pyspark==$SPARK_VERSION
+RUN pip3 install black build isort py4j requests
+RUN pip3 install gdal==$GDAL_VERSION
+
+# Clean up
+RUN apt-get purge -y --auto-remove $buildDeps \
+ && rm -rf /var/lib/apt/lists/*
diff --git a/scripts/docker/docker-build/ubuntu-22-spark-3.4/build b/scripts/docker/docker-build/ubuntu-22-spark-3.4/build
new file mode 100755
index 000000000..f3be3419c
--- /dev/null
+++ b/scripts/docker/docker-build/ubuntu-22-spark-3.4/build
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+sed -e "s/%%GDAL_VERSION%%/$GDAL_VERSION/" \
+ -e "s/%%LIBPROJ_VERSION%%/$LIBPROJ_VERSION/" \
+ -e "s/%%SPARK_VERSION%%/$SPARK_VERSION/" \
+ -e "s/%%CORES%%/$CORES/" "Dockerfile.template" > Dockerfile
+
+# use --no-cache to force clean build
+#docker build --no-cache -t "mosaic-dev:ubuntu22-gdal$GDAL_VERSION-spark$SPARK_VERSION" .
+docker build -t "mosaic-dev:ubuntu22-gdal$GDAL_VERSION-spark$SPARK_VERSION" .
\ No newline at end of file
diff --git a/scripts/docker/docker_init.sh b/scripts/docker/docker_init.sh
new file mode 100755
index 000000000..6c8b468c6
--- /dev/null
+++ b/scripts/docker/docker_init.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# [1] unset variable for this script
+echo "\n::: [1] ... unsetting JAVA_TOOL_OPTIONS (probably need to do in container as well) :::"
+unset JAVA_TOOL_OPTIONS
+
+# [2] copy custom settings.xml
+# - defaults to new skipScoverage profile
+# - compliments the pom config (profile sCoverage also added there)
+# - sets .m2 folder to be in project
+echo "\n::: [2] ... setting up new .m2 (in project) + new skipScoverage profile (as default) :::"
+mv /usr/local/share/maven/conf/settings.xml /usr/local/share/maven/conf/settings.xml.BAK
+cp /root/mosaic/scripts/docker/m2/settings.xml /usr/local/share/maven/conf
+echo " ... mvn active profile(s)\n"
+cd /root/mosaic && mvn help:active-profiles
+
+# [3] build JVM code
+# this is building for container JDK
+# see settings.xml for overrides
+echo "\n::: [3] ... maven package - JVM code version? :::\n"
+echo " $(javac -version)"
+cd /root/mosaic && mvn package -DskipTests
+
+# [4] build python
+# - refer to dockerfile for what is already built
+echo "\n::: [4] ... build python :::\n"
+cd /root/mosaic/python && pip install .
\ No newline at end of file
diff --git a/scripts/docker/exec-shell.sh b/scripts/docker/exec-shell.sh
new file mode 100644
index 000000000..c2efc3c49
--- /dev/null
+++ b/scripts/docker/exec-shell.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker exec -it mosaic-dev /bin/bash -c "unset JAVA_TOOL_OPTIONS && cd /root/mosaic && /bin/bash"
\ No newline at end of file
diff --git a/scripts/docker/m2/settings.xml b/scripts/docker/m2/settings.xml
new file mode 100644
index 000000000..46f695264
--- /dev/null
+++ b/scripts/docker/m2/settings.xml
@@ -0,0 +1,12 @@
+
+
+ /root/mosaic/scripts/docker/m2
+
+ skipScoverage
+
+
\ No newline at end of file
diff --git a/scripts/docker/mosaic-docker.sh b/scripts/docker/mosaic-docker.sh
new file mode 100644
index 000000000..ccf39c049
--- /dev/null
+++ b/scripts/docker/mosaic-docker.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# [1] Build the image under 'docker-build':
+# `GDAL_VERSION=3.4.1 LIBPROJ_VERSION=7.1.0 SPARK_VERSION=3.4.1 CORES=4 ./build`
+# - produces image 'ubuntu22-gdal3.4.1-spark3.4.1' [default is JDK 8]
+# [2] run this in root of (mosaic repo), e.g. `sh scripts/docker/mosaic-docker.sh`
+# - for IDE driven or Jupyter notebook testing
+# [3] if you want to run tests within the container shell
+# - [a] `unset JAVA_TOOL_OPTIONS` is needed to execute JVM tests
+# - [b] then can test e.g. `mvn -X test -DskipTests=false -Dsuites=com.databricks.labs.mosaic.core.raster.TestRasterGDAL`
+# and `python3 -m unittest mosaic test/test_fuse_install.py` from ./python dir
+# - [c] you may need to run `mvn clean` occasionally, especially around initial setup as intellij is JDK 11
+# and docker is JDK 8.
+# ... don't need to specify -PskipCoverage (see settings.xml)
+# [4] get shell with `docker exec -it mosaic-dev /bin/bash -c "unset JAVA_TOOL_OPTIONS && cd /root/mosaic && /bin/bash"`,
+# - can have multiple shells going; call `sh scripts/docker/exec-shell.sh` also
+# [5] `docker stop mosaic-dev` whenever done to terminate the container
+# NOTE: Ignore 'ERRO[0000] error waiting for container: context canceled'
+docker run -q --privileged --platform linux/amd64 --name mosaic-dev -p 5005:5005 -p 8888:8888 \
+-v $PWD:/root/mosaic -e JAVA_TOOL_OPTIONS="-agentlib:jdwp=transport=dt_socket,address=5005,server=y,suspend=n" \
+-itd --rm mosaic-dev:ubuntu22-gdal3.4.1-spark3.4.1 /bin/bash
+docker exec -it mosaic-dev /bin/bash -c "sh /root/mosaic/scripts/docker/docker_init.sh"
+docker exec -it mosaic-dev /bin/bash -c "unset JAVA_TOOL_OPTIONS && cd /root/mosaic && /bin/bash"
\ No newline at end of file
diff --git a/scripts/mosaic-docker.sh b/scripts/mosaic-docker.sh
deleted file mode 100644
index 85c867c6c..000000000
--- a/scripts/mosaic-docker.sh
+++ /dev/null
@@ -1 +0,0 @@
-docker run --name mosaic-dev --rm -p 5005:5005 -v $PWD:/root/mosaic -e JAVA_TOOL_OPTIONS="-agentlib:jdwp=transport=dt_socket,address=5005,server=y,suspend=n" -it mosaic-dev:jdk8-gdal3.4-spark3.2 /bin/bash
\ No newline at end of file
diff --git a/scripts/update_version.py b/scripts/update_version.py
index a32eeb4b8..270526d23 100644
--- a/scripts/update_version.py
+++ b/scripts/update_version.py
@@ -3,6 +3,13 @@
import sys
def main(version, snapshot):
+ """
+ TODO: Also need to adjust...
+ [1] /R/sparklyr-mosaic/tests.R
+ - e.g. "sparklyrMosaic_0.4.3.tar.gz"
+ [2] /src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala
+ - e.g. "mosaicVersion: String = "0.4.3"
+ """
update_pom_version('../pom.xml', version + snapshot)
update_python_version('../python/mosaic/__init__.py', version + snapshot)
update_r_version('../R/sparkR-mosaic/sparkrMosaic/DESCRIPTION', version)
diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/api/GDAL.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/api/GDAL.scala
index d840a3205..008717ee1 100644
--- a/src/main/scala/com/databricks/labs/mosaic/core/raster/api/GDAL.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/api/GDAL.scala
@@ -50,10 +50,18 @@ object GDAL {
/** @return Returns the name of the raster API. */
def name: String = "GDAL"
+ /** @return Returns whether using checkpoint (assumes `enable` called) */
+ def isUseCheckpoint: Boolean = MosaicGDAL.isUseCheckpoint
+
+ /** @return Returns checkpoint path (assumes `enable` called) */
+ def getCheckpointPath: String = MosaicGDAL.getCheckpointPath
+
/**
* Enables GDAL on the worker nodes. GDAL requires drivers to be registered
* on the worker nodes. This method registers all the drivers on the worker
* nodes.
+ * @param mosaicConfig
+ * The [[MosaicExpressionConfig]] for the op.
*/
def enable(mosaicConfig: MosaicExpressionConfig): Unit = {
configureGDAL(mosaicConfig)
@@ -61,6 +69,13 @@ object GDAL {
gdal.AllRegister()
}
+ /**
+ * Enables GDAL on the worker nodes. GDAL requires drivers to be registered
+ * on the worker nodes. This method registers all the drivers on the worker
+ * nodes.
+ * @param spark
+ * Spark session from which to populate the [[MosaicExpressionConfig]].
+ */
def enable(spark: SparkSession): Unit = {
val mosaicConfig = MosaicExpressionConfig(spark)
enable(mosaicConfig)
@@ -147,6 +162,10 @@ object GDAL {
*
* @param generatedRasters
* The rasters to write.
+ * @param rasterDT
+ * The type of raster to write.
+ * - if string write to checkpoint
+ * - otherwise, write to bytes
* @return
* Returns the paths of the written rasters.
*/
@@ -155,12 +174,7 @@ object GDAL {
if (raster != null) {
rasterDT match {
case StringType =>
- val uuid = UUID.randomUUID().toString
- val extension = GDAL.getExtension(raster.getDriversShortName)
- val writePath = s"${MosaicGDAL.checkpointPath}/$uuid.$extension"
- val outPath = raster.writeToPath(writePath)
- RasterCleaner.dispose(raster)
- UTF8String.fromString(outPath)
+ writeRasterString(raster)
case BinaryType =>
val bytes = raster.writeToBytes()
RasterCleaner.dispose(raster)
@@ -172,6 +186,15 @@ object GDAL {
)
}
+ private def writeRasterString(raster: MosaicRasterGDAL): UTF8String = {
+ val uuid = UUID.randomUUID().toString
+ val extension = GDAL.getExtension(raster.getDriversShortName)
+ val writePath = s"${getCheckpointPath}/$uuid.$extension"
+ val outPath = raster.writeToPath(writePath)
+ RasterCleaner.dispose(raster)
+ UTF8String.fromString(outPath)
+ }
+
/**
* Reads a raster from the given path. Assume not zipped file. If zipped,
* use raster(path, vsizip = true)
@@ -179,8 +202,10 @@ object GDAL {
* @param path
* The path to the raster. This path has to be a path to a single raster.
* Rasters with subdatasets are supported.
+ * @param parentPath
+ * Parent path can help with detecting driver.
* @return
- * Returns a Raster object.
+ * Returns a [[MosaicRasterGDAL]] object.
*/
def raster(path: String, parentPath: String): MosaicRasterGDAL = {
val createInfo = Map("path" -> path, "parentPath" -> parentPath)
@@ -193,8 +218,12 @@ object GDAL {
*
* @param content
* The byte array to read the raster from.
+ * @param parentPath
+ * Parent path can help with detecting driver.
+ * @param driverShortName
+ * Driver to use in reading.
* @return
- * Returns a Raster object.
+ * Returns a [[MosaicRasterGDAL]] object.
*/
def raster(content: Array[Byte], parentPath: String, driverShortName: String): MosaicRasterGDAL = {
val createInfo = Map("parentPath" -> parentPath, "driver" -> driverShortName)
@@ -210,8 +239,10 @@ object GDAL {
* Rasters with subdatasets are supported.
* @param bandIndex
* The index of the band to read from the raster.
+ * @param parentPath
+ * Parent path can help with detecting driver.
* @return
- * Returns a Raster band object.
+ * Returns a [[MosaicRasterBandGDAL]] object.
*/
def band(path: String, bandIndex: Int, parentPath: String): MosaicRasterBandGDAL = {
val createInfo = Map("path" -> path, "parentPath" -> parentPath)
diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/gdal/MosaicRasterGDAL.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/gdal/MosaicRasterGDAL.scala
index b399fa0a2..252043ff3 100644
--- a/src/main/scala/com/databricks/labs/mosaic/core/raster/gdal/MosaicRasterGDAL.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/gdal/MosaicRasterGDAL.scala
@@ -22,7 +22,14 @@ import java.util.{Locale, Vector => JVector}
import scala.collection.JavaConverters.dictionaryAsScalaMapConverter
import scala.util.{Failure, Success, Try}
-/** GDAL implementation of the MosaicRaster trait. */
+/**
+ * Mosaic's GDAL internal object for rasters.
+ * - Constructor invoked from various functions, including the
+ * [[MosaicRasterGDAL]] scala companion object.
+ * - When invoked, raster is already a GDAL [[Dataset]].
+ * - "path" expected to be either "no_path" or fuse accessible.
+ * - same for "parent_path"
+ */
//noinspection DuplicatedCode
case class MosaicRasterGDAL(
raster: Dataset,
@@ -31,11 +38,8 @@ case class MosaicRasterGDAL(
) extends RasterWriter
with RasterCleaner {
- def path: String = createInfo("path")
-
- def parentPath: String = createInfo("parentPath")
-
- def driverShortName: Option[String] = createInfo.get("driver")
+ // Factory for creating CRS objects
+ protected val crsFactory: CRSFactory = new CRSFactory
def getWriteOptions: MosaicRasterWriteOptions = MosaicRasterWriteOptions(this)
@@ -47,114 +51,134 @@ case class MosaicRasterGDAL(
compression
}
- def getSpatialReference: SpatialReference = {
- val spatialRef =
- if (raster != null) {
- raster.GetSpatialRef
- } else {
- val tmp = refresh()
- val result = tmp.raster.GetSpatialRef
- dispose(tmp)
- result
- }
- if (spatialRef == null) {
- MosaicGDAL.WSG84
- } else {
- spatialRef
- }
- }
+ /////////////////////////////////////////
+ // FROM createInfo
+ /////////////////////////////////////////
- def isSubDataset: Boolean = {
- val isSubdataset = PathUtils.isSubdataset(path)
- isSubdataset
- }
+ /** @return The raster's path on disk. */
+ def path: String = createInfo("path")
- // Factory for creating CRS objects
- protected val crsFactory: CRSFactory = new CRSFactory
+ /** @return The raster's path on disk. Usually this is a parent file for the tile. */
+ def parentPath: String = createInfo("parentPath")
- /**
- * @return
- * The raster's driver short name.
- */
- def getDriversShortName: String =
- driverShortName.getOrElse(
- Try(raster.GetDriver().getShortName).getOrElse("NONE")
- )
+ /** @return The driver as option. */
+ def driverShortName: Option[String] = createInfo.get("driver")
- /**
- * @return
- * The raster's path on disk. Usually this is a parent file for the tile.
- */
- def getParentPath: String = parentPath
+ /////////////////////////////////////////
+ // GDAL Dataset
+ /////////////////////////////////////////
/**
+ * For the provided geometry and CRS, get bounding box polygon.
+ * @param geometryAPI
+ * Default is JTS.
+ * @param destCRS
+ * CRS for the bbox, default is [[MosaicGDAL.WSG84]].
* @return
- * The diagonal size of a raster.
+ * Returns [[MosaicGeometry]] representing bounding box polygon.
*/
- def diagSize: Double = math.sqrt(xSize * xSize + ySize * ySize)
-
- /** @return Returns pixel x size. */
- def pixelXSize: Double = getGeoTransform(1)
-
- /** @return Returns pixel y size. */
- def pixelYSize: Double = getGeoTransform(5)
-
- /** @return Returns the origin x coordinate. */
- def originX: Double = getGeoTransform(0)
-
- /** @return Returns the origin y coordinate. */
- def originY: Double = getGeoTransform(3)
+ def bbox(geometryAPI: GeometryAPI, destCRS: SpatialReference = MosaicGDAL.WSG84): MosaicGeometry = {
+ val gt = getGeoTransform
- /** @return Returns the max x coordinate. */
- def xMax: Double = originX + xSize * pixelXSize
+ val sourceCRS = getSpatialReference
+ val transform = new osr.CoordinateTransformation(sourceCRS, destCRS)
- /** @return Returns the max y coordinate. */
- def yMax: Double = originY + ySize * pixelYSize
+ val bbox = geometryAPI.geometry(
+ Seq(
+ Seq(gt(0), gt(3)),
+ Seq(gt(0) + gt(1) * xSize, gt(3)),
+ Seq(gt(0) + gt(1) * xSize, gt(3) + gt(5) * ySize),
+ Seq(gt(0), gt(3) + gt(5) * ySize)
+ ).map(geometryAPI.fromCoords),
+ POLYGON
+ )
- /** @return Returns the min x coordinate. */
- def xMin: Double = originX
+ val geom1 = org.gdal.ogr.ogr.CreateGeometryFromWkb(bbox.toWKB)
+ geom1.Transform(transform)
- /** @return Returns the min y coordinate. */
- def yMin: Double = originY
+ geometryAPI.geometry(geom1.ExportToWkb(), "WKB")
+ }
- /** @return Returns the diagonal size of a pixel. */
- def pixelDiagSize: Double = math.sqrt(pixelXSize * pixelXSize + pixelYSize * pixelYSize)
+ /** @return The diagonal size of a raster. */
+ def diagSize: Double = math.sqrt(xSize * xSize + ySize * ySize)
- /** @return Returns file extension. */
- def getRasterFileExtension: String = GDAL.getExtension(getDriversShortName)
+ // noinspection ZeroIndexToHead
+ /** @return Returns the raster's extent as a Seq(xmin, ymin, xmax, ymax). */
+ def extent: Seq[Double] = {
+ val minX = getGeoTransform(0)
+ val maxY = getGeoTransform(3)
+ val maxX = minX + getGeoTransform(1) * xSize
+ val minY = maxY + getGeoTransform(5) * ySize
+ Seq(minX, minY, maxX, maxY)
+ }
- /** @return Returns the raster's bands as a Seq. */
- def getBands: Seq[MosaicRasterBandGDAL] = (1 to numBands).map(getBand)
+ /** @return Returns the raster's geotransform as a Seq. */
+ def getGeoTransform: Array[Double] = raster.GetGeoTransform()
/**
- * Flushes the cache of the raster. This is needed to ensure that the
- * raster is written to disk. This is needed for operations like
- * RasterProject.
+ * @note
+ * If memory size is -1 this will destroy the raster and you will need to
+ * refresh it to use it again.
* @return
- * Returns the raster object.
+ * Returns the amount of memory occupied by the file in bytes.
*/
- def flushCache(): MosaicRasterGDAL = {
- // Note: Do not wrap GDAL objects into Option
- if (getRaster != null) getRaster.FlushCache()
- this.destroy()
- this.refresh()
+ def getMemSize: Long = {
+ if (memSize == -1) {
+ val toRead = if (path.startsWith("/vsizip/")) path.replace("/vsizip/", "") else path
+ if (Files.notExists(Paths.get(toRead))) {
+ throw new Exception(s"File not found: ${gdal.GetLastErrorMsg()}")
+ }
+ Files.size(Paths.get(toRead))
+ } else {
+ memSize
+ }
+
}
/**
- * Opens a raster from a file system path.
- * @param path
- * The path to the raster file.
+ * Get spatial reference.
+ * - may be already set on the raster
+ * - if not, load and detect it.
+ * - defaults to [[MosaicGDAL.WSG84]]
* @return
- * A MosaicRaster object.
+ * Raster's [[SpatialReference]] object.
*/
- def openRaster(path: String): Dataset = {
- MosaicRasterGDAL.openRaster(path, driverShortName)
+ def getSpatialReference: SpatialReference = {
+ val spatialRef =
+ if (raster != null) {
+ raster.GetSpatialRef
+ } else {
+ val tmp = refresh()
+ val result = tmp.raster.GetSpatialRef
+ dispose(tmp)
+ result
+ }
+ if (spatialRef == null) {
+ MosaicGDAL.WSG84
+ } else {
+ spatialRef
+ }
}
/**
- * @return
- * Returns the raster's metadata as a Map.
- */
+ * @return
+ * True if the raster is empty, false otherwise. May be expensive to
+ * compute since it requires reading the raster and computing statistics.
+ */
+ def isEmpty: Boolean = {
+ val bands = getBands
+ if (bands.isEmpty) {
+ subdatasets.values
+ .filter(_.toLowerCase(Locale.ROOT).startsWith(getDriversShortName.toLowerCase(Locale.ROOT)))
+ .flatMap(bp => readRaster(createInfo + ("path" -> bp)).getBands)
+ .takeWhile(_.isEmpty)
+ .nonEmpty
+ } else {
+ bands.takeWhile(_.isEmpty).nonEmpty
+ }
+ }
+
+ /** @return Returns the raster's metadata as a Map. */
def metadata: Map[String, String] = {
Option(raster.GetMetadataDomainList())
.map(_.toArray)
@@ -171,69 +195,42 @@ case class MosaicRasterGDAL(
.getOrElse(Map.empty[String, String])
}
- /**
- * @return
- * Returns the raster's subdatasets as a Map.
- */
- def subdatasets: Map[String, String] = {
- val dict = Try(raster.GetMetadata_Dict("SUBDATASETS"))
- .getOrElse(new java.util.Hashtable[String, String]())
- val subdatasetsMap = Option(dict)
- .map(_.asScala.toMap.asInstanceOf[Map[String, String]])
- .getOrElse(Map.empty[String, String])
- val keys = subdatasetsMap.keySet
- val sanitizedParentPath = PathUtils.getCleanPath(parentPath)
- keys.flatMap(key =>
- if (key.toUpperCase(Locale.ROOT).contains("NAME")) {
- val path = subdatasetsMap(key)
- val pieces = path.split(":")
- Seq(
- key -> pieces.last,
- s"${pieces.last}_tmp" -> path,
- pieces.last -> s"${pieces.head}:$sanitizedParentPath:${pieces.last}"
- )
- } else Seq(key -> subdatasetsMap(key))
- ).toMap
+ /** @return Returns the raster's number of bands. */
+ def numBands: Int = {
+ val bandCount = Try(raster.GetRasterCount())
+ bandCount match {
+ case Success(value) => value
+ case Failure(_) => 0
+ }
}
- /**
- * @return
- * Returns the raster's SRID. This is the EPSG code of the raster's CRS.
- */
- def SRID: Int = {
- Try(crsFactory.readEpsgFromParameters(proj4String))
- .filter(_ != null)
- .getOrElse("EPSG:0")
- .split(":")
- .last
- .toInt
- }
+ /** @return Returns the origin x coordinate. */
+ def originX: Double = getGeoTransform(0)
+
+ /** @return Returns the origin y coordinate. */
+ def originY: Double = getGeoTransform(3)
/**
+ * Opens a raster from a file system path.
+ * @param path
+ * The path to the raster file.
* @return
- * Sets the raster's SRID. This is the EPSG code of the raster's CRS.
+ * A GDAL [[Dataset]] object.
*/
- def setSRID(srid: Int): MosaicRasterGDAL = {
- val srs = new osr.SpatialReference()
- srs.ImportFromEPSG(srid)
- raster.SetSpatialRef(srs)
- val driver = raster.GetDriver()
- val newPath = PathUtils.createTmpFilePath(GDAL.getExtension(getDriversShortName))
- driver.CreateCopy(newPath, raster)
- val newRaster = MosaicRasterGDAL.openRaster(newPath, driverShortName)
- dispose(this)
- val createInfo = Map(
- "path" -> newPath,
- "parentPath" -> parentPath,
- "driver" -> getDriversShortName
- )
- MosaicRasterGDAL(newRaster, createInfo, -1)
+ def pathAsDataset(path: String): Dataset = {
+ MosaicRasterGDAL.pathAsDataset(path, driverShortName)
}
- /**
- * @return
- * Returns the raster's proj4 string.
- */
+ /** @return Returns the diagonal size of a pixel. */
+ def pixelDiagSize: Double = math.sqrt(pixelXSize * pixelXSize + pixelYSize * pixelYSize)
+
+ /** @return Returns pixel x size. */
+ def pixelXSize: Double = getGeoTransform(1)
+
+ /** @return Returns pixel y size. */
+ def pixelYSize: Double = getGeoTransform(5)
+
+ /** @return Returns the raster's proj4 string. */
def proj4String: String = {
try {
@@ -243,68 +240,124 @@ case class MosaicRasterGDAL(
}
}
- /**
- * @param bandId
- * The band index to read.
- * @return
- * Returns the raster's band as a MosaicRasterBand object.
- */
- def getBand(bandId: Int): MosaicRasterBandGDAL = {
- if (bandId > 0 && numBands >= bandId) {
- MosaicRasterBandGDAL(raster.GetRasterBand(bandId), bandId)
- } else {
- throw new ArrayIndexOutOfBoundsException()
- }
+ /** @return Sets the raster's SRID. This is the EPSG code of the raster's CRS. */
+ def setSRID(srid: Int): MosaicRasterGDAL = {
+ val srs = new osr.SpatialReference()
+ srs.ImportFromEPSG(srid)
+ raster.SetSpatialRef(srs)
+ val driver = raster.GetDriver()
+ val tmpPath = PathUtils.createTmpFilePath(GDAL.getExtension(getDriversShortName))
+ driver.CreateCopy(tmpPath, raster)
+ val newRaster = MosaicRasterGDAL.pathAsDataset(tmpPath, driverShortName)
+ dispose(this)
+ val newCreateInfo = Map(
+ "path" -> tmpPath,
+ "parentPath" -> parentPath,
+ "driver" -> getDriversShortName
+ )
+ MosaicRasterGDAL(newRaster, newCreateInfo, -1)
}
- /**
- * @return
- * Returns the raster's number of bands.
- */
- def numBands: Int = {
- val bandCount = Try(raster.GetRasterCount())
- bandCount match {
- case Success(value) => value
- case Failure(_) => 0
- }
+ /** @return Returns the raster's SRID. This is the EPSG code of the raster's CRS. */
+ def SRID: Int = {
+ Try(crsFactory.readEpsgFromParameters(proj4String))
+ .filter(_ != null)
+ .getOrElse("EPSG:0")
+ .split(":")
+ .last
+ .toInt
}
- // noinspection ZeroIndexToHead
- /**
- * @return
- * Returns the raster's extent as a Seq(xmin, ymin, xmax, ymax).
- */
- def extent: Seq[Double] = {
- val minX = getGeoTransform(0)
- val maxY = getGeoTransform(3)
- val maxX = minX + getGeoTransform(1) * xSize
- val minY = maxY + getGeoTransform(5) * ySize
- Seq(minX, minY, maxX, maxY)
- }
+ /** @return Returns the min x coordinate. */
+ def xMin: Double = originX
- /**
- * @return
- * Returns x size of the raster.
- */
+ /** @return Returns the max x coordinate. */
+ def xMax: Double = originX + xSize * pixelXSize
+
+ /** @return Returns x size of the raster. */
def xSize: Int = raster.GetRasterXSize
- /**
- * @return
- * Returns y size of the raster.
- */
+ /** @return Returns the min y coordinate. */
+ def yMin: Double = originY
+
+ /** @return Returns the max y coordinate. */
+ def yMax: Double = originY + ySize * pixelYSize
+
+ /** @return Returns y size of the raster. */
def ySize: Int = raster.GetRasterYSize
+ /////////////////////////////////////////
+ // Apply Functions
+ /////////////////////////////////////////
+
/**
+ * Applies a convolution filter to the raster.
+ * - operator applied per band.
+ * @param kernel
+ * [[Array[Double]]] kernel to apply to the raster.
* @return
- * Returns the raster's geotransform as a Seq.
+ * [[MosaicRasterGDAL]] object.
*/
- def getGeoTransform: Array[Double] = raster.GetGeoTransform()
+ def convolve(kernel: Array[Array[Double]]): MosaicRasterGDAL = {
+ val tmpPath = PathUtils.createTmpFilePath(getRasterFileExtension)
+
+ this.raster
+ .GetDriver()
+ .CreateCopy(tmpPath, this.raster, 1)
+ .delete()
+
+ val outputRaster = gdal.Open(tmpPath, GF_Write)
+
+ for (bandIndex <- 1 to this.numBands) {
+ val band = this.getBand(bandIndex)
+ val outputBand = outputRaster.GetRasterBand(bandIndex)
+ band.convolve(kernel, outputBand)
+ }
+
+ val newCreateInfo = Map(
+ "path" -> tmpPath,
+ "parentPath" -> parentPath,
+ "driver" -> getDriversShortName
+ )
+
+ val result = MosaicRasterGDAL(outputRaster, newCreateInfo, this.memSize)
+ result.flushCache()
+ }
/**
+ * Applies a filter to the raster.
+ * @param kernelSize
+ * Number of pixels to compare; it must be odd.
+ * @param operation
+ * Op to apply, e.g. ‘avg’, ‘median’, ‘mode’, ‘max’, ‘min’.
* @return
- * Underlying GDAL raster object.
+ * Returns a new [[MosaicRasterGDAL]] with the filter applied.
*/
- def getRaster: Dataset = this.raster
+ def filter(kernelSize: Int, operation: String): MosaicRasterGDAL = {
+ val tmpPath = PathUtils.createTmpFilePath(getRasterFileExtension)
+
+ this.raster
+ .GetDriver()
+ .CreateCopy(tmpPath, this.raster, 1)
+ .delete()
+
+ val outputRaster = gdal.Open(tmpPath, GF_Write)
+
+ for (bandIndex <- 1 to this.numBands) {
+ val band = this.getBand(bandIndex)
+ val outputBand = outputRaster.GetRasterBand(bandIndex)
+ band.filter(kernelSize, operation, outputBand)
+ }
+
+ val newCreateInfo = Map(
+ "path" -> tmpPath,
+ "parentPath" -> parentPath,
+ "driver" -> getDriversShortName
+ )
+
+ val result = MosaicRasterGDAL(outputRaster, newCreateInfo, this.memSize)
+ result.flushCache()
+ }
/**
* Applies a function to each band of the raster.
@@ -316,159 +369,219 @@ case class MosaicRasterGDAL(
def transformBands[T](f: MosaicRasterBandGDAL => T): Seq[T] = for (i <- 1 to numBands) yield f(getBand(i))
/**
+ * Applies clipping to get cellid raster.
+ * @param cellID
+ * Clip the raster based on the cell id geometry.
+ * @param indexSystem
+ * Default is H3.
+ * @param geometryAPI
+ * Default is JTS.
* @return
- * Returns MosaicGeometry representing bounding box of the raster.
+ * Returns [[MosaicRasterGDAL]] for a given cell ID. Used for tessellation.
*/
- def bbox(geometryAPI: GeometryAPI, destCRS: SpatialReference = MosaicGDAL.WSG84): MosaicGeometry = {
- val gt = getGeoTransform
+ def getRasterForCell(cellID: Long, indexSystem: IndexSystem, geometryAPI: GeometryAPI): MosaicRasterGDAL = {
+ val cellGeom = indexSystem.indexToGeometry(cellID, geometryAPI)
+ val geomCRS = indexSystem.osrSpatialRef
+ RasterClipByVector.clip(this, cellGeom, geomCRS, geometryAPI)
+ }
- val sourceCRS = getSpatialReference
- val transform = new osr.CoordinateTransformation(sourceCRS, destCRS)
+ /////////////////////////////////////////
+ // Subdataset Functions
+ /////////////////////////////////////////
- val bbox = geometryAPI.geometry(
- Seq(
- Seq(gt(0), gt(3)),
- Seq(gt(0) + gt(1) * xSize, gt(3)),
- Seq(gt(0) + gt(1) * xSize, gt(3) + gt(5) * ySize),
- Seq(gt(0), gt(3) + gt(5) * ySize)
- ).map(geometryAPI.fromCoords),
- POLYGON
+ /**
+ * Get a particular subdataset by name.
+ * @param subsetName
+ * The name of the subdataset to get.
+ * @return
+ * Returns [[MosaicRasterGDAL]].
+ */
+ def getSubdataset(subsetName: String): MosaicRasterGDAL = {
+ val sPath = subdatasets.get(s"${subsetName}_tmp")
+ val gdalError = gdal.GetLastErrorMsg()
+ val error = sPath match {
+ case Some(_) => ""
+ case None => s"""
+ |Subdataset $subsetName not found!
+ |Available subdatasets:
+ | ${subdatasets.keys.filterNot(_.startsWith("SUBDATASET_")).mkString(", ")}
+ | """.stripMargin
+ }
+ val sanitized = PathUtils.getCleanPath(sPath.getOrElse(PathUtils.NO_PATH_STRING))
+ val subdatasetPath = PathUtils.getSubdatasetPath(sanitized)
+
+ val ds = pathAsDataset(subdatasetPath)
+ // Avoid costly IO to compute MEM size here
+ // It will be available when the raster is serialized for next operation
+ // If value is needed then it will be computed when getMemSize is called
+ val newCreateInfo = Map(
+ "path" -> sPath.getOrElse(PathUtils.NO_PATH_STRING),
+ "parentPath" -> parentPath,
+ "driver" -> getDriversShortName,
+ "last_error" -> {
+ if (gdalError.nonEmpty || error.nonEmpty) s"""
+ |GDAL Error: $gdalError
+ |$error
+ |""".stripMargin
+ else ""
+ }
)
+ MosaicRasterGDAL(ds, newCreateInfo, -1)
+ }
- val geom1 = org.gdal.ogr.ogr.CreateGeometryFromWkb(bbox.toWKB)
- geom1.Transform(transform)
+ /**
+ * Test if path is a subdataset.
+ * @return boolean
+ */
+ def isSubDataset: Boolean = {
+ val isSubdataset = PathUtils.isSubdataset(path)
+ isSubdataset
+ }
- geometryAPI.geometry(geom1.ExportToWkb(), "WKB")
+ /** @return Returns the raster's subdatasets as a Map. */
+ def subdatasets: Map[String, String] = {
+ val dict = Try(raster.GetMetadata_Dict("SUBDATASETS"))
+ .getOrElse(new java.util.Hashtable[String, String]())
+ val subdatasetsMap = Option(dict)
+ .map(_.asScala.toMap.asInstanceOf[Map[String, String]])
+ .getOrElse(Map.empty[String, String])
+ val keys = subdatasetsMap.keySet
+ val sanitizedParentPath = PathUtils.getCleanPath(parentPath)
+ keys.flatMap(key =>
+ if (key.toUpperCase(Locale.ROOT).contains("NAME")) {
+ val path = subdatasetsMap(key)
+ val pieces = path.split(":")
+ Seq(
+ key -> pieces.last,
+ s"${pieces.last}_tmp" -> path,
+ pieces.last -> s"${pieces.head}:$sanitizedParentPath:${pieces.last}"
+ )
+ } else Seq(key -> subdatasetsMap(key))
+ ).toMap
}
+ /////////////////////////////////////////
+ // Band Functions
+ /////////////////////////////////////////
+
/**
+ * @param bandId
+ * The band index to read.
* @return
- * True if the raster is empty, false otherwise. May be expensive to
- * compute since it requires reading the raster and computing statistics.
+ * Returns the raster's band as a [[MosaicRasterBandGDAL]] object.
*/
- def isEmpty: Boolean = {
- val bands = getBands
- if (bands.isEmpty) {
- subdatasets.values
- .filter(_.toLowerCase(Locale.ROOT).startsWith(getDriversShortName.toLowerCase(Locale.ROOT)))
- .flatMap(bp => readRaster(createInfo + ("path" -> bp)).getBands)
- .takeWhile(_.isEmpty)
- .nonEmpty
+ def getBand(bandId: Int): MosaicRasterBandGDAL = {
+ if (bandId > 0 && numBands >= bandId) {
+ MosaicRasterBandGDAL(raster.GetRasterBand(bandId), bandId)
} else {
- bands.takeWhile(_.isEmpty).nonEmpty
+ throw new ArrayIndexOutOfBoundsException()
}
}
- /**
- * @return
- * Returns the raster's path.
- */
- def getPath: String = path
+ /** @return Returns a map of the raster band(s) statistics. */
+ def getBandStats: Map[Int, Map[String, Double]] = {
+ (1 to numBands)
+ .map(i => {
+ val band = raster.GetRasterBand(i)
+ val min = Array.ofDim[Double](1)
+ val max = Array.ofDim[Double](1)
+ val mean = Array.ofDim[Double](1)
+ val stddev = Array.ofDim[Double](1)
+ band.GetStatistics(true, true, min, max, mean, stddev)
+ i -> Map(
+ "min" -> min(0),
+ "max" -> max(0),
+ "mean" -> mean(0),
+ "stddev" -> stddev(0)
+ )
+ })
+ .toMap
+ }
- /**
- * @return
- * Returns the raster for a given cell ID. Used for tessellation.
- */
- def getRasterForCell(cellID: Long, indexSystem: IndexSystem, geometryAPI: GeometryAPI): MosaicRasterGDAL = {
- val cellGeom = indexSystem.indexToGeometry(cellID, geometryAPI)
- val geomCRS = indexSystem.osrSpatialRef
- RasterClipByVector.clip(this, cellGeom, geomCRS, geometryAPI)
+ /** @return Returns a map of raster band(s) valid pixel count. */
+ def getValidCount: Map[Int, Long] = {
+ (1 to numBands)
+ .map(i => {
+ val band = raster.GetRasterBand(i)
+ val validCount = band.AsMDArray().GetStatistics().getValid_count
+ i -> validCount
+ })
+ .toMap
}
+ /////////////////////////////////////////
+ // Raster Lifecycle Functions
+ /////////////////////////////////////////
+
/**
* Cleans up the raster driver and references.
- *
+ * - This will not clean up a file stored in a Databricks location,
+ * meaning DBFS, Volumes, or Workspace paths are skipped.
* Unlinks the raster file. After this operation the raster object is no
* longer usable. To be used as last step in expression after writing to
* bytes.
*/
def cleanUp(): Unit = {
- val isSubdataset = PathUtils.isSubdataset(path)
- val filePath = if (isSubdataset) PathUtils.fromSubdatasetPath(path) else path
- val pamFilePath = s"$filePath.aux.xml"
- val cleanPath = filePath.replace("/vsizip/", "")
- val zipPath = if (cleanPath.endsWith("zip")) cleanPath else s"$cleanPath.zip"
- if (path != PathUtils.getCleanPath(parentPath)) {
+ // 0.4.2 - don't delete any fuse locations.
+ if (!PathUtils.isFuseLocation(path) && path != PathUtils.getCleanPath(parentPath)) {
Try(gdal.GetDriverByName(getDriversShortName).Delete(path))
- Try(Files.deleteIfExists(Paths.get(cleanPath)))
- Try(Files.deleteIfExists(Paths.get(path)))
- Try(Files.deleteIfExists(Paths.get(filePath)))
- Try(Files.deleteIfExists(Paths.get(pamFilePath)))
- if (Files.exists(Paths.get(zipPath))) {
- Try(Files.deleteIfExists(Paths.get(zipPath.replace(".zip", ""))))
- }
- Try(Files.deleteIfExists(Paths.get(zipPath)))
+ PathUtils.cleanUpPath(path)
}
}
/**
- * @note
- * If memory size is -1 this will destroy the raster and you will need to
- * refresh it to use it again.
- * @return
- * Returns the amount of memory occupied by the file in bytes.
+ * Destroys the raster object. After this operation the raster object is no
+ * longer usable. If the raster is needed again, use the refresh method.
*/
- def getMemSize: Long = {
- if (memSize == -1) {
- val toRead = if (path.startsWith("/vsizip/")) path.replace("/vsizip/", "") else path
- if (Files.notExists(Paths.get(toRead))) {
- throw new Exception(s"File not found: ${gdal.GetLastErrorMsg()}")
- }
- Files.size(Paths.get(toRead))
- } else {
- memSize
+ def destroy(): Unit = {
+ val raster = getRaster
+ if (raster != null) {
+ raster.FlushCache()
+ raster.delete()
}
-
}
/**
- * Writes a raster to a file system path. This method disposes of the
- * raster object. If the raster is needed again, load it from the path.
- *
- * @param path
- * The path to the raster file.
+ * Flushes the cache of the raster. This is needed to ensure that the
+ * raster is written to disk. This is needed for operations like
+ * RasterProject.
* @return
- * A boolean indicating if the write was successful.
+ * Returns the [[MosaicRasterGDAL]] object.
+ */
+ def flushCache(): MosaicRasterGDAL = {
+ // Note: Do not wrap GDAL objects into Option
+ if (getRaster != null) getRaster.FlushCache()
+ this.destroy()
+ this.refresh()
+ }
+
+ /**
+ * Refreshes the raster object. This is needed after writing to a file
+ * system path. GDAL only properly writes to a file system path if the
+ * raster object is destroyed. After refresh operation the raster object is
+ * usable again.
+ * Returns [[MosaicRasterGDAL]].
*/
- def writeToPath(path: String, dispose: Boolean = true): String = {
- if (isSubDataset) {
- val driver = raster.GetDriver()
- val ds = driver.CreateCopy(path, this.flushCache().getRaster, 1)
- if (ds == null) {
- val error = gdal.GetLastErrorMsg()
- throw new Exception(s"Error writing raster to path: $error")
- }
- ds.FlushCache()
- ds.delete()
- if (dispose) RasterCleaner.dispose(this)
- path
- } else {
- val thisPath = Paths.get(this.path)
- val fromDir = thisPath.getParent
- val toDir = Paths.get(path).getParent
- val stemRegex = PathUtils.getStemRegex(this.path)
- PathUtils.wildcardCopy(fromDir.toString, toDir.toString, stemRegex)
- if (dispose) RasterCleaner.dispose(this)
- s"$toDir/${thisPath.getFileName}"
- }
+ def refresh(): MosaicRasterGDAL = {
+ MosaicRasterGDAL(pathAsDataset(path), createInfo, memSize)
}
/**
* Writes a raster to a byte array.
- *
+ * @param dispose
+ * Whether to dispose of the raster object, default is true.
* @return
* A byte array containing the raster data.
*/
def writeToBytes(dispose: Boolean = true): Array[Byte] = {
- val isSubdataset = PathUtils.isSubdataset(path)
val readPath = {
val tmpPath =
- if (isSubdataset) {
+ if (isSubDataset) {
val tmpPath = PathUtils.createTmpFilePath(getRasterFileExtension)
writeToPath(tmpPath, dispose = false)
tmpPath
} else {
- path
+ this.path
}
if (Files.isDirectory(Paths.get(tmpPath))) {
val parentDir = Paths.get(tmpPath).getParent.toString
@@ -496,187 +609,73 @@ case class MosaicRasterGDAL(
}
/**
- * Destroys the raster object. After this operation the raster object is no
- * longer usable. If the raster is needed again, use the refresh method.
- */
- def destroy(): Unit = {
- val raster = getRaster
- if (raster != null) {
- raster.FlushCache()
- raster.delete()
- }
- }
-
- /**
- * Refreshes the raster object. This is needed after writing to a file
- * system path. GDAL only properly writes to a file system path if the
- * raster object is destroyed. After refresh operation the raster object is
- * usable again.
- */
- def refresh(): MosaicRasterGDAL = {
- MosaicRasterGDAL(openRaster(path), createInfo, memSize)
- }
-
- /**
- * @return
- * Returns the raster's size.
- */
- def getDimensions: (Int, Int) = (xSize, ySize)
-
- /**
- * @return
- * Returns the raster's band statistics.
- */
- def getBandStats: Map[Int, Map[String, Double]] = {
- (1 to numBands)
- .map(i => {
- val band = raster.GetRasterBand(i)
- val min = Array.ofDim[Double](1)
- val max = Array.ofDim[Double](1)
- val mean = Array.ofDim[Double](1)
- val stddev = Array.ofDim[Double](1)
- band.GetStatistics(true, true, min, max, mean, stddev)
- i -> Map(
- "min" -> min(0),
- "max" -> max(0),
- "mean" -> mean(0),
- "stddev" -> stddev(0)
- )
- })
- .toMap
- }
-
- /**
- * @return
- * Returns the raster's band valid pixel count.
- */
- def getValidCount: Map[Int, Long] = {
- (1 to numBands)
- .map(i => {
- val band = raster.GetRasterBand(i)
- val validCount = band.AsMDArray().GetStatistics().getValid_count
- i -> validCount
- })
- .toMap
- }
-
- /**
- * @param subsetName
- * The name of the subdataset to get.
+ * Writes a raster to a file system path. This method disposes of the
+ * raster object. If the raster is needed again, load it from the path.
+ * @param newPath
+ * The path to the raster file.
+ * @param dispose
+ * Whether to dispose of the raster object, default is true.
* @return
- * Returns the raster's subdataset with given name.
+ * The path where written.
*/
- def getSubdataset(subsetName: String): MosaicRasterGDAL = {
- val path = subdatasets.get(s"${subsetName}_tmp")
- val gdalError = gdal.GetLastErrorMsg()
- val error = path match {
- case Some(_) => ""
- case None => s"""
- |Subdataset $subsetName not found!
- |Available subdatasets:
- | ${subdatasets.keys.filterNot(_.startsWith("SUBDATASET_")).mkString(", ")}
- | """.stripMargin
+ def writeToPath(newPath: String, dispose: Boolean = true): String = {
+ if (isSubDataset) {
+ val driver = raster.GetDriver()
+ val ds = driver.CreateCopy(newPath, this.flushCache().getRaster, 1)
+ if (ds == null) {
+ val error = gdal.GetLastErrorMsg()
+ throw new Exception(s"Error writing raster to path: $error")
+ }
+ ds.FlushCache()
+ ds.delete()
+ if (dispose) RasterCleaner.dispose(this)
+ newPath
+ } else {
+ val thisPath = Paths.get(this.path)
+ val fromDir = thisPath.getParent
+ val toDir = Paths.get(newPath).getParent
+ val stemRegex = PathUtils.getStemRegex(this.path)
+ PathUtils.wildcardCopy(fromDir.toString, toDir.toString, stemRegex)
+ if (dispose) RasterCleaner.dispose(this)
+ s"$toDir/${thisPath.getFileName}"
}
- val sanitized = PathUtils.getCleanPath(path.getOrElse(PathUtils.NO_PATH_STRING))
- val subdatasetPath = PathUtils.getSubdatasetPath(sanitized)
-
- val ds = openRaster(subdatasetPath)
- // Avoid costly IO to compute MEM size here
- // It will be available when the raster is serialized for next operation
- // If value is needed then it will be computed when getMemSize is called
- val createInfo = Map(
- "path" -> path.getOrElse(PathUtils.NO_PATH_STRING),
- "parentPath" -> parentPath,
- "driver" -> getDriversShortName,
- "last_error" -> {
- if (gdalError.nonEmpty || error.nonEmpty) s"""
- |GDAL Error: $gdalError
- |$error
- |""".stripMargin
- else ""
- }
- )
- MosaicRasterGDAL(ds, createInfo, -1)
}
- def convolve(kernel: Array[Array[Double]]): MosaicRasterGDAL = {
- val resultRasterPath = PathUtils.createTmpFilePath(getRasterFileExtension)
+ ///////////////////////////////////////////////////
+ // Additional Getters
+ ///////////////////////////////////////////////////
- this.raster
- .GetDriver()
- .CreateCopy(resultRasterPath, this.raster, 1)
- .delete()
+ /** @return Returns the raster's bands as a Seq. */
+ def getBands: Seq[MosaicRasterBandGDAL] = (1 to numBands).map(getBand)
- val outputRaster = gdal.Open(resultRasterPath, GF_Write)
-
- for (bandIndex <- 1 to this.numBands) {
- val band = this.getBand(bandIndex)
- val outputBand = outputRaster.GetRasterBand(bandIndex)
- band.convolve(kernel, outputBand)
- }
+ /** @return Returns a tuple with the raster's size. */
+ def getDimensions: (Int, Int) = (xSize, ySize)
- val createInfo = Map(
- "path" -> resultRasterPath,
- "parentPath" -> parentPath,
- "driver" -> getDriversShortName
+ /** @return The raster's driver short name. */
+ def getDriversShortName: String =
+ driverShortName.getOrElse(
+ Try(raster.GetDriver().getShortName).getOrElse("NONE")
)
- val result = MosaicRasterGDAL(outputRaster, createInfo, this.memSize)
- result.flushCache()
- }
-
- def filter(kernelSize: Int, operation: String): MosaicRasterGDAL = {
- val resultRasterPath = PathUtils.createTmpFilePath(getRasterFileExtension)
-
- this.raster
- .GetDriver()
- .CreateCopy(resultRasterPath, this.raster, 1)
- .delete()
-
- val outputRaster = gdal.Open(resultRasterPath, GF_Write)
+ /** @return The raster's path on disk. Usually this is a parent file for the tile. */
+ def getParentPath: String = parentPath
- for (bandIndex <- 1 to this.numBands) {
- val band = this.getBand(bandIndex)
- val outputBand = outputRaster.GetRasterBand(bandIndex)
- band.filter(kernelSize, operation, outputBand)
- }
+ /** @return Returns the raster's path. */
+ def getPath: String = path
- val createInfo = Map(
- "path" -> resultRasterPath,
- "parentPath" -> parentPath,
- "driver" -> getDriversShortName
- )
+ /** @return Underlying GDAL raster object. */
+ def getRaster: Dataset = this.raster
- val result = MosaicRasterGDAL(outputRaster, createInfo, this.memSize)
- result.flushCache()
- }
+ /** @return Returns file extension. */
+ def getRasterFileExtension: String = GDAL.getExtension(getDriversShortName)
}
+
//noinspection ZeroIndexToHead
/** Companion object for MosaicRasterGDAL Implements RasterReader APIs */
object MosaicRasterGDAL extends RasterReader {
- /**
- * Opens a raster from a file system path with a given driver.
- * @param driverShortName
- * The driver short name to use. If None, then GDAL will try to identify
- * the driver from the file extension
- * @param path
- * The path to the raster file.
- * @return
- * A MosaicRaster object.
- */
- def openRaster(path: String, driverShortName: Option[String]): Dataset = {
- driverShortName match {
- case Some(driverShortName) =>
- val drivers = new JVector[String]()
- drivers.add(driverShortName)
- gdal.OpenEx(path, GA_ReadOnly, drivers)
- case None => gdal.Open(path, GA_ReadOnly)
- }
- }
-
/**
* Identifies the driver of a raster from a file system path.
* @param parentPath
@@ -696,63 +695,54 @@ object MosaicRasterGDAL extends RasterReader {
}
/**
- * Reads a raster from a file system path. Reads a subdataset if the path
- * is to a subdataset.
- *
+ * Opens a raster from a file system path with a given driver.
+ * @param path
+ * The path to the raster file.
+ * @param driverShortName
+ * The driver short name to use. If None, then GDAL will try to identify
+ * the driver from the file extension
+ * @return
+ * A GDAL [[Dataset]] object.
+ */
+ def pathAsDataset(path: String, driverShortName: Option[String]): Dataset = {
+ driverShortName match {
+ case Some(driverShortName) =>
+ val drivers = new JVector[String]()
+ drivers.add(driverShortName)
+ gdal.OpenEx(path, GA_ReadOnly, drivers)
+ case None => gdal.Open(path, GA_ReadOnly)
+ }
+ }
+
+ /**
+ * Reads a raster band from a file system path. Reads a subdataset band if
+ * the path is to a subdataset.
* @example
- * Raster: path = "file:///path/to/file.tif" Subdataset: path =
- * "file:///path/to/file.tif:subdataset"
+ * Raster: path = "/path/to/file.tif" Subdataset: path =
+ * "FORMAT:/path/to/file.tif:subdataset"
+ * @param bandIndex
+ * The band index to read (1+ indexed).
* @param createInfo
- * The create info for the raster. This should contain the following
- * keys:
- * - path: The path to the raster file.
- * - parentPath: The path of the parent raster file.
+ * Map of create info for the raster.
* @return
- * A MosaicRaster object.
+ * A [[MosaicRasterGDAL]] object.
*/
- override def readRaster(createInfo: Map[String, String]): MosaicRasterGDAL = {
- val inPath = createInfo("path")
- val isSubdataset = PathUtils.isSubdataset(inPath)
- val path = PathUtils.getCleanPath(inPath)
- val readPath =
- if (isSubdataset) PathUtils.getSubdatasetPath(path)
- else PathUtils.getZipPath(path)
- val dataset = openRaster(readPath, None)
- val error =
- if (dataset == null) {
- val error = gdal.GetLastErrorMsg()
- s"""
- Error reading raster from path: $readPath
- Error: $error
- """
- } else ""
- val driverShortName = Try(dataset.GetDriver().getShortName).getOrElse("NONE")
- // Avoid costly IO to compute MEM size here
- // It will be available when the raster is serialized for next operation
- // If value is needed then it will be computed when getMemSize is called
- // We cannot just use memSize value of the parent due to the fact that the raster could be a subdataset
- val raster = MosaicRasterGDAL(
- dataset,
- createInfo ++
- Map(
- "driver" -> driverShortName,
- "last_error" -> error
- ),
- -1
- )
- raster
+ override def readBand(bandIndex: Int, createInfo: Map[String, String]): MosaicRasterBandGDAL = {
+ val raster = readRaster(createInfo)
+ // Note: Raster and Band are coupled, this can cause a pointer leak
+ raster.getBand(bandIndex)
}
/**
- * Reads a raster from a byte array.
+ * Reads a raster from a byte array. Expects "driver" in createInfo.
* @param contentBytes
* The byte array containing the raster data.
* @param createInfo
* Mosaic creation info of the raster. Note: This is not the same as the
* metadata of the raster. This is not the same as GDAL creation options.
* @return
- * A MosaicRaster object.
- */
+ * A [[MosaicRasterGDAL]] object.
+ */
override def readRaster(contentBytes: Array[Byte], createInfo: Map[String, String]): MosaicRasterGDAL = {
if (Option(contentBytes).isEmpty || contentBytes.isEmpty) {
MosaicRasterGDAL(null, createInfo, -1)
@@ -764,13 +754,13 @@ object MosaicRasterGDAL extends RasterReader {
val tmpPath = PathUtils.createTmpFilePath(extension)
Files.write(Paths.get(tmpPath), contentBytes)
// Try reading as a tmp file, if that fails, rename as a zipped file
- val dataset = openRaster(tmpPath, Some(driverShortName))
+ val dataset = pathAsDataset(tmpPath, Some(driverShortName))
if (dataset == null) {
val zippedPath = s"$tmpPath.zip"
Files.move(Paths.get(tmpPath), Paths.get(zippedPath), StandardCopyOption.REPLACE_EXISTING)
val readPath = PathUtils.getZipPath(zippedPath)
- val ds = openRaster(readPath, Some(driverShortName))
- if (ds == null) {
+ val ds1 = pathAsDataset(readPath, Some(driverShortName))
+ if (ds1 == null) {
// the way we zip using uuid is not compatible with GDAL
// we need to unzip and read the file if it was zipped by us
val parentDir = Paths.get(zippedPath).getParent
@@ -781,13 +771,13 @@ object MosaicRasterGDAL extends RasterReader {
val extension = GDAL.getExtension(driverShortName)
val lastExtracted = SysUtils.getLastOutputLine(prompt)
val unzippedPath = PathUtils.parseUnzippedPathFromExtracted(lastExtracted, extension)
- val dataset = openRaster(unzippedPath, Some(driverShortName))
- if (dataset == null) {
+ val ds2 = pathAsDataset(unzippedPath, Some(driverShortName))
+ if (ds2 == null) {
throw new Exception(s"Error reading raster from bytes: ${prompt._3}")
}
- MosaicRasterGDAL(dataset, createInfo + ("path" -> unzippedPath), contentBytes.length)
+ MosaicRasterGDAL(ds2, createInfo + ("path" -> unzippedPath), contentBytes.length)
} else {
- MosaicRasterGDAL(ds, createInfo + ("path" -> readPath), contentBytes.length)
+ MosaicRasterGDAL(ds1, createInfo + ("path" -> readPath), contentBytes.length)
}
} else {
MosaicRasterGDAL(dataset, createInfo + ("path" -> tmpPath), contentBytes.length)
@@ -796,27 +786,47 @@ object MosaicRasterGDAL extends RasterReader {
}
/**
- * Reads a raster band from a file system path. Reads a subdataset band if
- * the path is to a subdataset.
- *
+ * Reads a raster from a file system path. Reads a subdataset if the path
+ * is to a subdataset.
* @example
- * Raster: path = "file:///path/to/file.tif" Subdataset: path =
- * "file:///path/to/file.tif:subdataset"
+ * Raster: path = "/path/to/file.tif" Subdataset: path =
+ * "FORMAT:/path/to/file.tif:subdataset"
* @param createInfo
- * The create info for the raster. This should contain the following
- * keys:
- * - path: The path to the raster file.
- * - parentPath: The path of the parent raster file.
- * - driver: Optional: The driver short name of the raster file
- * @param bandIndex
- * The band index to read.
+ * Map of create info for the raster.
* @return
- * A MosaicRaster object.
+ * A [[MosaicRasterGDAL]] object.
*/
- override def readBand(bandIndex: Int, createInfo: Map[String, String]): MosaicRasterBandGDAL = {
- val raster = readRaster(createInfo)
- // TODO: Raster and Band are coupled, this can cause a pointer leak
- raster.getBand(bandIndex)
+ override def readRaster(createInfo: Map[String, String]): MosaicRasterGDAL = {
+ val inPath = createInfo("path")
+ val isSubdataset = PathUtils.isSubdataset(inPath)
+ val cleanPath = PathUtils.getCleanPath(inPath)
+ val readPath =
+ if (isSubdataset) PathUtils.getSubdatasetPath(cleanPath)
+ else PathUtils.getZipPath(cleanPath)
+ val dataset = pathAsDataset(readPath, None)
+ val error =
+ if (dataset == null) {
+ val error = gdal.GetLastErrorMsg()
+ s"""
+ Error reading raster from path: $readPath
+ Error: $error
+ """
+ } else ""
+ val driverShortName = Try(dataset.GetDriver().getShortName).getOrElse("NONE")
+ // Avoid costly IO to compute MEM size here
+ // It will be available when the raster is serialized for next operation
+ // If value is needed then it will be computed when getMemSize is called
+ // We cannot just use memSize value of the parent due to the fact that the raster could be a subdataset
+ val raster = MosaicRasterGDAL(
+ dataset,
+ createInfo ++
+ Map(
+ "driver" -> driverShortName,
+ "last_error" -> error
+ ),
+ -1
+ )
+ raster
}
}
diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/io/RasterReader.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/io/RasterReader.scala
index d8a1a90c1..07add8e0e 100644
--- a/src/main/scala/com/databricks/labs/mosaic/core/raster/io/RasterReader.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/io/RasterReader.scala
@@ -4,9 +4,10 @@ import com.databricks.labs.mosaic.core.raster.gdal.{MosaicRasterBandGDAL, Mosaic
import org.apache.spark.internal.Logging
/**
- * RasterReader is a trait that defines the interface for reading raster data
- * from a file system path. It is used by the RasterAPI to read raster and
- * raster band data.
+ * RasterReader is a trait that defines the interface for loading raster data into
+ * tile struct from a file system path or contents. It is used by the RasterAPI to
+ * read raster and raster band data. MosaicRasterGDAL is the internal object generated
+ * from the data.
* @note
* For subdatasets the path should be the path to the subdataset and not to
* the file.
@@ -14,55 +15,45 @@ import org.apache.spark.internal.Logging
trait RasterReader extends Logging {
/**
- * Reads a raster from a file system path. Reads a subdataset if the path
- * is to a subdataset.
- *
- * @example
- * Raster: path = "/path/to/file.tif" Subdataset: path =
- * "FORMAT:/path/to/file.tif:subdataset"
- * @param createInfo
- * The create info for the raster. This should contain the following
- * keys:
- * - path: The path to the raster file.
- * - parentPath: The path of the parent raster file.
- * - driver: Optional: The driver short name of the raster file
- * @return
- * A MosaicRaster object.
- */
- def readRaster(createInfo: Map[String, String]): MosaicRasterGDAL
+ * Reads a raster band from a file system path. Reads a subdataset band if
+ * the path is to a subdataset. Assumes "path" is a key in createInfo.
+ *
+ * @example
+ * Raster: path = "/path/to/file.tif" Subdataset: path =
+ * "FORMAT:/path/to/file.tif:subdataset"
+ * @param bandIndex
+ * The band index to read (1+ indexed).
+ * @param createInfo
+ * Map of create info for the raster.
+ * @return
+ * A [[MosaicRasterBandGDAL]] object.
+ */
+ def readBand(bandIndex: Int, createInfo: Map[String, String]): MosaicRasterBandGDAL
/**
- * Reads a raster from an in memory buffer. Use the buffer bytes to produce
- * a uuid of the raster.
- *
- * @param contentBytes
- * The file bytes.
- * @param createInfo
- * The create info for the raster. This should contain the following
- * keys:
- * - parentPath: The path of the parent raster file.
- * - driver: The driver short name of the raster file
- * @return
- * A MosaicRaster object.
- */
+ * Reads a raster from a byte array. Expects "driver" in createInfo.
+ * @param contentBytes
+ * The byte array containing the raster data.
+ * @param createInfo
+ * Mosaic creation info of the raster. Note: This is not the same as the
+ * metadata of the raster. This is not the same as GDAL creation options.
+ * @return
+ * A [[MosaicRasterGDAL]] object.
+ */
def readRaster(contentBytes: Array[Byte], createInfo: Map[String, String]): MosaicRasterGDAL
/**
- * Reads a raster band from a file system path. Reads a subdataset band if
- * the path is to a subdataset.
+ * Reads a raster from a file system path. Reads a subdataset if the path
+ * is to a subdataset. Assumes "path" is a key in createInfo.
*
* @example
* Raster: path = "/path/to/file.tif" Subdataset: path =
* "FORMAT:/path/to/file.tif:subdataset"
* @param createInfo
- * The create info for the raster. This should contain the following
- * keys:
- * - path: The path to the raster file.
- * - parentPath: The path of the parent raster file.
- * - driver: Optional: The driver short name of the raster file
+ * Map of create info for the raster.
* @return
- * A MosaicRaster object.
+ * A [[MosaicRasterGDAL]] object.
*/
- def readBand(bandIndex: Int, createInfo: Map[String, String]): MosaicRasterBandGDAL
+ def readRaster(createInfo: Map[String, String]): MosaicRasterGDAL
}
diff --git a/src/main/scala/com/databricks/labs/mosaic/core/raster/io/RasterWriter.scala b/src/main/scala/com/databricks/labs/mosaic/core/raster/io/RasterWriter.scala
index fa6848d65..2f80b97c7 100644
--- a/src/main/scala/com/databricks/labs/mosaic/core/raster/io/RasterWriter.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/core/raster/io/RasterWriter.scala
@@ -2,31 +2,32 @@ package com.databricks.labs.mosaic.core.raster.io
/**
* RasterWriter is a trait that defines the interface for writing raster data
- * to a file system path or as bytes. It is used by the RasterAPI to write
- * rasters.
+ * to a file system path or as bytes. It is used by the [[com.databricks.labs.mosaic.core.raster.api.GDAL]]
+ * Raster API to write rasters from the internal [[com.databricks.labs.mosaic.core.raster.gdal.MosaicRasterGDAL]]
+ * object.
*/
trait RasterWriter {
/**
- * Writes a raster to a file system path.
- *
- * @param path
- * The path to the raster file.
- * @param destroy
- * A boolean indicating if the raster should be destroyed after writing.
- * @return
- * A boolean indicating if the write was successful.
- */
- def writeToPath(path: String, destroy: Boolean = true): String
+ * Writes a raster to a byte array.
+ *
+ * @param destroy
+ * A boolean indicating if the raster should be destroyed after writing.
+ * @return
+ * A byte array containing the raster data.
+ */
+ def writeToBytes(destroy: Boolean = true): Array[Byte]
/**
- * Writes a raster to a byte array.
+ * Writes a raster to a specified file system path.
*
+ * @param newPath
+ * The path to write the raster.
* @param destroy
* A boolean indicating if the raster should be destroyed after writing.
* @return
- * A byte array containing the raster data.
+ * The path where written (may differ, e.g. due to subdatasets).
*/
- def writeToBytes(destroy: Boolean = true): Array[Byte]
+ def writeToPath(newPath: String, destroy: Boolean = true): String
}
diff --git a/src/main/scala/com/databricks/labs/mosaic/core/types/RasterTileType.scala b/src/main/scala/com/databricks/labs/mosaic/core/types/RasterTileType.scala
index 137d482ce..d611252ad 100644
--- a/src/main/scala/com/databricks/labs/mosaic/core/types/RasterTileType.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/core/types/RasterTileType.scala
@@ -1,12 +1,14 @@
package com.databricks.labs.mosaic.core.types
+import com.databricks.labs.mosaic.core.types.RasterTileType.getRasterDataType
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.types._
/** Type definition for the raster tile. */
-class RasterTileType(fields: Array[StructField]) extends StructType(fields) {
+class RasterTileType(fields: Array[StructField], useCheckpoint: Boolean) extends StructType(fields) {
- def rasterType: DataType = fields.find(_.name == "raster").get.dataType
+ def rasterType: DataType = getRasterDataType(
+ fields.find(_.name == "raster").get.dataType, useCheckpoint)
override def simpleString: String = "RASTER_TILE"
@@ -17,54 +19,89 @@ class RasterTileType(fields: Array[StructField]) extends StructType(fields) {
object RasterTileType {
/**
- * Creates a new instance of [[RasterTileType]].
+ * Change data type to [[StringType]] if using checkpointing.
+ * @param dataType
+ * Data type to use if not checkpointing.
+ * @param useCheckpoint
+ * Use to test for checkpointing enabled.
+ * @return
+ * Returns [[DataType]] based on checkpointing enabled.
+ */
+ def getRasterDataType(dataType: DataType, useCheckpoint: Boolean): DataType = {
+ // change to StringType if using checkpoint
+ dataType match {
+ case _: BinaryType if useCheckpoint => StringType
+ case t => t
+ }
+ }
+
+ /**
+ * [APPLY-1] - Creates a new instance of [[RasterTileType]] with Array.
*
* @param idType
- * Type of the index ID. Can be one of [[LongType]], [[IntegerType]] or
- * [[StringType]].
+ * Cellid type, can be one of [[LongType]], [[IntegerType]] or [[StringType]].
* @param rasterType
* Type of the raster. Can be one of [[ByteType]] or [[StringType]]. Not
* to be confused with the data type of the raster. This is the type of
* the column that contains the raster.
- *
+ * @param useCheckpoint
+ * Use to test for checkpointing enabled.
* @return
- * An instance of [[RasterTileType]].
+ * Array RasterTileType [[DataType]].
*/
- def apply(idType: DataType, rasterType: DataType): DataType = {
+ def apply(idType: DataType, rasterType: DataType, useCheckpoint: Boolean): DataType = {
require(Seq(LongType, IntegerType, StringType).contains(idType))
new RasterTileType(
- Array(
- StructField("index_id", idType),
- StructField("raster", rasterType),
- StructField("metadata", MapType(StringType, StringType))
- )
+ Array(
+ StructField("index_id", idType),
+ StructField("raster", getRasterDataType(rasterType, useCheckpoint)),
+ StructField("metadata", MapType(StringType, StringType))
+ ),
+ useCheckpoint
)
}
/**
- * Creates a new instance of [[RasterTileType]].
+ * [APPLY-2] - Creates a new instance of [[RasterTileType]].
+ * Internally, calls [APPLY-1].
*
* @param idType
- * Type of the index ID. Can be one of [[LongType]], [[IntegerType]] or
- * [[StringType]].
+ * Cellid type, can be one of [[LongType]], [[IntegerType]] or [[StringType]].
* @param tileExpr
- * Expression containing a tile. This is used to infer the raster type
- * when chaining expressions.
+ * Expression containing a tile. This is used to infer the raster type
+ * when chaining expressions; may be an array of tiles.
+ * @param useCheckpoint
+ * Use to test for checkpointing enabled.
* @return
+ * RasterTileType as [[DataType]]
*/
- def apply(idType: DataType, tileExpr: Expression): DataType = {
+ def apply(idType: DataType, tileExpr: Expression, useCheckpoint: Boolean): DataType = {
require(Seq(LongType, IntegerType, StringType).contains(idType))
tileExpr.dataType match {
- case st @ StructType(_) => apply(idType, st.find(_.name == "raster").get.dataType)
- case _ @ArrayType(elementType: StructType, _) => apply(idType, elementType.find(_.name == "raster").get.dataType)
+ case st @ StructType(_) =>
+ apply(idType, st.find(_.name == "raster").get.dataType, useCheckpoint)
+ case _ @ArrayType(elementType: StructType, _) =>
+ apply(idType, elementType.find(_.name == "raster").get.dataType, useCheckpoint)
case _ => throw new IllegalArgumentException("Unsupported raster type.")
}
}
- def apply(tileExpr: Expression): RasterTileType = {
+ /**
+ * [APPLY-3] - Creates a new instance of [[RasterTileType]].
+ * Internally, calls class constructor.
+ *
+ * @param tileExpr
+ * Expression containing a tile. This is used to infer the raster type
+ * when chaining expressions; may be an array of tiles.
+ * @param useCheckpoint
+ * Use to test for checkpointing enabled.
+ * @return
+ * [[RasterTileType]]
+ */
+ def apply(tileExpr: Expression, useCheckpoint: Boolean): RasterTileType = {
tileExpr.dataType match {
- case StructType(fields) => new RasterTileType(fields)
- case ArrayType(elementType: StructType, _) => new RasterTileType(elementType.fields)
+ case StructType(fields) => new RasterTileType(fields, useCheckpoint)
+ case ArrayType(elementType: StructType, _) => new RasterTileType(elementType.fields, useCheckpoint)
case _ => throw new IllegalArgumentException("Unsupported raster type.")
}
}
diff --git a/src/main/scala/com/databricks/labs/mosaic/core/types/model/MosaicRasterTile.scala b/src/main/scala/com/databricks/labs/mosaic/core/types/model/MosaicRasterTile.scala
index 97122a7d7..43bbf4d62 100644
--- a/src/main/scala/com/databricks/labs/mosaic/core/types/model/MosaicRasterTile.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/core/types/model/MosaicRasterTile.scala
@@ -23,16 +23,16 @@ case class MosaicRasterTile(
raster: MosaicRasterGDAL
) {
- def parentPath: String = raster.createInfo("parentPath")
-
- def driver: String = raster.createInfo("driver")
-
def getIndex: Either[Long, String] = index
def getParentPath: String = parentPath
+ def parentPath: String = raster.createInfo("parentPath")
+
def getDriver: String = driver
+ def driver: String = raster.createInfo("driver")
+
def getRaster: MosaicRasterGDAL = raster
/**
@@ -49,7 +49,7 @@ case class MosaicRasterTile(
* @param indexSystem
* Index system to use for formatting.
* @return
- * MosaicChip with formatted index ID.
+ * [[MosaicRasterTile]] with formatted index ID.
*/
def formatCellId(indexSystem: IndexSystem): MosaicRasterTile = {
if (Option(index).isEmpty) return this
@@ -68,7 +68,7 @@ case class MosaicRasterTile(
* @param indexSystem
* Index system to use for formatting.
* @return
- * MosaicChip with formatted index ID.
+ * number variation of index id.
*/
def cellIdAsLong(indexSystem: IndexSystem): Long =
index match {
@@ -82,7 +82,7 @@ case class MosaicRasterTile(
* @param indexSystem
* Index system to use for formatting.
* @return
- * MosaicChip with formatted index ID.
+ * string variation of index id.
*/
def cellIdAsStr(indexSystem: IndexSystem): String =
index match {
@@ -91,17 +91,25 @@ case class MosaicRasterTile(
}
/**
- * Serialise to spark internal representation.
+ * Serialize to spark internal representation.
*
+ * @param rasterDataType
+ * How to encode the raster.
+ * - Options are [[StringType]] or [[BinaryType]]
+ * - If checkpointing is used, [[StringType]] will be forced
* @return
* An instance of [[InternalRow]].
*/
- def serialize(
- rasterDataType: DataType
- ): InternalRow = {
+ def serialize(rasterDataType: DataType): InternalRow = {
val encodedRaster = encodeRaster(rasterDataType)
- val path = if (rasterDataType == StringType) encodedRaster.toString else raster.createInfo("path")
- val parentPath = if (raster.createInfo("parentPath").isEmpty) raster.createInfo("path") else raster.createInfo("parentPath")
+ val path = encodedRaster match {
+ case uStr: UTF8String => uStr.toString
+ case _ => raster.createInfo("path")
+ }
+ val parentPath = {
+ if (raster.createInfo("parentPath").isEmpty) raster.createInfo("path")
+ else raster.createInfo("parentPath")
+ }
val newCreateInfo = raster.createInfo + ("path" -> path, "parentPath" -> parentPath)
val mapData = buildMapString(newCreateInfo)
if (Option(index).isDefined) {
@@ -116,17 +124,19 @@ case class MosaicRasterTile(
} else {
InternalRow.fromSeq(Seq(null, encodedRaster, mapData))
}
-
}
/**
- * Encodes the chip geometry as WKB.
+ * Encodes the raster according to the [[DataType]].
*
+ * @param rasterDataType
+ * Specify [[BinaryType]] for byte array or [[StringType]] for path,
+ * as used in checkpointing.
* @return
- * An instance of [[Array]] of [[Byte]] representing WKB.
+ * According to the [[DataType]].
*/
private def encodeRaster(
- rasterDataType: DataType = BinaryType
+ rasterDataType: DataType
): Any = {
GDAL.writeRasters(Seq(raster), rasterDataType).head
}
@@ -144,19 +154,24 @@ object MosaicRasterTile {
/**
* Smart constructor based on Spark internal instance.
+ * - Can handle based on provided raster type.
*
* @param row
* An instance of [[InternalRow]].
* @param idDataType
* The data type of the index ID.
+ * @param rasterDataType
+ * The data type of the tile's raster.
* @return
* An instance of [[MosaicRasterTile]].
*/
- def deserialize(row: InternalRow, idDataType: DataType, rasterType: DataType): MosaicRasterTile = {
+ def deserialize(row: InternalRow, idDataType: DataType, rasterDataType: DataType): MosaicRasterTile = {
val index = row.get(0, idDataType)
- val rawRaster = row.get(1, rasterType)
+ // handle checkpoint related de-serialization
+ val rawRaster = row.get(1, rasterDataType)
val createInfo = extractMap(row.getMap(2))
- val raster = GDAL.readRaster(rawRaster, createInfo, rasterType)
+ val raster = GDAL.readRaster(rawRaster, createInfo, rasterDataType)
+
// noinspection TypeCheckCanBeMatch
if (Option(index).isDefined) {
if (index.isInstanceOf[Long]) {
@@ -167,7 +182,6 @@ object MosaicRasterTile {
} else {
new MosaicRasterTile(null, raster)
}
-
}
}
diff --git a/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReTileOnRead.scala b/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReTileOnRead.scala
index b68a580d2..208f5ffd5 100644
--- a/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReTileOnRead.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReTileOnRead.scala
@@ -63,7 +63,7 @@ object ReTileOnRead extends ReadStrategy {
// Note that for retiling we always use checkpoint location.
// In this case rasters are stored off spark rows.
// If you need the tiles in memory please load them from path stored in the tile returned by the reader.
- .add(StructField(TILE, RasterTileType(indexSystem.getCellIdDataType, tileDataType), nullable = false))
+ .add(StructField(TILE, RasterTileType(indexSystem.getCellIdDataType, tileDataType, useCheckpoint = true), nullable = false))
}
/**
diff --git a/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReadAsPath.scala b/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReadAsPath.scala
index 0973146ef..62bef4c9b 100644
--- a/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReadAsPath.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReadAsPath.scala
@@ -62,7 +62,7 @@ object ReadAsPath extends ReadStrategy {
// Note that for retiling we always use checkpoint location.
// In this case rasters are stored off spark rows.
// If you need the tiles in memory please load them from path stored in the tile returned by the reader.
- .add(StructField(TILE, RasterTileType(indexSystem.getCellIdDataType, tileDataType), nullable = false))
+ .add(StructField(TILE, RasterTileType(indexSystem.getCellIdDataType, tileDataType, useCheckpoint = true), nullable = false))
}
/**
diff --git a/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReadInMemory.scala b/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReadInMemory.scala
index 7e6687079..def6cae19 100644
--- a/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReadInMemory.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/datasource/gdal/ReadInMemory.scala
@@ -51,7 +51,7 @@ object ReadInMemory extends ReadStrategy {
.add(StructField(SRID, IntegerType, nullable = false))
// Note, for in memory reads the rasters are stored in the tile.
// For that we use Binary Columns.
- .add(StructField(TILE, RasterTileType(indexSystem.getCellIdDataType, BinaryType), nullable = false))
+ .add(StructField(TILE, RasterTileType(indexSystem.getCellIdDataType, BinaryType, useCheckpoint = false), nullable = false))
}
/**
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Clip.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Clip.scala
index 84e4577be..f9ea405f6 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Clip.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Clip.scala
@@ -1,6 +1,7 @@
package com.databricks.labs.mosaic.expressions.raster
import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI
+import com.databricks.labs.mosaic.core.raster.api.GDAL
import com.databricks.labs.mosaic.core.raster.operator.clip.RasterClipByVector
import com.databricks.labs.mosaic.core.types.RasterTileType
import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
@@ -25,7 +26,10 @@ case class RST_Clip(
with NullIntolerant
with CodegenFallback {
- override def dataType: org.apache.spark.sql.types.DataType = RasterTileType(expressionConfig.getCellIdType, rastersExpr)
+ override def dataType: org.apache.spark.sql.types.DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, rastersExpr, expressionConfig.isRasterUseCheckpoint)
+ }
val geometryAPI: GeometryAPI = GeometryAPI(expressionConfig.getGeometryAPI)
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvg.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvg.scala
index de163ca37..e2e024fee 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvg.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvg.scala
@@ -1,5 +1,6 @@
package com.databricks.labs.mosaic.expressions.raster
+import com.databricks.labs.mosaic.core.raster.api.GDAL
import com.databricks.labs.mosaic.core.raster.operator.CombineAVG
import com.databricks.labs.mosaic.core.types.RasterTileType
import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
@@ -23,7 +24,10 @@ case class RST_CombineAvg(
with NullIntolerant
with CodegenFallback {
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
/** Combines the rasters using average of pixels. */
override def rasterTransform(tiles: Seq[MosaicRasterTile]): Any = {
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgAgg.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgAgg.scala
index 3bf9248c2..bb4872a0e 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgAgg.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgAgg.scala
@@ -31,12 +31,13 @@ case class RST_CombineAvgAgg(
) extends TypedImperativeAggregate[ArrayBuffer[Any]]
with UnaryLike[Expression]
with RasterExpressionSerialization {
-
+ GDAL.enable(expressionConfig)
override lazy val deterministic: Boolean = true
override val child: Expression = tileExpr
override val nullable: Boolean = false
- override lazy val dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
- lazy val tileType: DataType = dataType.asInstanceOf[RasterTileType].rasterType
+ override lazy val dataType: DataType = RasterTileType(
+ expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ lazy val rasterType: DataType = dataType.asInstanceOf[RasterTileType].rasterType
override def prettyName: String = "rst_combine_avg_agg"
val cellIDType: DataType = expressionConfig.getCellIdType
@@ -73,7 +74,7 @@ case class RST_CombineAvgAgg(
} else {
// Do do move the expression
- var tiles = buffer.map(row => deserializeTile(row.asInstanceOf[InternalRow], cellIDType, tileType))
+ var tiles = buffer.map(row => deserializeTile(row.asInstanceOf[InternalRow], cellIDType, rasterType))
buffer.clear()
// If merging multiple index rasters, the index value is dropped
@@ -82,7 +83,7 @@ case class RST_CombineAvgAgg(
val result = MosaicRasterTile(idx, combined)
.formatCellId(IndexSystemFactory.getIndexSystem(expressionConfig.getIndexSystem))
- .serialize(tileType)
+ .serialize(rasterType)
tiles.foreach(RasterCleaner.dispose)
RasterCleaner.dispose(result)
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Convolve.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Convolve.scala
index d831ad849..90325feff 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Convolve.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Convolve.scala
@@ -1,6 +1,7 @@
package com.databricks.labs.mosaic.expressions.raster
import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI
+import com.databricks.labs.mosaic.core.raster.api.GDAL
import com.databricks.labs.mosaic.core.types.RasterTileType
import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo}
@@ -26,7 +27,10 @@ case class RST_Convolve(
with NullIntolerant
with CodegenFallback {
- override def dataType: org.apache.spark.sql.types.DataType = RasterTileType(expressionConfig.getCellIdType, rastersExpr)
+ override def dataType: org.apache.spark.sql.types.DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, rastersExpr, expressionConfig.isRasterUseCheckpoint)
+ }
val geometryAPI: GeometryAPI = GeometryAPI(expressionConfig.getGeometryAPI)
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBand.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBand.scala
index 459f1774a..11d05e5cf 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBand.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBand.scala
@@ -1,5 +1,6 @@
package com.databricks.labs.mosaic.expressions.raster
+import com.databricks.labs.mosaic.core.raster.api.GDAL
import com.databricks.labs.mosaic.core.raster.operator.pixel.PixelCombineRasters
import com.databricks.labs.mosaic.core.types.RasterTileType
import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
@@ -28,7 +29,10 @@ case class RST_DerivedBand(
with NullIntolerant
with CodegenFallback {
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
/** Combines the rasters using average of pixels. */
override def rasterTransform(tiles: Seq[MosaicRasterTile], arg1: Any, arg2: Any): Any = {
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandAgg.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandAgg.scala
index 836b79cbd..d8db6879d 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandAgg.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandAgg.scala
@@ -36,7 +36,10 @@ case class RST_DerivedBandAgg(
override lazy val deterministic: Boolean = true
override val nullable: Boolean = false
- override lazy val dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override lazy val dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
override def prettyName: String = "rst_combine_avg_agg"
private lazy val projection = UnsafeProjection.create(Array[DataType](ArrayType(elementType = dataType, containsNull = false)))
@@ -74,7 +77,7 @@ case class RST_DerivedBandAgg(
// This works for Literals only
val pythonFunc = pythonFuncExpr.eval(null).asInstanceOf[UTF8String].toString
val funcName = funcNameExpr.eval(null).asInstanceOf[UTF8String].toString
- val rasterType = RasterTileType(tileExpr).rasterType
+ val rasterType = RasterTileType(tileExpr, expressionConfig.isRasterUseCheckpoint).rasterType
// Do do move the expression
var tiles = buffer.map(row =>
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Filter.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Filter.scala
index ee8b34d3b..6472c77f5 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Filter.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Filter.scala
@@ -1,6 +1,7 @@
package com.databricks.labs.mosaic.expressions.raster
import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI
+import com.databricks.labs.mosaic.core.raster.api.GDAL
import com.databricks.labs.mosaic.core.types.RasterTileType
import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo}
@@ -27,7 +28,10 @@ case class RST_Filter(
with NullIntolerant
with CodegenFallback {
- override def dataType: org.apache.spark.sql.types.DataType = RasterTileType(expressionConfig.getCellIdType, rastersExpr)
+ override def dataType: org.apache.spark.sql.types.DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, rastersExpr, expressionConfig.isRasterUseCheckpoint)
+ }
val geometryAPI: GeometryAPI = GeometryAPI(expressionConfig.getGeometryAPI)
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromBands.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromBands.scala
index 8325dc0f4..839004f87 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromBands.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromBands.scala
@@ -1,5 +1,6 @@
package com.databricks.labs.mosaic.expressions.raster
+import com.databricks.labs.mosaic.core.raster.api.GDAL
import com.databricks.labs.mosaic.core.raster.operator.merge.MergeBands
import com.databricks.labs.mosaic.core.types.RasterTileType
import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
@@ -9,7 +10,7 @@ import com.databricks.labs.mosaic.functions.MosaicExpressionConfig
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant}
-import org.apache.spark.sql.types.ArrayType
+
/** The expression for stacking and resampling input bands. */
case class RST_FromBands(
@@ -23,11 +24,14 @@ case class RST_FromBands(
with NullIntolerant
with CodegenFallback {
- override def dataType: org.apache.spark.sql.types.DataType =
+ override def dataType: org.apache.spark.sql.types.DataType = {
+ GDAL.enable(expressionConfig)
RasterTileType(
expressionConfig.getCellIdType,
- RasterTileType(bandsExpr).rasterType
+ RasterTileType(bandsExpr, expressionConfig.isRasterUseCheckpoint).rasterType,
+ expressionConfig.isRasterUseCheckpoint
)
+ }
/**
* Stacks and resamples input bands.
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromContent.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromContent.scala
index 1021eb083..0c53261ab 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromContent.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromContent.scala
@@ -33,10 +33,11 @@ case class RST_FromContent(
with Serializable
with NullIntolerant
with CodegenFallback {
-
- val tileType: DataType = BinaryType
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileType)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, BinaryType, expressionConfig.isRasterUseCheckpoint)
+ }
protected val geometryAPI: GeometryAPI = GeometryAPI.apply(expressionConfig.getGeometryAPI)
@@ -62,6 +63,7 @@ case class RST_FromContent(
*/
override def eval(input: InternalRow): TraversableOnce[InternalRow] = {
GDAL.enable(expressionConfig)
+ val rasterType = dataType.asInstanceOf[RasterTileType].rasterType
val driver = driverExpr.eval(input).asInstanceOf[UTF8String].toString
val ext = GDAL.getExtension(driver)
var rasterArr = contentExpr.eval(input).asInstanceOf[Array[Byte]]
@@ -71,7 +73,7 @@ case class RST_FromContent(
val createInfo = Map("parentPath" -> PathUtils.NO_PATH_STRING, "driver" -> driver)
var raster = MosaicRasterGDAL.readRaster(rasterArr, createInfo)
var tile = MosaicRasterTile(null, raster)
- val row = tile.formatCellId(indexSystem).serialize(tileType)
+ val row = tile.formatCellId(indexSystem).serialize(rasterType)
RasterCleaner.dispose(raster)
RasterCleaner.dispose(tile)
rasterArr = null
@@ -82,15 +84,15 @@ case class RST_FromContent(
// target size is > 0 and raster size > target size
// - write the initial raster to file (unsplit)
// - createDirectories in case of context isolation
- val rasterPath = PathUtils.createTmpFilePath(ext)
- Files.createDirectories(Paths.get(rasterPath).getParent)
- Files.write(Paths.get(rasterPath), rasterArr)
+ val tmpPath = PathUtils.createTmpFilePath(ext)
+ Files.createDirectories(Paths.get(tmpPath).getParent)
+ Files.write(Paths.get(tmpPath), rasterArr)
// split to tiles up to specifed threshold
- var tiles = ReTileOnRead.localSubdivide(rasterPath, PathUtils.NO_PATH_STRING, targetSize)
- val rows = tiles.map(_.formatCellId(indexSystem).serialize(tileType))
+ var tiles = ReTileOnRead.localSubdivide(tmpPath, PathUtils.NO_PATH_STRING, targetSize)
+ val rows = tiles.map(_.formatCellId(indexSystem).serialize(rasterType))
tiles.foreach(RasterCleaner.dispose(_))
- Files.deleteIfExists(Paths.get(rasterPath))
+ Files.deleteIfExists(Paths.get(tmpPath))
rasterArr = null
tiles = null
rows.map(row => InternalRow.fromSeq(Seq(row)))
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromFile.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromFile.scala
index 8e1dc213e..2627858d0 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromFile.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromFile.scala
@@ -32,10 +32,11 @@ case class RST_FromFile(
with Serializable
with NullIntolerant
with CodegenFallback {
-
- val tileType: DataType = BinaryType
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileType)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, BinaryType, expressionConfig.isRasterUseCheckpoint)
+ }
protected val geometryAPI: GeometryAPI = GeometryAPI.apply(expressionConfig.getGeometryAPI)
@@ -61,6 +62,7 @@ case class RST_FromFile(
*/
override def eval(input: InternalRow): TraversableOnce[InternalRow] = {
GDAL.enable(expressionConfig)
+ val rasterType = dataType.asInstanceOf[RasterTileType].rasterType
val path = rasterPathExpr.eval(input).asInstanceOf[UTF8String].toString
val readPath = PathUtils.getCleanPath(path)
val driver = MosaicRasterGDAL.identifyDriver(path)
@@ -70,7 +72,7 @@ case class RST_FromFile(
val createInfo = Map("path" -> readPath, "parentPath" -> path)
var raster = MosaicRasterGDAL.readRaster(createInfo)
var tile = MosaicRasterTile(null, raster)
- val row = tile.formatCellId(indexSystem).serialize(tileType)
+ val row = tile.formatCellId(indexSystem).serialize(rasterType)
RasterCleaner.dispose(raster)
RasterCleaner.dispose(tile)
raster = null
@@ -83,7 +85,7 @@ case class RST_FromFile(
Files.copy(Paths.get(readPath), Paths.get(tmpPath), StandardCopyOption.REPLACE_EXISTING)
val size = if (targetSize <= 0) 64 else targetSize
var tiles = ReTileOnRead.localSubdivide(tmpPath, path, size)
- val rows = tiles.map(_.formatCellId(indexSystem).serialize(tileType))
+ val rows = tiles.map(_.formatCellId(indexSystem).serialize(rasterType))
tiles.foreach(RasterCleaner.dispose(_))
Files.deleteIfExists(Paths.get(tmpPath))
tiles = null
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetSubdataset.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetSubdataset.scala
index 1589fa463..88df43f25 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetSubdataset.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetSubdataset.scala
@@ -1,5 +1,6 @@
package com.databricks.labs.mosaic.expressions.raster
+import com.databricks.labs.mosaic.core.raster.api.GDAL
import com.databricks.labs.mosaic.core.types.RasterTileType
import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo}
@@ -25,7 +26,10 @@ case class RST_GetSubdataset(
with NullIntolerant
with CodegenFallback {
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
/** Returns the subdatasets of the raster. */
override def rasterTransform(tile: MosaicRasterTile, arg1: Any): Any = {
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala
index 6f92b79ae..0902ecd4f 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoData.scala
@@ -25,7 +25,10 @@ case class RST_InitNoData(
with NullIntolerant
with CodegenFallback {
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
/**
* Initializes no data values of a raster.
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MakeTiles.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MakeTiles.scala
index f9cf7099d..eb96ded80 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MakeTiles.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MakeTiles.scala
@@ -24,7 +24,12 @@ import scala.util.Try
/**
* Creates raster tiles from the input column.
- *
+ * - spark config to turn checkpointing on for all functions in 0.4.2
+ * - this is the only function able to write raster to
+ * checkpoint (even if the spark config is set to false).
+ * - can be useful when you want to start from the configured checkpoint
+ * but work with binary payloads from there.
+ * - more at [[com.databricks.labs.mosaic.gdal.MosaicGDAL]].
* @param inputExpr
* The expression for the raster. If the raster is stored on disc, the path
* to the raster is provided. If the raster is stored in memory, the bytes of
@@ -56,14 +61,17 @@ case class RST_MakeTiles(
with NullIntolerant
with CodegenFallback {
+ /** @return Returns StringType if either */
override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
require(withCheckpointExpr.isInstanceOf[Literal])
- if (withCheckpointExpr.eval().asInstanceOf[Boolean]) {
+
+ if (withCheckpointExpr.eval().asInstanceOf[Boolean] || expressionConfig.isRasterUseCheckpoint) {
// Raster is referenced via a path
- RasterTileType(expressionConfig.getCellIdType, StringType)
+ RasterTileType(expressionConfig.getCellIdType, StringType, useCheckpoint = true)
} else {
// Raster is referenced via a byte array
- RasterTileType(expressionConfig.getCellIdType, BinaryType)
+ RasterTileType(expressionConfig.getCellIdType, BinaryType, useCheckpoint = false)
}
}
@@ -116,7 +124,7 @@ case class RST_MakeTiles(
override def eval(input: InternalRow): TraversableOnce[InternalRow] = {
GDAL.enable(expressionConfig)
- val tileType = dataType.asInstanceOf[StructType].find(_.name == "raster").get.dataType
+ val rasterType = dataType.asInstanceOf[RasterTileType].rasterType
val rawDriver = driverExpr.eval(input).asInstanceOf[UTF8String].toString
val rawInput = inputExpr.eval(input)
@@ -130,7 +138,7 @@ case class RST_MakeTiles(
val createInfo = Map("parentPath" -> PathUtils.NO_PATH_STRING, "driver" -> driver, "path" -> path)
val raster = GDAL.readRaster(rawInput, createInfo, inputExpr.dataType)
val tile = MosaicRasterTile(null, raster)
- val row = tile.formatCellId(indexSystem).serialize(tileType)
+ val row = tile.formatCellId(indexSystem).serialize(rasterType)
RasterCleaner.dispose(raster)
RasterCleaner.dispose(tile)
Seq(InternalRow.fromSeq(Seq(row)))
@@ -138,20 +146,20 @@ case class RST_MakeTiles(
// target size is > 0 and raster size > target size
// - write the initial raster to file (unsplit)
// - createDirectories in case of context isolation
- val rasterPath =
+ val readPath =
if (inputExpr.dataType == StringType) {
PathUtils.copyToTmpWithRetry(path, 5)
} else {
- val rasterPath = PathUtils.createTmpFilePath(GDAL.getExtension(driver))
- Files.createDirectories(Paths.get(rasterPath).getParent)
- Files.write(Paths.get(rasterPath), rawInput.asInstanceOf[Array[Byte]])
- rasterPath
+ val tmpPath = PathUtils.createTmpFilePath(GDAL.getExtension(driver))
+ Files.createDirectories(Paths.get(tmpPath).getParent)
+ Files.write(Paths.get(tmpPath), rawInput.asInstanceOf[Array[Byte]])
+ tmpPath
}
val size = if (targetSize <= 0) 64 else targetSize
- var tiles = ReTileOnRead.localSubdivide(rasterPath, PathUtils.NO_PATH_STRING, size)
- val rows = tiles.map(_.formatCellId(indexSystem).serialize(tileType))
+ var tiles = ReTileOnRead.localSubdivide(readPath, PathUtils.NO_PATH_STRING, size)
+ val rows = tiles.map(_.formatCellId(indexSystem).serialize(rasterType))
tiles.foreach(RasterCleaner.dispose(_))
- Files.deleteIfExists(Paths.get(rasterPath))
+ Files.deleteIfExists(Paths.get(readPath))
tiles = null
rows.map(row => InternalRow.fromSeq(Seq(row)))
}
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MapAlgebra.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MapAlgebra.scala
index bc2aea949..ff3d85351 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MapAlgebra.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MapAlgebra.scala
@@ -28,7 +28,10 @@ case class RST_MapAlgebra(
with NullIntolerant
with CodegenFallback {
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
/**
* Map Algebra.
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Merge.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Merge.scala
index 09d4b9d3d..e57f1c65b 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Merge.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Merge.scala
@@ -1,5 +1,6 @@
package com.databricks.labs.mosaic.expressions.raster
+import com.databricks.labs.mosaic.core.raster.api.GDAL
import com.databricks.labs.mosaic.core.raster.operator.merge.MergeRasters
import com.databricks.labs.mosaic.core.types.RasterTileType
import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
@@ -23,7 +24,10 @@ case class RST_Merge(
with NullIntolerant
with CodegenFallback {
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
/**
* Merges an array of rasters.
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAgg.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAgg.scala
index ae56a01ab..dd3d91ef4 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAgg.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAgg.scala
@@ -31,7 +31,10 @@ case class RST_MergeAgg(
override lazy val deterministic: Boolean = true
override val child: Expression = tileExpr
override val nullable: Boolean = false
- override lazy val dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override lazy val dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
override def prettyName: String = "rst_merge_agg"
private lazy val projection = UnsafeProjection.create(Array[DataType](ArrayType(elementType = dataType, containsNull = false)))
@@ -66,7 +69,7 @@ case class RST_MergeAgg(
// This is a trick to get the rasters sorted by their parent path to ensure more consistent results
// when merging rasters with large overlaps
- val rasterType = RasterTileType(tileExpr).rasterType
+ val rasterType = RasterTileType(tileExpr, expressionConfig.isRasterUseCheckpoint).rasterType
var tiles = buffer
.map(row =>
MosaicRasterTile.deserialize(
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NDVI.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NDVI.scala
index 2d7b55fd7..669b661a3 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NDVI.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_NDVI.scala
@@ -1,5 +1,6 @@
package com.databricks.labs.mosaic.expressions.raster
+import com.databricks.labs.mosaic.core.raster.api.GDAL
import com.databricks.labs.mosaic.core.raster.operator.NDVI
import com.databricks.labs.mosaic.core.types.RasterTileType
import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
@@ -27,7 +28,10 @@ case class RST_NDVI(
with NullIntolerant
with CodegenFallback {
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
/**
* Computes NDVI index.
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTile.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTile.scala
index cc1b0e0ee..27eecdedd 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTile.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTile.scala
@@ -23,6 +23,7 @@ case class RST_ReTile(
with NullIntolerant
with CodegenFallback {
+ /** @return provided raster data type (assumes that was handled for checkpointing.)*/
override def dataType: DataType = rasterExpr.dataType
/**
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala
index c42af9c2e..ce56d62b9 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoData.scala
@@ -28,7 +28,10 @@ case class RST_SetNoData(
with NullIntolerant
with CodegenFallback {
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
/**
* Returns a raster with the specified no data values.
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetSRID.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetSRID.scala
index 2aabb3df9..51d234fd4 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetSRID.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetSRID.scala
@@ -1,6 +1,7 @@
package com.databricks.labs.mosaic.expressions.raster
import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI
+import com.databricks.labs.mosaic.core.raster.api.GDAL
import com.databricks.labs.mosaic.core.types.RasterTileType
import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
import com.databricks.labs.mosaic.expressions.base.{GenericExpressionFactory, WithExpressionInfo}
@@ -25,7 +26,10 @@ case class RST_SetSRID(
with NullIntolerant
with CodegenFallback {
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, rastersExpr)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, rastersExpr, expressionConfig.isRasterUseCheckpoint)
+ }
val geometryAPI: GeometryAPI = GeometryAPI(expressionConfig.getGeometryAPI)
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Transform.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Transform.scala
index 7681f2bba..2781c8364 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Transform.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/RST_Transform.scala
@@ -1,5 +1,6 @@
package com.databricks.labs.mosaic.expressions.raster
+import com.databricks.labs.mosaic.core.raster.api.GDAL
import com.databricks.labs.mosaic.core.raster.operator.proj.RasterProject
import com.databricks.labs.mosaic.core.types.RasterTileType
import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
@@ -26,7 +27,10 @@ case class RST_Transform(
with NullIntolerant
with CodegenFallback {
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
/** Returns the upper left x of the raster. */
override def rasterTransform(tile: MosaicRasterTile, arg1: Any): Any = {
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/Raster1ArgExpression.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/Raster1ArgExpression.scala
index 35ad927c6..01285b652 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/Raster1ArgExpression.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/Raster1ArgExpression.scala
@@ -69,7 +69,7 @@ abstract class Raster1ArgExpression[T <: Expression: ClassTag](
// noinspection DuplicatedCode
override def nullSafeEval(input: Any, arg1: Any): Any = {
GDAL.enable(expressionConfig)
- val rasterType = RasterTileType(rasterExpr).rasterType
+ val rasterType = RasterTileType(rasterExpr, expressionConfig.isRasterUseCheckpoint).rasterType
val tile = MosaicRasterTile.deserialize(
input.asInstanceOf[InternalRow],
expressionConfig.getCellIdType,
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/Raster2ArgExpression.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/Raster2ArgExpression.scala
index c5be60724..758802f62 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/Raster2ArgExpression.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/Raster2ArgExpression.scala
@@ -8,7 +8,6 @@ import com.databricks.labs.mosaic.expressions.base.GenericExpressionFactory
import com.databricks.labs.mosaic.functions.MosaicExpressionConfig
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant, TernaryExpression}
-import org.apache.spark.sql.types.DataType
import scala.reflect.ClassTag
@@ -78,7 +77,7 @@ abstract class Raster2ArgExpression[T <: Expression: ClassTag](
// noinspection DuplicatedCode
override def nullSafeEval(input: Any, arg1: Any, arg2: Any): Any = {
GDAL.enable(expressionConfig)
- val rasterType = RasterTileType(rasterExpr).rasterType
+ val rasterType = RasterTileType(rasterExpr, expressionConfig.isRasterUseCheckpoint).rasterType
val tile = MosaicRasterTile.deserialize(
input.asInstanceOf[InternalRow],
expressionConfig.getCellIdType,
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArray1ArgExpression.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArray1ArgExpression.scala
index 5dbfd08cc..85cd4b810 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArray1ArgExpression.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArray1ArgExpression.scala
@@ -7,7 +7,6 @@ import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
import com.databricks.labs.mosaic.expressions.base.GenericExpressionFactory
import com.databricks.labs.mosaic.functions.MosaicExpressionConfig
import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, NullIntolerant}
-import org.apache.spark.sql.types.ArrayType
import scala.reflect.ClassTag
@@ -67,7 +66,7 @@ abstract class RasterArray1ArgExpression[T <: Expression: ClassTag](
GDAL.enable(expressionConfig)
val tiles = RasterArrayUtils.getTiles(input, rastersExpr, expressionConfig)
val result = rasterTransform(tiles, arg1)
- val resultType = if (returnsRaster) RasterTileType(rastersExpr).rasterType else dataType
+ val resultType = if (returnsRaster) RasterTileType(rastersExpr, expressionConfig.isRasterUseCheckpoint).rasterType else dataType
val serialized = serialize(result, returnsRaster, resultType, expressionConfig)
tiles.foreach(t => RasterCleaner.dispose(t))
serialized
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArray2ArgExpression.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArray2ArgExpression.scala
index 9de963684..d4e362895 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArray2ArgExpression.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArray2ArgExpression.scala
@@ -7,7 +7,6 @@ import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
import com.databricks.labs.mosaic.expressions.base.GenericExpressionFactory
import com.databricks.labs.mosaic.functions.MosaicExpressionConfig
import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant, TernaryExpression}
-import org.apache.spark.sql.types.ArrayType
import scala.reflect.ClassTag
@@ -72,7 +71,7 @@ abstract class RasterArray2ArgExpression[T <: Expression: ClassTag](
GDAL.enable(expressionConfig)
val tiles = RasterArrayUtils.getTiles(input, rastersExpr, expressionConfig)
val result = rasterTransform(tiles, arg1, arg2)
- val resultType = if (returnsRaster) RasterTileType(rastersExpr).rasterType else dataType
+ val resultType = if (returnsRaster) RasterTileType(rastersExpr, expressionConfig.isRasterUseCheckpoint).rasterType else dataType
val serialized = serialize(result, returnsRaster, resultType, expressionConfig)
tiles.foreach(t => RasterCleaner.dispose(t))
serialized
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArrayExpression.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArrayExpression.scala
index 8c3a52d9a..55f2d3646 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArrayExpression.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArrayExpression.scala
@@ -7,7 +7,6 @@ import com.databricks.labs.mosaic.core.types.model.MosaicRasterTile
import com.databricks.labs.mosaic.expressions.base.GenericExpressionFactory
import com.databricks.labs.mosaic.functions.MosaicExpressionConfig
import org.apache.spark.sql.catalyst.expressions.{Expression, NullIntolerant, UnaryExpression}
-import org.apache.spark.sql.types.ArrayType
import scala.reflect.ClassTag
@@ -19,8 +18,8 @@ import scala.reflect.ClassTag
* @param rastersExpr
* The rasters expression. It is an array column containing rasters as either
* paths or as content byte arrays.
- * @param outputType
- * The output type of the result.
+ * @param returnsRaster
+ * Whether raster is returned.
* @param expressionConfig
* Additional arguments for the expression (expressionConfigs).
* @tparam T
@@ -64,7 +63,7 @@ abstract class RasterArrayExpression[T <: Expression: ClassTag](
GDAL.enable(expressionConfig)
val tiles = RasterArrayUtils.getTiles(input, rastersExpr, expressionConfig)
val result = rasterTransform(tiles)
- val resultType = if (returnsRaster) RasterTileType(rastersExpr).rasterType else dataType
+ val resultType = if (returnsRaster) RasterTileType(rastersExpr, expressionConfig.isRasterUseCheckpoint).rasterType else dataType
val serialized = serialize(result, returnsRaster, resultType, expressionConfig)
tiles.foreach(t => RasterCleaner.dispose(t))
serialized
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArrayUtils.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArrayUtils.scala
index f2d399350..d97cefde1 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArrayUtils.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterArrayUtils.scala
@@ -13,7 +13,7 @@ object RasterArrayUtils {
def getTiles(input: Any, rastersExpr: Expression, expressionConfig: MosaicExpressionConfig): Seq[MosaicRasterTile] = {
val rasterDT = rastersExpr.dataType.asInstanceOf[ArrayType].elementType
val arrayData = input.asInstanceOf[ArrayData]
- val rasterType = RasterTileType(rastersExpr).rasterType
+ val rasterType = RasterTileType(rastersExpr, expressionConfig.isRasterUseCheckpoint).rasterType
val n = arrayData.numElements()
(0 until n)
.map(i =>
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterBandExpression.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterBandExpression.scala
index 97bd3e333..e36bc195f 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterBandExpression.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterBandExpression.scala
@@ -73,7 +73,7 @@ abstract class RasterBandExpression[T <: Expression: ClassTag](
// noinspection DuplicatedCode
override def nullSafeEval(inputRaster: Any, inputBand: Any): Any = {
GDAL.enable(expressionConfig)
- val rasterType = RasterTileType(rasterExpr).rasterType
+ val rasterType = RasterTileType(rasterExpr, expressionConfig.isRasterUseCheckpoint).rasterType
val tile = MosaicRasterTile.deserialize(
inputRaster.asInstanceOf[InternalRow],
expressionConfig.getCellIdType,
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterExpression.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterExpression.scala
index 66435f101..47f274583 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterExpression.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterExpression.scala
@@ -64,7 +64,7 @@ abstract class RasterExpression[T <: Expression: ClassTag](
*/
override def nullSafeEval(input: Any): Any = {
GDAL.enable(expressionConfig)
- val rasterType = RasterTileType(rasterExpr).rasterType
+ val rasterType = RasterTileType(rasterExpr, expressionConfig.isRasterUseCheckpoint).rasterType
val tile = MosaicRasterTile.deserialize(
input.asInstanceOf[InternalRow],
cellIdDataType,
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterGeneratorExpression.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterGeneratorExpression.scala
index 3fc80752d..e124f1917 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterGeneratorExpression.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterGeneratorExpression.scala
@@ -38,7 +38,10 @@ abstract class RasterGeneratorExpression[T <: Expression: ClassTag](
with NullIntolerant
with Serializable {
- override def dataType: DataType = RasterTileType(expressionConfig.getCellIdType, tileExpr)
+ override def dataType: DataType = {
+ GDAL.enable(expressionConfig)
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint)
+ }
val uuid: String = java.util.UUID.randomUUID().toString.replace("-", "_")
@@ -72,7 +75,7 @@ abstract class RasterGeneratorExpression[T <: Expression: ClassTag](
override def eval(input: InternalRow): TraversableOnce[InternalRow] = {
GDAL.enable(expressionConfig)
- val rasterType = RasterTileType(tileExpr).rasterType
+ val rasterType = RasterTileType(tileExpr, expressionConfig.isRasterUseCheckpoint).rasterType
val tile = MosaicRasterTile.deserialize(tileExpr.eval(input).asInstanceOf[InternalRow], cellIdDataType, rasterType)
val generatedRasters = rasterGenerator(tile)
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterTessellateGeneratorExpression.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterTessellateGeneratorExpression.scala
index 98ff86ca7..80871b66d 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterTessellateGeneratorExpression.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterTessellateGeneratorExpression.scala
@@ -27,6 +27,8 @@ import scala.reflect.ClassTag
* The expression for the raster. If the raster is stored on disc, the path
* to the raster is provided. If the raster is stored in memory, the bytes of
* the raster are provided.
+ * @param resolutionExpr
+ * The resolution of the index system to use for tessellation.
* @param expressionConfig
* Additional arguments for the expression (expressionConfigs).
* @tparam T
@@ -55,8 +57,14 @@ abstract class RasterTessellateGeneratorExpression[T <: Expression: ClassTag](
* needs to be wrapped in a StructType. The actually type is that of the
* structs element.
*/
- override def elementSchema: StructType =
- StructType(Array(StructField("element", RasterTileType(indexSystem.getCellIdDataType, tileExpr))))
+ override def elementSchema: StructType = {
+ StructType(
+ Array(StructField(
+ "element",
+ RasterTileType(expressionConfig.getCellIdType, tileExpr, expressionConfig.isRasterUseCheckpoint))
+ )
+ )
+ }
/**
* The function to be overridden by the extending class. It is called when
@@ -71,8 +79,7 @@ abstract class RasterTessellateGeneratorExpression[T <: Expression: ClassTag](
override def eval(input: InternalRow): TraversableOnce[InternalRow] = {
GDAL.enable(expressionConfig)
-
- val rasterType = RasterTileType(tileExpr).rasterType
+ val rasterType = RasterTileType(tileExpr, expressionConfig.isRasterUseCheckpoint).rasterType
val tile = MosaicRasterTile
.deserialize(tileExpr.eval(input).asInstanceOf[InternalRow], indexSystem.getCellIdDataType, rasterType)
val inResolution: Int = indexSystem.getResolution(resolutionExpr.eval(input))
diff --git a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterToGridExpression.scala b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterToGridExpression.scala
index e7b04f989..a69f73151 100644
--- a/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterToGridExpression.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/expressions/raster/base/RasterToGridExpression.scala
@@ -25,6 +25,8 @@ import scala.reflect.ClassTag
* @param rasterExpr
* The raster expression. It can be a path to a raster file or a byte array
* containing the raster file content.
+ * @param resolutionExpr
+ * The resolution of the index system to use.
* @param measureType
* The output type of the result.
* @param expressionConfig
@@ -34,10 +36,10 @@ import scala.reflect.ClassTag
*/
abstract class RasterToGridExpression[T <: Expression: ClassTag, P](
rasterExpr: Expression,
- resolution: Expression,
+ resolutionExpr: Expression,
measureType: DataType,
expressionConfig: MosaicExpressionConfig
-) extends Raster1ArgExpression[T](rasterExpr, resolution, returnsRaster = false, expressionConfig)
+) extends Raster1ArgExpression[T](rasterExpr, resolutionExpr, returnsRaster = false, expressionConfig)
with RasterGridExpression
with NullIntolerant
with Serializable {
diff --git a/src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala b/src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala
index 8a1075a76..9736d9081 100644
--- a/src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala
@@ -75,19 +75,32 @@ class MosaicContext(indexSystem: IndexSystem, geometryAPI: GeometryAPI) extends
)
}
+ /**
+ * Registers required parsers for SQL for Mosaic functionality.
+ * - uses spark from existing session.
+ * - called on driver.
+ */
def register(): Unit = {
- val spark = SparkSession.builder().getOrCreate()
register(spark)
}
+ /**
+ * Registers required parsers for SQL for Mosaic functionality.
+ * - uses spark from existing session.
+ * - called on driver.
+ *
+ * @param database
+ * A database to which functions are added to. By default none is passed
+ * resulting in functions being registered in default database.
+ */
def register(database: String): Unit = {
- val spark = SparkSession.builder().getOrCreate()
spark.sql(s"create database if not exists $database")
register(spark, Some(database))
}
/**
* Registers required parsers for SQL for Mosaic functionality.
+ * - called on driver.
*
* @param spark
* SparkSession to which the parsers are registered to.
@@ -101,6 +114,7 @@ class MosaicContext(indexSystem: IndexSystem, geometryAPI: GeometryAPI) extends
spark: SparkSession,
database: Option[String] = None
): Unit = {
+ expressionConfig.updateSparkConf(spark) // any changes?
val registry = spark.sessionState.functionRegistry
val mosaicRegistry = MosaicRegistry(registry, database)
@@ -550,7 +564,6 @@ class MosaicContext(indexSystem: IndexSystem, geometryAPI: GeometryAPI) extends
}
def shouldUseDatabricksH3(): Boolean = {
- val spark = SparkSession.builder().getOrCreate()
val isDatabricksH3Enabled = spark.conf.get(SPARK_DATABRICKS_GEO_H3_ENABLED, "false") == "true"
indexSystem.name == H3.name && isDatabricksH3Enabled
}
@@ -1043,7 +1056,7 @@ class MosaicContext(indexSystem: IndexSystem, geometryAPI: GeometryAPI) extends
object MosaicContext extends Logging {
var _tmpDir: String = ""
- val mosaicVersion: String = "0.4.2"
+ val mosaicVersion: String = "0.4.3"
private var instance: Option[MosaicContext] = None
@@ -1075,6 +1088,12 @@ object MosaicContext extends Logging {
case None => throw new Error("MosaicContext was not built.")
}
+ def checkContext: Boolean =
+ instance match {
+ case Some(_) => true
+ case None => false
+ }
+
def reset(): Unit = instance = None
// noinspection ScalaStyle,ScalaWeakerAccess
diff --git a/src/main/scala/com/databricks/labs/mosaic/functions/MosaicExpressionConfig.scala b/src/main/scala/com/databricks/labs/mosaic/functions/MosaicExpressionConfig.scala
index b16b719cc..9a76c29d2 100644
--- a/src/main/scala/com/databricks/labs/mosaic/functions/MosaicExpressionConfig.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/functions/MosaicExpressionConfig.scala
@@ -5,6 +5,8 @@ import com.databricks.labs.mosaic.core.index.IndexSystemFactory
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{RuntimeConfig, SparkSession}
+import scala.util.Try
+
/**
* Mosaic Expression Config is a class that contains the configuration for the
* Mosaic Expression. Singleton objects are not accessible outside the JVM, so
@@ -17,9 +19,23 @@ import org.apache.spark.sql.{RuntimeConfig, SparkSession}
case class MosaicExpressionConfig(configs: Map[String, String]) {
def updateSparkConf(): Unit = {
+ // populate initial set configs
val spark = SparkSession.builder().getOrCreate()
+ updateSparkConf(spark)
+ }
+
+ def updateSparkConf(spark: SparkSession): Unit = {
val sparkConf = spark.sparkContext.getConf
configs.foreach { case (k, v) => sparkConf.set(k, v) }
+
+ // update defaults as well
+ this
+ .setGeometryAPI(spark.conf.get(MOSAIC_GEOMETRY_API, JTS.name))
+ .setIndexSystem(spark.conf.get(MOSAIC_INDEX_SYSTEM, H3.name))
+ .setRasterCheckpoint(spark.conf.get(MOSAIC_RASTER_CHECKPOINT, MOSAIC_RASTER_CHECKPOINT_DEFAULT))
+ .setRasterUseCheckpoint(spark.conf.get(MOSAIC_RASTER_USE_CHECKPOINT, MOSAIC_RASTER_USE_CHECKPOINT_DEFAULT))
+ .setTmpPrefix(spark.conf.get(MOSAIC_RASTER_TMP_PREFIX, "/tmp"))
+ .setGDALConf(spark.conf)
}
def getGDALConf: Map[String, String] = {
@@ -28,11 +44,17 @@ case class MosaicExpressionConfig(configs: Map[String, String]) {
def getGeometryAPI: String = configs.getOrElse(MOSAIC_GEOMETRY_API, JTS.name)
- def getIndexSystem: String = configs.getOrElse(MOSAIC_INDEX_SYSTEM, H3.name)
-
def getRasterCheckpoint: String = configs.getOrElse(MOSAIC_RASTER_CHECKPOINT, MOSAIC_RASTER_CHECKPOINT_DEFAULT)
+ def getRasterUseCheckpoint: String = configs.getOrElse(MOSAIC_RASTER_USE_CHECKPOINT, MOSAIC_RASTER_USE_CHECKPOINT_DEFAULT)
+
+ def isRasterUseCheckpoint: Boolean = {
+ Try(getRasterUseCheckpoint == "true").getOrElse(false)
+ }
+
def getCellIdType: DataType = IndexSystemFactory.getIndexSystem(getIndexSystem).cellIdType
+
+ def getIndexSystem: String = configs.getOrElse(MOSAIC_INDEX_SYSTEM, H3.name)
def getRasterBlockSize: Int = configs.getOrElse(MOSAIC_RASTER_BLOCKSIZE, MOSAIC_RASTER_BLOCKSIZE_DEFAULT).toInt
@@ -58,6 +80,10 @@ case class MosaicExpressionConfig(configs: Map[String, String]) {
def setRasterCheckpoint(checkpoint: String): MosaicExpressionConfig = {
MosaicExpressionConfig(configs + (MOSAIC_RASTER_CHECKPOINT -> checkpoint))
}
+
+ def setRasterUseCheckpoint(checkpoint: String): MosaicExpressionConfig = {
+ MosaicExpressionConfig(configs + (MOSAIC_RASTER_USE_CHECKPOINT -> checkpoint))
+ }
def setTmpPrefix(prefix: String): MosaicExpressionConfig = {
MosaicExpressionConfig(configs + (MOSAIC_RASTER_TMP_PREFIX -> prefix))
@@ -81,9 +107,9 @@ object MosaicExpressionConfig {
.setGeometryAPI(spark.conf.get(MOSAIC_GEOMETRY_API, JTS.name))
.setIndexSystem(spark.conf.get(MOSAIC_INDEX_SYSTEM, H3.name))
.setRasterCheckpoint(spark.conf.get(MOSAIC_RASTER_CHECKPOINT, MOSAIC_RASTER_CHECKPOINT_DEFAULT))
+ .setRasterUseCheckpoint(spark.conf.get(MOSAIC_RASTER_USE_CHECKPOINT, MOSAIC_RASTER_USE_CHECKPOINT_DEFAULT))
.setTmpPrefix(spark.conf.get(MOSAIC_RASTER_TMP_PREFIX, "/tmp"))
.setGDALConf(spark.conf)
-
}
}
diff --git a/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala b/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala
index 6cc928edd..12986d601 100644
--- a/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/gdal/MosaicGDAL.scala
@@ -1,18 +1,22 @@
package com.databricks.labs.mosaic.gdal
-import com.databricks.labs.mosaic.MOSAIC_RASTER_BLOCKSIZE_DEFAULT
+import com.databricks.labs.mosaic.core.geometry.api.GeometryAPI
+import com.databricks.labs.mosaic.core.index.IndexSystemFactory
+import com.databricks.labs.mosaic.{MOSAIC_RASTER_BLOCKSIZE_DEFAULT, MOSAIC_RASTER_CHECKPOINT, MOSAIC_RASTER_CHECKPOINT_DEFAULT, MOSAIC_RASTER_USE_CHECKPOINT, MOSAIC_RASTER_USE_CHECKPOINT_DEFAULT, MOSAIC_TEST_MODE}
import com.databricks.labs.mosaic.functions.{MosaicContext, MosaicExpressionConfig}
+import com.databricks.labs.mosaic.utils.PathUtils
import org.apache.spark.internal.Logging
import org.apache.spark.sql.SparkSession
import org.gdal.gdal.gdal
import org.gdal.osr.SpatialReference
-import java.nio.file.{Files, Paths}
+import java.io.{File, FileNotFoundException}
+import java.nio.file.{Files, InvalidPathException, Paths}
import scala.language.postfixOps
import scala.util.Try
//noinspection DuplicatedCode
-/** GDAL environment preparation and configuration. */
+/** GDAL environment preparation and configuration. Some functions only for driver. */
object MosaicGDAL extends Logging {
private val usrlibsoPath = "/usr/lib/libgdal.so"
@@ -31,6 +35,8 @@ object MosaicGDAL extends Logging {
val GDAL_ENABLED = "spark.mosaic.gdal.native.enabled"
var isEnabled = false
var checkpointPath: String = _
+ var useCheckpoint: Boolean = _
+
// Only use this with GDAL rasters
val WSG84: SpatialReference = {
@@ -59,7 +65,12 @@ object MosaicGDAL extends Logging {
gdal.SetConfigOption("GDAL_NUM_THREADS", "ALL_CPUS")
mosaicConfig.getGDALConf.foreach { case (k, v) => gdal.SetConfigOption(k.split("\\.").last, v) }
setBlockSize(mosaicConfig)
- checkpointPath = mosaicConfig.getRasterCheckpoint
+ configureCheckpoint(mosaicConfig)
+ }
+
+ def configureCheckpoint(mosaicConfig: MosaicExpressionConfig): Unit = {
+ this.checkpointPath = mosaicConfig.getRasterCheckpoint
+ this.useCheckpoint = mosaicConfig.isRasterUseCheckpoint
}
def setBlockSize(mosaicConfig: MosaicExpressionConfig): Unit = {
@@ -75,14 +86,22 @@ object MosaicGDAL extends Logging {
}
}
- /** Enables the GDAL environment. */
+ /**
+ * Enables the GDAL environment, called from driver.
+ * - see mosaic_context.py as well for use.
+ *
+ * @param spark
+ * spark session to use.
+ */
def enableGDAL(spark: SparkSession): Unit = {
+ // refresh configs in case spark had changes
+ val mosaicConfig = MosaicExpressionConfig(spark)
+
if (!wasEnabled(spark) && !isEnabled) {
Try {
isEnabled = true
loadSharedObjects()
- val expressionConfig = MosaicExpressionConfig(spark)
- configureGDAL(expressionConfig)
+ configureGDAL(mosaicConfig)
gdal.AllRegister()
spark.conf.set(GDAL_ENABLED, "true")
} match {
@@ -95,9 +114,125 @@ object MosaicGDAL extends Logging {
isEnabled = false
throw exception
}
+ } else {
+ configureCheckpoint(mosaicConfig)
}
}
+ /**
+ * Enables the GDAL environment with checkpointing, called from driver.
+ * - alternative to setting spark configs prior to init.
+ * - can be called multiple times in a session as you want to change
+ * checkpoint location.
+ * - sets [[checkpointPath]] to provided path.
+ * - sets [[useCheckpoint]] to "true".
+ * - see mosaic_context.py as well for use.
+ * @param spark
+ * spark session to use.
+ * @param withCheckpointPath
+ * path to set.
+ */
+ def enableGDALWithCheckpoint(spark: SparkSession, withCheckpointPath: String): Unit = {
+ // - set spark config to enable checkpointing
+ // - initial checks + update path
+ // - also inits MosaicContext
+ // - also enables GDAL and refreshes accessors
+ spark.conf.set(MOSAIC_RASTER_USE_CHECKPOINT, "true")
+ updateCheckpointPath(spark, withCheckpointPath)
+ logInfo(s"Checkpoint enabled for this session under $checkpointPath (overrides existing spark confs).")
+ }
+
+ /**
+ * Go back to defaults.
+ * - spark conf unset for use checkpoint (off).
+ * - spark conf unset for checkpoint path.
+ * - see mosaic_context.py as well for use.
+ *
+ * @param spark
+ * spark session to use.
+ */
+ def resetCheckpoint(spark: SparkSession): Unit = {
+ spark.conf.set(MOSAIC_RASTER_USE_CHECKPOINT, MOSAIC_RASTER_USE_CHECKPOINT_DEFAULT)
+ spark.conf.set(MOSAIC_RASTER_CHECKPOINT, MOSAIC_RASTER_CHECKPOINT_DEFAULT)
+ updateMosaicContext(spark)
+ }
+
+ /**
+ * Update the checkpoint path.
+ * - will make dirs if conditions met.
+ * - will make sure the session is consistent with these settings.
+ * - see mosaic_context.py as well for use.
+ *
+ * @param spark
+ * spark session to use.
+ * @param path
+ * supported cloud object path to use.
+ */
+ def updateCheckpointPath(spark: SparkSession, path: String): Unit = {
+ val isTestMode = spark.conf.get(MOSAIC_TEST_MODE, "false").toBoolean
+ if (path == null) {
+ val msg = "Null checkpoint path provided."
+ logError(msg)
+ throw new NullPointerException(msg)
+ } else if (!isTestMode && !PathUtils.isFuseLocation(path)) {
+ val msg = "Checkpoint path must be a (non-local) fuse location."
+ logError(msg)
+ throw new InvalidPathException(path, msg)
+ } else if (!Files.exists(Paths.get(path))) {
+ if (path.startsWith("/Volumes/")) {
+ val msg = "Volume checkpoint path doesn't exist and must be created through Databricks catalog."
+ logError(msg)
+ throw new FileNotFoundException(msg)
+ } else {
+ val dir = new File(path)
+ dir.mkdirs
+ }
+ }
+ spark.conf.set(MOSAIC_RASTER_CHECKPOINT, path)
+ updateMosaicContext(spark)
+ }
+
+ /**
+ * Set spark config to disable checkpointing.
+ * - will make sure the session is consistent with these settings.
+ * - see mosaic_context.py as well for use.
+ *
+ * @param spark
+ * spark session to use.
+ */
+ def setCheckpointOff(spark: SparkSession): Unit = {
+ spark.conf.set(MOSAIC_RASTER_USE_CHECKPOINT, "false")
+ updateMosaicContext(spark)
+ }
+
+ /**
+ * Set spark config to enable checkpointing.
+ * - will make sure the session is consistent with these settings.
+ * - see mosaic_context.py as well for use.
+ *
+ * @param spark
+ * Spark session to use.
+ */
+ def setCheckpointOn(spark: SparkSession): Unit = {
+ spark.conf.set(MOSAIC_RASTER_USE_CHECKPOINT, "true")
+ updateMosaicContext(spark)
+ }
+
+ private def updateMosaicContext(spark: SparkSession): Unit = {
+ // - necessary to register with the latest context
+ // - registers spark expressions with the new config
+ // - will make sure the session is consistent with these settings
+ if (!MosaicContext.checkContext) {
+ val mosaicConfig = MosaicExpressionConfig(spark)
+ val indexSystem = IndexSystemFactory.getIndexSystem(mosaicConfig.getIndexSystem)
+ val geometryAPI = GeometryAPI.apply(mosaicConfig.getGeometryAPI)
+ MosaicContext.build(indexSystem, geometryAPI)
+ }
+ val mc = MosaicContext.context()
+ mc.register(spark)
+ enableGDAL(spark)
+ }
+
/** Loads the shared objects required for GDAL. */
private def loadSharedObjects(): Unit = {
loadOrNOOP(usrlibsoPath)
@@ -122,19 +257,13 @@ object MosaicGDAL extends Logging {
}
}
-// /** Reads the resource bytes. */
-// private def readResourceBytes(name: String): Array[Byte] = {
-// val bis = new BufferedInputStream(getClass.getResourceAsStream(name))
-// try { Stream.continually(bis.read()).takeWhile(-1 !=).map(_.toByte).toArray }
-// finally bis.close()
-// }
-
-// /** Reads the resource lines. */
-// // noinspection SameParameterValue
-// private def readResourceLines(name: String): Array[String] = {
-// val bytes = readResourceBytes(name)
-// val lines = new String(bytes).split("\n")
-// lines
-// }
+ /** @return value of useCheckpoint. */
+ def isUseCheckpoint: Boolean = this.useCheckpoint
+
+ /** @return value of checkpoint path. */
+ def getCheckpointPath: String = this.checkpointPath
+
+ /** @return default value of checkpoint path. */
+ def getCheckpointPathDefault: String = MOSAIC_RASTER_CHECKPOINT_DEFAULT
}
diff --git a/src/main/scala/com/databricks/labs/mosaic/package.scala b/src/main/scala/com/databricks/labs/mosaic/package.scala
index 7b5f1da73..4e4716fab 100644
--- a/src/main/scala/com/databricks/labs/mosaic/package.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/package.scala
@@ -22,6 +22,8 @@ package object mosaic {
val MOSAIC_GDAL_NATIVE = "spark.databricks.labs.mosaic.gdal.native"
val MOSAIC_RASTER_CHECKPOINT = "spark.databricks.labs.mosaic.raster.checkpoint"
val MOSAIC_RASTER_CHECKPOINT_DEFAULT = "/dbfs/tmp/mosaic/raster/checkpoint"
+ val MOSAIC_RASTER_USE_CHECKPOINT = "spark.databricks.labs.mosaic.raster.use.checkpoint"
+ val MOSAIC_RASTER_USE_CHECKPOINT_DEFAULT = "false"
val MOSAIC_RASTER_TMP_PREFIX = "spark.databricks.labs.mosaic.raster.tmp.prefix"
val MOSAIC_RASTER_BLOCKSIZE = "spark.databricks.labs.mosaic.raster.blocksize"
val MOSAIC_RASTER_BLOCKSIZE_DEFAULT = "128"
diff --git a/src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala b/src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala
index 2f896c046..cbdb1b417 100644
--- a/src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala
+++ b/src/main/scala/com/databricks/labs/mosaic/utils/PathUtils.scala
@@ -4,51 +4,90 @@ import com.databricks.labs.mosaic.functions.MosaicContext
import java.nio.file.{Files, Path, Paths}
import scala.jdk.CollectionConverters._
+import scala.util.Try
object PathUtils {
val NO_PATH_STRING = "no_path"
+ val FILE_TOKEN = "file:"
+ val VSI_ZIP_TOKEN = "/vsizip/"
+ val DBFS_FUSE_TOKEN = "/dbfs"
+ val DBFS_TOKEN = "dbfs:"
+ val VOLUMES_TOKEN = "/Volumes"
+ val WORKSPACE_TOKEN = "/Workspace"
- def replaceDBFSTokens(path: String): String = {
- path
- .replace("file:/", "/")
- .replace("dbfs:/Volumes", "/Volumes")
- .replace("dbfs:/", "/dbfs/")
- }
+ /**
+ * Cleans up variations of path.
+ * - handles subdataset path
+ * - handles "aux.xml" sidecar file
+ * - handles zips, including "/vsizip/"
+ * @param path
+ */
+ def cleanUpPath(path: String): Unit = {
+ // 0.4.3 - new function
+ val isSD = isSubdataset(path)
+ val filePath = if (isSD) fromSubdatasetPath(path) else path
+ val pamFilePath = s"$filePath.aux.xml"
+ val cleanPath = filePath.replace(VSI_ZIP_TOKEN, "")
+ val zipPath = if (cleanPath.endsWith("zip")) cleanPath else s"$cleanPath.zip"
- def getCleanPath(path: String): String = {
- val cleanPath = replaceDBFSTokens(path)
- if (cleanPath.endsWith(".zip") || cleanPath.contains(".zip:")) {
- getZipPath(cleanPath)
- } else {
- cleanPath
+ Try(Files.deleteIfExists(Paths.get(cleanPath)))
+ Try(Files.deleteIfExists(Paths.get(path)))
+ Try(Files.deleteIfExists(Paths.get(filePath)))
+ Try(Files.deleteIfExists(Paths.get(pamFilePath)))
+ if (Files.exists(Paths.get(zipPath))) {
+ Try(Files.deleteIfExists(Paths.get(zipPath.replace(".zip", ""))))
}
+ Try(Files.deleteIfExists(Paths.get(zipPath)))
}
- def isSubdataset(path: String): Boolean = {
- path.split(":").length == 3
- }
+ /**
+ * Copy provided path to tmp.
+ * @param inPath
+ * Path to copy from.
+ * @return
+ * The copied path.
+ */
+ def copyToTmp(inPath: String): String = {
+ val copyFromPath = replaceDBFSTokens(inPath)
+ val inPathDir = Paths.get(copyFromPath).getParent.toString
- def getSubdatasetPath(path: String): String = {
- // Subdatasets are paths with a colon in them.
- // We need to check for this condition and handle it.
- // Subdatasets paths are formatted as: "FORMAT:/path/to/file.tif:subdataset"
- val format :: filePath :: subdataset :: Nil = path.split(":").toList
- val isZip = filePath.endsWith(".zip")
- val vsiPrefix = if (isZip) "/vsizip/" else ""
- s"$format:$vsiPrefix$filePath:$subdataset"
+ val fullFileName = copyFromPath.split("/").last
+ val stemRegex = getStemRegex(inPath)
+
+ wildcardCopy(inPathDir, MosaicContext.tmpDir(null), stemRegex)
+
+ s"${MosaicContext.tmpDir(null)}/$fullFileName"
}
- def getZipPath(path: String): String = {
- // It is really important that the resulting path is /vsizip// and not /vsizip/
- // /vsizip// is for absolute paths /viszip/ is relative to the current working directory
- // /vsizip/ wont work on a cluster
- // see: https://gdal.org/user/virtual_file_systems.html#vsizip-zip-archives
- val isZip = path.endsWith(".zip")
- val readPath = if (path.startsWith("/vsizip/")) path else if (isZip) s"/vsizip/$path" else path
- readPath
+ /**
+ * Copy path to tmp with retries.
+ * @param inPath
+ * Path to copy from.
+ * @param retries
+ * How many times to retry copy, default = 3.
+ * @return
+ * The tmp path.
+ */
+ def copyToTmpWithRetry(inPath: String, retries: Int = 3): String = {
+ var tmpPath = copyToTmp(inPath)
+ var i = 0
+ while (Files.notExists(Paths.get(tmpPath)) && i < retries) {
+ tmpPath = copyToTmp(inPath)
+ i += 1
+ }
+ tmpPath
}
+ /**
+ * Create a file under tmp dir.
+ * - Directories are created.
+ * - File itself is not create.
+ * @param extension
+ * The file extension to use.
+ * @return
+ * The tmp path.
+ */
def createTmpFilePath(extension: String): String = {
val tmpDir = MosaicContext.tmpDir(null)
val uuid = java.util.UUID.randomUUID.toString
@@ -57,6 +96,16 @@ object PathUtils {
outPath
}
+ /**
+ * File path which had a subdataset.
+ * - split on ":" and return just the path,
+ * not the subdataset.
+ * - remove any quotes at start and end.
+ * @param path
+ * Provided path.
+ * @return
+ * The path without subdataset.
+ */
def fromSubdatasetPath(path: String): String = {
val _ :: filePath :: _ :: Nil = path.split(":").toList
var result = filePath
@@ -65,6 +114,15 @@ object PathUtils {
result
}
+ /**
+ * Generate regex string of path filename.
+ * - handles fuse paths.
+ * - handles "." in the filename "stem".
+ * @param path
+ * Provided path.
+ * @return
+ * Regex string.
+ */
def getStemRegex(path: String): String = {
val cleanPath = replaceDBFSTokens(path)
val fileName = Paths.get(cleanPath).getFileName.toString
@@ -73,29 +131,139 @@ object PathUtils {
val stemRegex = s"$stemEscaped\\..*".r
stemRegex.toString
}
-
- def copyToTmpWithRetry(inPath: String, retries: Int = 3): String = {
- var tmpPath = copyToTmp(inPath)
- var i = 0
- while (Files.notExists(Paths.get(tmpPath)) && i < retries) {
- tmpPath = copyToTmp(inPath)
- i += 1
+
+ /**
+ * Get subdataset path.
+ * - these paths end with ":subdataset".
+ * - adds "/vsizip/" if needed.
+ * @param path
+ * Provided path.
+ * @return
+ * Standardized path.
+ */
+ def getSubdatasetPath(path: String): String = {
+ // Subdatasets are paths with a colon in them.
+ // We need to check for this condition and handle it.
+ // Subdatasets paths are formatted as: "FORMAT:/path/to/file.tif:subdataset"
+ val format :: filePath :: subdataset :: Nil = path.split(":").toList
+ val isZip = filePath.endsWith(".zip")
+ val vsiPrefix = if (isZip) VSI_ZIP_TOKEN else ""
+ s"$format:$vsiPrefix$filePath:$subdataset"
+ }
+
+ /**
+ * Clean path.
+ * - handles fuse paths.
+ * - handles zip paths
+ * @param path
+ * Provided path.
+ * @return
+ * Standardized string.
+ */
+ def getCleanPath(path: String): String = {
+ val cleanPath = replaceDBFSTokens(path)
+ if (cleanPath.endsWith(".zip") || cleanPath.contains(".zip:")) {
+ getZipPath(cleanPath)
+ } else {
+ cleanPath
}
- tmpPath
}
- def copyToTmp(inPath: String): String = {
- val copyFromPath = replaceDBFSTokens(inPath)
- val inPathDir = Paths.get(copyFromPath).getParent.toString
+ /**
+ * Standardize zip paths.
+ * - Add "/vsizip/" as needed.
+ * @param path
+ * Provided path.
+ * @return
+ * Standardized path.
+ */
+ def getZipPath(path: String): String = {
+ // It is really important that the resulting path is /vsizip// and not /vsizip/
+ // /vsizip// is for absolute paths /viszip/ is relative to the current working directory
+ // /vsizip/ wont work on a cluster
+ // see: https://gdal.org/user/virtual_file_systems.html#vsizip-zip-archives
+ val isZip = path.endsWith(".zip")
+ val readPath = if (path.startsWith(VSI_ZIP_TOKEN)) path else if (isZip) s"$VSI_ZIP_TOKEN$path" else path
+ readPath
+ }
- val fullFileName = copyFromPath.split("/").last
- val stemRegex = getStemRegex(inPath)
+ /**
+ * Test for whether path is in a fuse location,
+ * looks ahead somewhat beyond DBFS.
+ * - handles DBFS, Volumes, and Workspace paths.
+ * @param path
+ * Provided path.
+ * @return
+ * True if path is in a fuse location.
+ */
+ def isFuseLocation(path: String): Boolean = {
+ // 0.4.3 - new function
+ val p = getCleanPath(path)
+ val isFuse = p match {
+ case _ if (
+ p.startsWith(s"$DBFS_FUSE_TOKEN/") ||
+ p.startsWith(s"$VOLUMES_TOKEN/") ||
+ p.startsWith(s"$WORKSPACE_TOKEN/")) => true
+ case _ => false
+ }
+ isFuse
+ }
- wildcardCopy(inPathDir, MosaicContext.tmpDir(null), stemRegex)
+ /**
+ * Is the path a subdataset?
+ * - Known by ":" after the filename.
+ * @param path
+ * Provided path.
+ * @return
+ * True if is a subdataset.
+ */
+ def isSubdataset(path: String): Boolean = {
+ path.split(":").length == 3
+ }
- s"${MosaicContext.tmpDir(null)}/$fullFileName"
+ /**
+ * Parse the unzipped path from standard out.
+ * - Called after a prompt is executed to unzip.
+ * @param lastExtracted
+ * Standard out line to parse.
+ * @param extension
+ * Extension of file, e.g. "shp".
+ * @return
+ * The parsed path.
+ */
+ def parseUnzippedPathFromExtracted(lastExtracted: String, extension: String): String = {
+ val trimmed = lastExtracted.replace("extracting: ", "").replace(" ", "")
+ val indexOfFormat = trimmed.indexOf(s".$extension/")
+ trimmed.substring(0, indexOfFormat + extension.length + 1)
+ }
+
+ /**
+ * Replace various file path schemas that are not needed
+ * for internal / local handling.
+ * - handles "file:". "dbfs:"
+ * - appropriate for "/dbfs/", "/Volumes/", and "/Workspace/"
+ * paths, which can be read locally.
+ * @param path
+ * Provided path.
+ * @return
+ * Replaced string.
+ */
+ def replaceDBFSTokens(path: String): String = {
+ path
+ .replace(s"$FILE_TOKEN/", "/")
+ .replace(s"$DBFS_TOKEN$VOLUMES_TOKEN/", s"$VOLUMES_TOKEN/")
+ .replace(s"$DBFS_TOKEN/", s"$DBFS_FUSE_TOKEN/")
}
+ /**
+ * Perform a wildcard copy.
+ * @param inDirPath
+ * Provided in dir.
+ * @param outDirPath
+ * Provided out dir.
+ * @param pattern
+ * Regex pattern to match.
+ */
def wildcardCopy(inDirPath: String, outDirPath: String, pattern: String): Unit = {
import org.apache.commons.io.FileUtils
val copyFromPath = replaceDBFSTokens(inDirPath)
@@ -106,7 +274,7 @@ object PathUtils {
.filter(_.getFileName.toString.matches(pattern))
.collect(java.util.stream.Collectors.toList[Path])
.asScala
-
+
for (path <- toCopy) {
val destination = Paths.get(copyToPath, path.getFileName.toString)
// noinspection SimplifyBooleanMatch
@@ -115,10 +283,4 @@ object PathUtils {
}
}
- def parseUnzippedPathFromExtracted(lastExtracted: String, extension: String): String = {
- val trimmed = lastExtracted.replace("extracting: ", "").replace(" ", "")
- val indexOfFormat = trimmed.indexOf(s".$extension/")
- trimmed.substring(0, indexOfFormat + extension.length + 1)
- }
-
}
diff --git a/src/test/resources/binary/geotiff-small/venice_landsat_etm_2001-08-26_panchromatic.tif b/src/test/resources/binary/geotiff-small/venice_landsat_etm_2001-08-26_panchromatic.tif
new file mode 100644
index 000000000..520c9c550
Binary files /dev/null and b/src/test/resources/binary/geotiff-small/venice_landsat_etm_2001-08-26_panchromatic.tif differ
diff --git a/src/test/scala/com/databricks/labs/mosaic/core/raster/TestRasterGDAL.scala b/src/test/scala/com/databricks/labs/mosaic/core/raster/TestRasterGDAL.scala
index 8bbe4b1f3..40124029a 100644
--- a/src/test/scala/com/databricks/labs/mosaic/core/raster/TestRasterGDAL.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/core/raster/TestRasterGDAL.scala
@@ -1,5 +1,6 @@
package com.databricks.labs.mosaic.core.raster
+import com.databricks.labs.mosaic.{MOSAIC_RASTER_CHECKPOINT, MOSAIC_RASTER_USE_CHECKPOINT, MOSAIC_TEST_MODE}
import com.databricks.labs.mosaic.core.raster.gdal.MosaicRasterGDAL
import com.databricks.labs.mosaic.gdal.MosaicGDAL
import com.databricks.labs.mosaic.test.mocks.filePath
@@ -32,6 +33,11 @@ class TestRasterGDAL extends SharedSparkSessionGDAL {
resultExecutors.foreach(s => s should include("GDAL"))
}
+ test("Verify that checkpoint is not used.") {
+ spark.conf.get(MOSAIC_TEST_MODE) shouldBe "true"
+ MosaicGDAL.isUseCheckpoint shouldBe false
+ }
+
test("Read raster metadata from GeoTIFF file.") {
assume(System.getProperty("os.name") == "Linux")
@@ -338,4 +344,12 @@ class TestRasterGDAL extends SharedSparkSessionGDAL {
}
+ test("Verify that checkpoint is configured.") {
+ MosaicGDAL.enableGDALWithCheckpoint(spark, spark.conf.get(MOSAIC_RASTER_CHECKPOINT))
+ spark.conf.get(MOSAIC_TEST_MODE) shouldBe "true"
+ MosaicGDAL.isUseCheckpoint shouldBe true
+ MosaicGDAL.getCheckpointPath shouldBe spark.conf.get(MOSAIC_RASTER_CHECKPOINT)
+ spark.conf.get(MOSAIC_RASTER_USE_CHECKPOINT) shouldBe "true"
+ }
+
}
diff --git a/src/test/scala/com/databricks/labs/mosaic/datasource/OGRFileFormatTest.scala b/src/test/scala/com/databricks/labs/mosaic/datasource/OGRFileFormatTest.scala
index f37bef949..5d96c6e4a 100644
--- a/src/test/scala/com/databricks/labs/mosaic/datasource/OGRFileFormatTest.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/datasource/OGRFileFormatTest.scala
@@ -144,7 +144,7 @@ class OGRFileFormatTest extends QueryTest with SharedSparkSessionGDAL {
test("OGRFileFormat should handle partial schema: ISSUE 351") {
assume(System.getProperty("os.name") == "Linux")
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(H3, JTS)
import mc.functions._
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/format/ConvertToBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/format/ConvertToBehaviors.scala
index 614d860f9..052703fca 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/format/ConvertToBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/format/ConvertToBehaviors.scala
@@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers._
trait ConvertToBehaviors extends QueryTest {
def checkInputTypeBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -58,7 +58,7 @@ trait ConvertToBehaviors extends QueryTest {
}
def passthroughBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -67,7 +67,7 @@ trait ConvertToBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
val wkts = getWKTRowsDf().select("wkt")
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/FlattenPolygonsBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/FlattenPolygonsBehaviors.scala
index 53d4c24cd..b589d2590 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/FlattenPolygonsBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/FlattenPolygonsBehaviors.scala
@@ -16,7 +16,7 @@ import org.apache.spark.sql.functions.{col, lit}
trait FlattenPolygonsBehaviors extends QueryTest {
def flattenWKBPolygon(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -66,7 +66,7 @@ trait FlattenPolygonsBehaviors extends QueryTest {
}
def flattenWKTPolygon(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -116,7 +116,7 @@ trait FlattenPolygonsBehaviors extends QueryTest {
}
def flattenCOORDSPolygon(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -177,7 +177,7 @@ trait FlattenPolygonsBehaviors extends QueryTest {
}
def flattenHEXPolygon(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -238,7 +238,7 @@ trait FlattenPolygonsBehaviors extends QueryTest {
}
def failDataTypeCheck(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -256,7 +256,7 @@ trait FlattenPolygonsBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AreaBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AreaBehaviors.scala
index 38a94ac88..0f4bd013b 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AreaBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AreaBehaviors.scala
@@ -14,7 +14,7 @@ import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
trait ST_AreaBehaviors extends MosaicSpatialQueryTest {
def areaBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
@@ -45,7 +45,7 @@ trait ST_AreaBehaviors extends MosaicSpatialQueryTest {
}
def areaCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
@@ -70,7 +70,7 @@ trait ST_AreaBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AsGeoJSONTileAggBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AsGeoJSONTileAggBehaviors.scala
index a77b90df4..8cd0fac6a 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AsGeoJSONTileAggBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AsGeoJSONTileAggBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers.{be, convertToAnyShouldWrapper}
trait ST_AsGeoJSONTileAggBehaviors extends MosaicSpatialQueryTest {
def behavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AsMVTTileAggBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AsMVTTileAggBehaviors.scala
index 2dbb0fbcf..5047631a3 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AsMVTTileAggBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_AsMVTTileAggBehaviors.scala
@@ -14,7 +14,7 @@ import java.nio.file.{Files, Paths}
trait ST_AsMVTTileAggBehaviors extends MosaicSpatialQueryTest {
def behavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_BufferBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_BufferBehaviors.scala
index 972a474a0..b6ebaa1d0 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_BufferBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_BufferBehaviors.scala
@@ -20,7 +20,7 @@ import scala.collection.JavaConverters._
trait ST_BufferBehaviors extends QueryTest {
def bufferBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
val sc = spark
@@ -99,7 +99,7 @@ trait ST_BufferBehaviors extends QueryTest {
}
def bufferCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -206,7 +206,7 @@ trait ST_BufferBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
import mc.functions._
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_BufferLoopBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_BufferLoopBehaviors.scala
index 90855c817..8e5a5e165 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_BufferLoopBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_BufferLoopBehaviors.scala
@@ -34,7 +34,7 @@ trait ST_BufferLoopBehaviors extends MosaicSpatialQueryTest {
}
def codegenCompilation(mc: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
mc.register(sc)
@@ -57,7 +57,7 @@ trait ST_BufferLoopBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mc: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
mc.register(sc)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_CentroidBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_CentroidBehaviors.scala
index 055df10db..e6e947090 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_CentroidBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_CentroidBehaviors.scala
@@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_CentroidBehaviors extends MosaicSpatialQueryTest {
def centroidBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
@@ -62,7 +62,7 @@ trait ST_CentroidBehaviors extends MosaicSpatialQueryTest {
}
def centroidCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -92,7 +92,7 @@ trait ST_CentroidBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ConcaveHullBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ConcaveHullBehaviors.scala
index 1d75bc146..63feb8d03 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ConcaveHullBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ConcaveHullBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_ConcaveHullBehaviors extends QueryTest {
def concaveHullBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
val sc = spark
@@ -50,7 +50,7 @@ trait ST_ConcaveHullBehaviors extends QueryTest {
}
def concaveHullCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -81,7 +81,7 @@ trait ST_ConcaveHullBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ContainsBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ContainsBehaviors.scala
index 98c8999c3..33e127533 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ContainsBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ContainsBehaviors.scala
@@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_ContainsBehaviors extends MosaicSpatialQueryTest {
def containsBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
@@ -37,7 +37,7 @@ trait ST_ContainsBehaviors extends MosaicSpatialQueryTest {
}
def containsCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -79,7 +79,7 @@ trait ST_ContainsBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ConvexHullBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ConvexHullBehaviors.scala
index 607e0f3e5..660e02f5e 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ConvexHullBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ConvexHullBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_ConvexHullBehaviors extends QueryTest {
def convexHullBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
val sc = spark
@@ -37,7 +37,7 @@ trait ST_ConvexHullBehaviors extends QueryTest {
}
def convexHullCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -70,7 +70,7 @@ trait ST_ConvexHullBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DifferenceBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DifferenceBehaviors.scala
index 1b96c90ab..e2abd573d 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DifferenceBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DifferenceBehaviors.scala
@@ -28,7 +28,7 @@ trait ST_DifferenceBehaviors extends MosaicSpatialQueryTest {
}
def codegenCompilation(mc: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
mc.register(sc)
@@ -55,7 +55,7 @@ trait ST_DifferenceBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mc: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
mc.register(sc)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DimensionBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DimensionBehaviors.scala
index 605e305b6..2db320846 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DimensionBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DimensionBehaviors.scala
@@ -14,7 +14,7 @@ import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
trait ST_DimensionBehaviors extends MosaicSpatialQueryTest {
def dimensionBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
@@ -45,7 +45,7 @@ trait ST_DimensionBehaviors extends MosaicSpatialQueryTest {
}
def areaCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
@@ -70,7 +70,7 @@ trait ST_DimensionBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DistanceBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DistanceBehaviors.scala
index 54de04600..5615e21c4 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DistanceBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_DistanceBehaviors.scala
@@ -18,7 +18,7 @@ import scala.collection.immutable
trait ST_DistanceBehaviors extends QueryTest {
def distanceBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
val sc = spark
@@ -51,7 +51,7 @@ trait ST_DistanceBehaviors extends QueryTest {
}
def distanceCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -87,7 +87,7 @@ trait ST_DistanceBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_EnvelopeBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_EnvelopeBehaviors.scala
index fda89a75f..e659aa098 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_EnvelopeBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_EnvelopeBehaviors.scala
@@ -24,7 +24,7 @@ trait ST_EnvelopeBehaviors extends MosaicSpatialQueryTest {
}
def codegenCompilation(mc: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
mc.register(sc)
@@ -47,7 +47,7 @@ trait ST_EnvelopeBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mc: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
mc.register(sc)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_GeometryTypeBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_GeometryTypeBehaviors.scala
index 6a6bbe620..abc4bfd9f 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_GeometryTypeBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_GeometryTypeBehaviors.scala
@@ -15,7 +15,7 @@ import java.util.Locale
trait ST_GeometryTypeBehaviors extends MosaicSpatialQueryTest {
def wktTypesBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -48,7 +48,7 @@ trait ST_GeometryTypeBehaviors extends MosaicSpatialQueryTest {
}
def wktTypesCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -79,7 +79,7 @@ trait ST_GeometryTypeBehaviors extends MosaicSpatialQueryTest {
}
def hexTypesBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -114,7 +114,7 @@ trait ST_GeometryTypeBehaviors extends MosaicSpatialQueryTest {
}
def hexTypesCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -143,7 +143,7 @@ trait ST_GeometryTypeBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_HasValidCoordinatesBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_HasValidCoordinatesBehaviors.scala
index 3c7c2deb0..37ff278c2 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_HasValidCoordinatesBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_HasValidCoordinatesBehaviors.scala
@@ -19,7 +19,7 @@ trait ST_HasValidCoordinatesBehaviors extends MosaicSpatialQueryTest {
// noinspection AccessorLikeMethodIsUnit
def hasValidCoordinatesBehaviours(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("FATAL") // <- otherwise asserted exceptions print
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -103,7 +103,7 @@ trait ST_HasValidCoordinatesBehaviors extends MosaicSpatialQueryTest {
}
def expressionCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -130,7 +130,7 @@ trait ST_HasValidCoordinatesBehaviors extends MosaicSpatialQueryTest {
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_HaversineBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_HaversineBehaviors.scala
index f9fbd8c4b..189dbb9bd 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_HaversineBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_HaversineBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
trait ST_HaversineBehaviors extends QueryTest {
def haversineBehaviour(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -25,7 +25,7 @@ trait ST_HaversineBehaviors extends QueryTest {
}
def haversineCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -45,7 +45,7 @@ trait ST_HaversineBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IntersectionBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IntersectionBehaviors.scala
index 7007e93dc..0a244abf1 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IntersectionBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IntersectionBehaviors.scala
@@ -20,7 +20,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_IntersectionBehaviors extends QueryTest {
def intersectionBehaviour(indexSystem: IndexSystem, geometryAPI: GeometryAPI, resolution: Int): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -71,7 +71,7 @@ trait ST_IntersectionBehaviors extends QueryTest {
}
def intersectionAggBehaviour(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -166,7 +166,7 @@ trait ST_IntersectionBehaviors extends QueryTest {
}
def selfIntersectionBehaviour(indexSystem: IndexSystem, geometryAPI: GeometryAPI, resolution: Int): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -212,7 +212,7 @@ trait ST_IntersectionBehaviors extends QueryTest {
}
def intersectionCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -238,7 +238,7 @@ trait ST_IntersectionBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IntersectsBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IntersectsBehaviors.scala
index a94409e34..a921969a0 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IntersectsBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IntersectsBehaviors.scala
@@ -17,7 +17,7 @@ import org.scalatest.matchers.should.Matchers.{be, convertToAnyShouldWrapper, no
trait ST_IntersectsBehaviors extends QueryTest {
def intersectsBehaviour(indexSystem: IndexSystem, geometryAPI: GeometryAPI, resolution: Int): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -83,7 +83,7 @@ trait ST_IntersectsBehaviors extends QueryTest {
}
def intersectsAggBehaviour(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -138,7 +138,7 @@ trait ST_IntersectsBehaviors extends QueryTest {
}
def selfIntersectsBehaviour(indexSystem: IndexSystem, geometryAPI: GeometryAPI, resolution: Int): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
import mc.functions._
mc.register(spark)
@@ -182,7 +182,7 @@ trait ST_IntersectsBehaviors extends QueryTest {
}
def intersectsCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -208,7 +208,7 @@ trait ST_IntersectsBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidBehaviors.scala
index a11710f9e..40da78805 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_IsValidBehaviors.scala
@@ -14,7 +14,7 @@ import org.scalatest.matchers.should.Matchers.{all, an, be, convertToAnyShouldWr
trait ST_IsValidBehaviors extends MosaicSpatialQueryTest {
def isValidBehaviour(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
val mc = mosaicContext
import mc.functions._
@@ -80,7 +80,7 @@ trait ST_IsValidBehaviors extends MosaicSpatialQueryTest {
}
def isValidCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -108,7 +108,7 @@ trait ST_IsValidBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_LengthBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_LengthBehaviors.scala
index f24bb18b1..f673d0ba8 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_LengthBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_LengthBehaviors.scala
@@ -11,7 +11,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_LengthBehaviors extends MosaicSpatialQueryTest {
def lengthBehaviour(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
val mc = mosaicContext
import mc.functions._
@@ -61,7 +61,7 @@ trait ST_LengthBehaviors extends MosaicSpatialQueryTest {
}
def lengthCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -91,7 +91,7 @@ trait ST_LengthBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_MinMaxXYZBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_MinMaxXYZBehaviors.scala
index b6d79406c..3358fa789 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_MinMaxXYZBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_MinMaxXYZBehaviors.scala
@@ -15,7 +15,7 @@ import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
trait ST_MinMaxXYZBehaviors extends MosaicSpatialQueryTest {
def xMinBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -38,7 +38,7 @@ trait ST_MinMaxXYZBehaviors extends MosaicSpatialQueryTest {
}
def xMaxBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -62,7 +62,7 @@ trait ST_MinMaxXYZBehaviors extends MosaicSpatialQueryTest {
}
def yMinBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -85,7 +85,7 @@ trait ST_MinMaxXYZBehaviors extends MosaicSpatialQueryTest {
}
def yMaxBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -109,7 +109,7 @@ trait ST_MinMaxXYZBehaviors extends MosaicSpatialQueryTest {
}
def xMinCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -131,7 +131,7 @@ trait ST_MinMaxXYZBehaviors extends MosaicSpatialQueryTest {
}
def xMaxCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -153,7 +153,7 @@ trait ST_MinMaxXYZBehaviors extends MosaicSpatialQueryTest {
}
def yMinCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -175,7 +175,7 @@ trait ST_MinMaxXYZBehaviors extends MosaicSpatialQueryTest {
}
def yMaxCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -197,7 +197,7 @@ trait ST_MinMaxXYZBehaviors extends MosaicSpatialQueryTest {
}
def auxiliary(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val df = getWKTRowsDf().orderBy("id")
val expr = ST_MinMaxXYZ(df.col("wkt").expr, mc.expressionConfig, "X", "MAX")
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_NumPointsBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_NumPointsBehaviors.scala
index c33d4d19f..ea895b863 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_NumPointsBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_NumPointsBehaviors.scala
@@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, contain, convertToAnyShou
trait ST_NumPointsBehaviors extends MosaicSpatialQueryTest {
def numPointsBehaviour(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -41,7 +41,7 @@ trait ST_NumPointsBehaviors extends MosaicSpatialQueryTest {
}
def codegenCompilation(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
mc.register(sc)
@@ -64,7 +64,7 @@ trait ST_NumPointsBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_RotateBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_RotateBehaviors.scala
index 3ff0d38b0..bba61f1a7 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_RotateBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_RotateBehaviors.scala
@@ -14,7 +14,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_RotateBehaviors extends QueryTest {
def rotateBehaviour(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -35,7 +35,7 @@ trait ST_RotateBehaviors extends QueryTest {
}
def rotateCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -64,7 +64,7 @@ trait ST_RotateBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SRIDBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SRIDBehaviors.scala
index 142d84569..2332996b4 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SRIDBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SRIDBehaviors.scala
@@ -18,7 +18,7 @@ import scala.collection.JavaConverters._
trait ST_SRIDBehaviors extends MosaicSpatialQueryTest {
def SRIDBehaviour(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
val mc = mosaicContext
import mc.functions._
@@ -63,7 +63,7 @@ trait ST_SRIDBehaviors extends MosaicSpatialQueryTest {
}
def SRIDCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
val mc = mosaicContext
import mc.functions._
@@ -107,7 +107,7 @@ trait ST_SRIDBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ScaleBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ScaleBehaviors.scala
index 8cd3962ba..8438789af 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ScaleBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ScaleBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_ScaleBehaviors extends QueryTest {
def scaleBehaviour(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -34,7 +34,7 @@ trait ST_ScaleBehaviors extends QueryTest {
}
def scaleCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -63,7 +63,7 @@ trait ST_ScaleBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SetSRIDBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SetSRIDBehaviors.scala
index 062c5bcc9..332d25c6d 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SetSRIDBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SetSRIDBehaviors.scala
@@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers.{be, convertToAnyShouldWrapper, no
trait ST_SetSRIDBehaviors extends QueryTest {
def setSRIDBehaviour(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -63,7 +63,7 @@ trait ST_SetSRIDBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SimplifyBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SimplifyBehaviors.scala
index 672ca0efa..b30dedf3f 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SimplifyBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_SimplifyBehaviors.scala
@@ -15,7 +15,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_SimplifyBehaviors extends QueryTest {
def simplifyBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -51,7 +51,7 @@ trait ST_SimplifyBehaviors extends QueryTest {
}
def simplifyCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -80,7 +80,7 @@ trait ST_SimplifyBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
import mc.functions._
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_TransformBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_TransformBehaviors.scala
index 6c11580ec..ee24ad485 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_TransformBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_TransformBehaviors.scala
@@ -20,7 +20,7 @@ trait ST_TransformBehaviors extends QueryTest {
val geomFactory = new GeometryFactory()
def reprojectGeometries(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -116,7 +116,7 @@ trait ST_TransformBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_TranslateBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_TranslateBehaviors.scala
index 39eb08ef6..ce3bfd282 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_TranslateBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_TranslateBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_TranslateBehaviors extends QueryTest {
def translateBehaviour(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -35,7 +35,7 @@ trait ST_TranslateBehaviors extends QueryTest {
}
def translateCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -64,7 +64,7 @@ trait ST_TranslateBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UnaryUnionBehaviours.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UnaryUnionBehaviours.scala
index 7f4508c24..5ea6f5ecc 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UnaryUnionBehaviours.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UnaryUnionBehaviours.scala
@@ -24,7 +24,7 @@ trait ST_UnaryUnionBehaviours extends MosaicSpatialQueryTest {
}
def codegenCompilation(mc: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
mc.register(sc)
@@ -47,7 +47,7 @@ trait ST_UnaryUnionBehaviours extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mc: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
mc.register(sc)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UnionBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UnionBehaviors.scala
index 3c51530e9..d2c3f89d2 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UnionBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UnionBehaviors.scala
@@ -15,7 +15,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_UnionBehaviors extends QueryTest {
def unionBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -40,7 +40,7 @@ trait ST_UnionBehaviors extends QueryTest {
}
def unionAggBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -78,7 +78,7 @@ trait ST_UnionBehaviors extends QueryTest {
}
def unionCodegen(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
val sc = spark
import mc.functions._
@@ -105,7 +105,7 @@ trait ST_UnionBehaviors extends QueryTest {
}
def auxiliaryMethods(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UpdateSRIDBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UpdateSRIDBehaviors.scala
index 7559698e8..95792e0f4 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UpdateSRIDBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_UpdateSRIDBehaviors.scala
@@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers.{be, convertToAnyShouldWrapper, no
trait ST_UpdateSRIDBehaviors extends MosaicSpatialQueryTest {
def updateSRIDBehaviour(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
val mc = mosaicContext
import mc.functions._
@@ -63,7 +63,7 @@ trait ST_UpdateSRIDBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_WithinBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_WithinBehaviors.scala
index fba4805b5..6d3c82a6d 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_WithinBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_WithinBehaviors.scala
@@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_WithinBehaviors extends MosaicSpatialQueryTest {
def withinBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
@@ -37,7 +37,7 @@ trait ST_WithinBehaviors extends MosaicSpatialQueryTest {
}
def withinCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -79,7 +79,7 @@ trait ST_WithinBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_XBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_XBehaviors.scala
index 81d110980..be6312548 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_XBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_XBehaviors.scala
@@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_XBehaviors extends MosaicSpatialQueryTest {
def stxBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
@@ -50,7 +50,7 @@ trait ST_XBehaviors extends MosaicSpatialQueryTest {
}
def stxCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -80,7 +80,7 @@ trait ST_XBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_YBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_YBehaviors.scala
index 547c8a62b..c0d9de6b8 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_YBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_YBehaviors.scala
@@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_YBehaviors extends MosaicSpatialQueryTest {
def styBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
@@ -50,7 +50,7 @@ trait ST_YBehaviors extends MosaicSpatialQueryTest {
}
def styCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -80,7 +80,7 @@ trait ST_YBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ZBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ZBehaviors.scala
index 3ce7ad57d..9e9b47919 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ZBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/geometry/ST_ZBehaviors.scala
@@ -12,7 +12,7 @@ import org.scalatest.matchers.should.Matchers.{an, be, convertToAnyShouldWrapper
trait ST_ZBehaviors extends MosaicSpatialQueryTest {
def stzBehavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
@@ -35,7 +35,7 @@ trait ST_ZBehaviors extends MosaicSpatialQueryTest {
}
def stzCodegen(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val sc = spark
import mc.functions._
@@ -73,7 +73,7 @@ trait ST_ZBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellAreaBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellAreaBehaviors.scala
index 7f75c2098..1e54c93d0 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellAreaBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellAreaBehaviors.scala
@@ -11,7 +11,7 @@ import org.scalatest.matchers.should.Matchers._
trait CellAreaBehaviors extends MosaicSpatialQueryTest {
def behaviorComputedColumns(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -45,7 +45,7 @@ trait CellAreaBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
val mc = mosaicContext
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellIntersectionAggBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellIntersectionAggBehaviors.scala
index 84f3ddec9..40a811860 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellIntersectionAggBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellIntersectionAggBehaviors.scala
@@ -11,7 +11,7 @@ import org.scalatest.matchers.should.Matchers._
trait CellIntersectionAggBehaviors extends MosaicSpatialQueryTest {
def behaviorComputedColumns(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -87,7 +87,7 @@ trait CellIntersectionAggBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
val mc = mosaicContext
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellIntersectionBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellIntersectionBehaviors.scala
index c8dce2f01..6afca59cb 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellIntersectionBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellIntersectionBehaviors.scala
@@ -11,7 +11,7 @@ import org.scalatest.matchers.should.Matchers._
trait CellIntersectionBehaviors extends MosaicSpatialQueryTest {
def behaviorComputedColumns(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -92,7 +92,7 @@ trait CellIntersectionBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
val mc = mosaicContext
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKLoopBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKLoopBehaviors.scala
index 15fe02c20..4e7e23d06 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKLoopBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKLoopBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers._
trait CellKLoopBehaviors extends MosaicSpatialQueryTest {
def behaviorComputedColumns(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -41,7 +41,7 @@ trait CellKLoopBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
val mc = mosaicContext
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKLoopExplodeBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKLoopExplodeBehaviors.scala
index 0583e002c..972a64e9c 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKLoopExplodeBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKLoopExplodeBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers._
trait CellKLoopExplodeBehaviors extends MosaicSpatialQueryTest {
def behaviorComputedColumns(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -44,7 +44,7 @@ trait CellKLoopExplodeBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKRingBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKRingBehaviors.scala
index 8302fd833..b751d6729 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKRingBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKRingBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers._
trait CellKRingBehaviors extends MosaicSpatialQueryTest {
def behaviorComputedColumns(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -41,7 +41,7 @@ trait CellKRingBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
val mc = mosaicContext
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKRingExplodeBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKRingExplodeBehaviors.scala
index 9d0a8b890..d56971f72 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKRingExplodeBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellKRingExplodeBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers._
trait CellKRingExplodeBehaviors extends MosaicSpatialQueryTest {
def behaviorComputedColumns(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -44,7 +44,7 @@ trait CellKRingExplodeBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellUnionAggBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellUnionAggBehaviors.scala
index e34df0b15..ee47df8c7 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellUnionAggBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellUnionAggBehaviors.scala
@@ -11,7 +11,7 @@ import org.scalatest.matchers.should.Matchers._
trait CellUnionAggBehaviors extends MosaicSpatialQueryTest {
def behaviorComputedColumns(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -86,7 +86,7 @@ trait CellUnionAggBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
val mc = mosaicContext
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellUnionBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellUnionBehaviors.scala
index f758d93ea..6e0557dac 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellUnionBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/CellUnionBehaviors.scala
@@ -11,7 +11,7 @@ import org.scalatest.matchers.should.Matchers._
trait CellUnionBehaviors extends MosaicSpatialQueryTest {
def behaviorComputedColumns(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -95,7 +95,7 @@ trait CellUnionBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
val mc = mosaicContext
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKLoopBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKLoopBehaviors.scala
index 99e0002c6..b45096c0d 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKLoopBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKLoopBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers._
trait GeometryKLoopBehaviors extends MosaicSpatialQueryTest {
def behavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -43,7 +43,7 @@ trait GeometryKLoopBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
val mc = mosaicContext
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKLoopExplodeBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKLoopExplodeBehaviors.scala
index b360fe8ab..41ae44d55 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKLoopExplodeBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKLoopExplodeBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers._
trait GeometryKLoopExplodeBehaviors extends MosaicSpatialQueryTest {
def behavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -46,7 +46,7 @@ trait GeometryKLoopExplodeBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKRingBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKRingBehaviors.scala
index ada0d9697..329119c69 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKRingBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKRingBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers._
trait GeometryKRingBehaviors extends MosaicSpatialQueryTest {
def behavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -43,7 +43,7 @@ trait GeometryKRingBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
val mc = mosaicContext
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKRingExplodeBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKRingExplodeBehaviors.scala
index 5b292b279..d1efe88ba 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKRingExplodeBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/GeometryKRingExplodeBehaviors.scala
@@ -15,7 +15,7 @@ import org.scalatest.matchers.should.Matchers._
trait GeometryKRingExplodeBehaviors extends MosaicSpatialQueryTest {
def behavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -48,7 +48,7 @@ trait GeometryKRingExplodeBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
val sc = spark
@@ -103,7 +103,7 @@ trait GeometryKRingExplodeBehaviors extends MosaicSpatialQueryTest {
}
def issue360(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/GridDistanceBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/GridDistanceBehaviors.scala
index ebb451039..89e20222d 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/GridDistanceBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/GridDistanceBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers._
trait GridDistanceBehaviors extends MosaicSpatialQueryTest {
def behaviorGridDistance(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
val sc = spark
@@ -57,7 +57,7 @@ trait GridDistanceBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
val mc = mosaicContext
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/IndexGeometryBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/IndexGeometryBehaviors.scala
index fb4e0ab00..d71ee64b1 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/IndexGeometryBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/IndexGeometryBehaviors.scala
@@ -14,7 +14,7 @@ import org.scalatest.matchers.should.Matchers._
trait IndexGeometryBehaviors extends MosaicSpatialQueryTest {
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/MosaicExplodeBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/MosaicExplodeBehaviors.scala
index dd5b71718..3bf277ad7 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/MosaicExplodeBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/MosaicExplodeBehaviors.scala
@@ -15,7 +15,7 @@ import org.scalatest.matchers.should.Matchers._
trait MosaicExplodeBehaviors extends MosaicSpatialQueryTest {
def wktDecompose(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -48,7 +48,7 @@ trait MosaicExplodeBehaviors extends MosaicSpatialQueryTest {
}
def wktDecomposeNoNulls(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -105,7 +105,7 @@ trait MosaicExplodeBehaviors extends MosaicSpatialQueryTest {
}
def wktDecomposeKeepCoreParamExpression(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
@@ -141,7 +141,7 @@ trait MosaicExplodeBehaviors extends MosaicSpatialQueryTest {
}
def lineDecompose(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -175,7 +175,7 @@ trait MosaicExplodeBehaviors extends MosaicSpatialQueryTest {
}
def lineDecomposeFirstPointOnBoundary(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
@@ -208,7 +208,7 @@ trait MosaicExplodeBehaviors extends MosaicSpatialQueryTest {
}
def wkbDecompose(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -241,7 +241,7 @@ trait MosaicExplodeBehaviors extends MosaicSpatialQueryTest {
}
def hexDecompose(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -274,7 +274,7 @@ trait MosaicExplodeBehaviors extends MosaicSpatialQueryTest {
}
def coordsDecompose(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -322,7 +322,7 @@ trait MosaicExplodeBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
val sc = spark
@@ -383,7 +383,7 @@ trait MosaicExplodeBehaviors extends MosaicSpatialQueryTest {
}
def issue360(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/PointIndexBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/PointIndexBehaviors.scala
index b616d6091..06e4c27f7 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/PointIndexBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/PointIndexBehaviors.scala
@@ -16,7 +16,7 @@ import org.scalatest.matchers.should.Matchers._
trait PointIndexBehaviors extends MosaicSpatialQueryTest {
def behaviorInt(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -49,7 +49,7 @@ trait PointIndexBehaviors extends MosaicSpatialQueryTest {
}
def behaviorString(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -92,7 +92,7 @@ trait PointIndexBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
import mc.functions._
@@ -144,7 +144,7 @@ trait PointIndexBehaviors extends MosaicSpatialQueryTest {
}
def issue_383(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
import mc.functions._
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/index/PolyfillBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/index/PolyfillBehaviors.scala
index f602d85c5..97dbe3b12 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/index/PolyfillBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/index/PolyfillBehaviors.scala
@@ -13,7 +13,7 @@ import org.scalatest.matchers.should.Matchers._
trait PolyfillBehaviors extends MosaicSpatialQueryTest {
def polyfillOnComputedColumns(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -36,7 +36,7 @@ trait PolyfillBehaviors extends MosaicSpatialQueryTest {
}
def wktPolyfill(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -69,7 +69,7 @@ trait PolyfillBehaviors extends MosaicSpatialQueryTest {
}
def wkbPolyfill(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -101,7 +101,7 @@ trait PolyfillBehaviors extends MosaicSpatialQueryTest {
}
def hexPolyfill(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -133,7 +133,7 @@ trait PolyfillBehaviors extends MosaicSpatialQueryTest {
}
def coordsPolyfill(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
import mc.functions._
mc.register(spark)
@@ -171,7 +171,7 @@ trait PolyfillBehaviors extends MosaicSpatialQueryTest {
}
def auxiliaryMethods(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
val mc = mosaicContext
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_BandMetadataBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_BandMetadataBehaviors.scala
index 6d570c757..31def8ecc 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_BandMetadataBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_BandMetadataBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_BandMetadataBehaviors extends QueryTest {
def bandMetadataBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
val sc = spark
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_BoundingBoxBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_BoundingBoxBehaviors.scala
index e8c7cc214..037674d2c 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_BoundingBoxBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_BoundingBoxBehaviors.scala
@@ -10,7 +10,7 @@ trait RST_BoundingBoxBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ClipBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ClipBehaviors.scala
index 397f02a95..9d9328ca6 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ClipBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ClipBehaviors.scala
@@ -10,7 +10,7 @@ trait RST_ClipBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgAggBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgAggBehaviors.scala
index 7756d0ff8..47989be20 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgAggBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgAggBehaviors.scala
@@ -10,7 +10,7 @@ trait RST_CombineAvgAggBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgBehaviors.scala
index 4b7943d8a..1ac5c963e 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_CombineAvgBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_CombineAvgBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ConvolveBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ConvolveBehaviors.scala
index fdc09cc28..6af94093d 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ConvolveBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ConvolveBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_ConvolveBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandAggBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandAggBehaviors.scala
index fac40c9a9..5aa00b04b 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandAggBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandAggBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_DerivedBandAggBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandBehaviors.scala
index d883fc5cc..03a5d7101 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_DerivedBandBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_DerivedBandBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromBandsBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromBandsBehaviors.scala
index 24e5897f7..da36a46a4 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromBandsBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromBandsBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_FromBandsBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromFileBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromFileBehaviors.scala
index 625123bad..7b1bc2135 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromFileBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_FromFileBehaviors.scala
@@ -10,7 +10,7 @@ trait RST_FromFileBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GeoReferenceBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GeoReferenceBehaviors.scala
index e5ab3c159..52f0345fa 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GeoReferenceBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GeoReferenceBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_GeoReferenceBehaviors extends QueryTest {
//noinspection MapGetGet
def geoReferenceBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetNoDataBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetNoDataBehaviors.scala
index 69bf7f0f9..ef7b7ee37 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetNoDataBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetNoDataBehaviors.scala
@@ -10,7 +10,7 @@ trait RST_GetNoDataBehaviors extends QueryTest {
//noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetSubdatasetBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetSubdatasetBehaviors.scala
index 99307a158..dfaa91dee 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetSubdatasetBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_GetSubdatasetBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_GetSubdatasetBehaviors extends QueryTest {
//noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_HeightBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_HeightBehaviors.scala
index a39c02ca3..533ac0f39 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_HeightBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_HeightBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_HeightBehaviors extends QueryTest {
def heightBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoDataBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoDataBehaviors.scala
index 1af81d15d..89a31b983 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoDataBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_InitNoDataBehaviors.scala
@@ -10,7 +10,7 @@ trait RST_InitNoDataBehaviors extends QueryTest {
//noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_IsEmptyBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_IsEmptyBehaviors.scala
index 6ce9771e9..4621af067 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_IsEmptyBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_IsEmptyBehaviors.scala
@@ -10,7 +10,7 @@ trait RST_IsEmptyBehaviors extends QueryTest {
// noinspection AccessorLikeMethodIsUnit
def isEmptyBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MemSizeBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MemSizeBehaviors.scala
index 316482c1c..967694ca7 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MemSizeBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MemSizeBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_MemSizeBehaviors extends QueryTest {
def memSizeBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala
index 8698b46af..d93337e3e 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeAggBehaviors.scala
@@ -10,7 +10,7 @@ trait RST_MergeAggBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala
index 330345f20..17a122afc 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MergeBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_MergeBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MetadataBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MetadataBehaviors.scala
index c335ee0e9..96a906b8e 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MetadataBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_MetadataBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_MetadataBehaviors extends QueryTest {
def metadataBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_NumBandsBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_NumBandsBehaviors.scala
index 49800ed40..5afc1324a 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_NumBandsBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_NumBandsBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_NumBandsBehaviors extends QueryTest {
def numBandsBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelHeightBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelHeightBehaviors.scala
index 0d6565300..4e1df37b3 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelHeightBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelHeightBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_PixelHeightBehaviors extends QueryTest {
def pixelHeightBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelWidthBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelWidthBehaviors.scala
index 346172bfb..ab37db577 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelWidthBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_PixelWidthBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_PixelWidthBehaviors extends QueryTest {
def pixelWidthBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridAvgBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridAvgBehaviors.scala
index d41882506..5233500e0 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridAvgBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridAvgBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_RasterToGridAvgBehaviors extends QueryTest {
def rasterToGridAvgBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridCountBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridCountBehaviors.scala
index 5d10766f3..a703ab1c8 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridCountBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridCountBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_RasterToGridCountBehaviors extends QueryTest {
def rasterToGridCountBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMaxBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMaxBehaviors.scala
index df5d8ee5e..c004185c3 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMaxBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMaxBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_RasterToGridMaxBehaviors extends QueryTest {
def rasterToGridMaxBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMedianBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMedianBehaviors.scala
index ade27ed78..c52557a27 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMedianBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMedianBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_RasterToGridMedianBehaviors extends QueryTest {
def rasterToGridMedianBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMinBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMinBehaviors.scala
index 38ab49db1..aba668d75 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMinBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToGridMinBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_RasterToGridMinBehaviors extends QueryTest {
def rasterToGridMinBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordBehaviors.scala
index cbd00572c..e0bbc6f91 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_RasterToWorldCoordBehaviors extends QueryTest {
def rasterToWorldCoordBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordXBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordXBehaviors.scala
index 2fab5d5ec..7befd7dda 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordXBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordXBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_RasterToWorldCoordXBehaviors extends QueryTest {
def rasterToWorldCoordX(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordYBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordYBehaviors.scala
index 9da7b5ec9..ac7b6bd38 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordYBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RasterToWorldCoordYBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_RasterToWorldCoordYBehaviors extends QueryTest {
def rasterToWorldCoordY(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTileBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTileBehaviors.scala
index 118a64e2a..4b73a868c 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTileBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ReTileBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_ReTileBehaviors extends QueryTest {
def retileBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RotationBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RotationBehaviors.scala
index 9bdbb7ef1..c86c6c905 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RotationBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_RotationBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_RotationBehaviors extends QueryTest {
def rotationBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SRIDBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SRIDBehaviors.scala
index 31fde84c4..ec4dbfd1b 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SRIDBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SRIDBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_SRIDBehaviors extends QueryTest {
def sridBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleXBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleXBehaviors.scala
index 3543fb469..5ffd13c0a 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleXBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleXBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_ScaleXBehaviors extends QueryTest {
def scaleXBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleYBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleYBehaviors.scala
index 1251f660a..57278a281 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleYBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ScaleYBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_ScaleYBehaviors extends QueryTest {
def scaleYBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoDataBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoDataBehaviors.scala
index 2d35388fd..0d2113ebe 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoDataBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SetNoDataBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_SetNoDataBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewXBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewXBehaviors.scala
index f7e745613..b031c0bd8 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewXBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewXBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_SkewXBehaviors extends QueryTest {
def skewXBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewYBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewYBehaviors.scala
index ef444858f..ec04eb739 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewYBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SkewYBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_SkewYBehaviors extends QueryTest {
def skewYBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SubdatasetsBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SubdatasetsBehaviors.scala
index c8f7d435b..ca424d36b 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SubdatasetsBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SubdatasetsBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_SubdatasetsBehaviors extends QueryTest {
def subdatasetsBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SummaryBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SummaryBehaviors.scala
index 610e7c657..72ce87e7d 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SummaryBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_SummaryBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_SummaryBehaviors extends QueryTest {
def summaryBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TessellateBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TessellateBehaviors.scala
index 38d3fc778..01e05f167 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TessellateBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TessellateBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_TessellateBehaviors extends QueryTest {
// noinspection MapGetGet
def tessellateBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ToOverlappingTilesBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ToOverlappingTilesBehaviors.scala
index 560e54dee..bfd202a3b 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ToOverlappingTilesBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_ToOverlappingTilesBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_ToOverlappingTilesBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TransformBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TransformBehaviors.scala
index 9ce449b13..8d6b1d111 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TransformBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TransformBehaviors.scala
@@ -11,7 +11,7 @@ trait RST_TransformBehaviors extends QueryTest {
// noinspection MapGetGet
def behavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TryOpenBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TryOpenBehaviors.scala
index 257d1cfc3..b2b82c79c 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TryOpenBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_TryOpenBehaviors.scala
@@ -10,7 +10,7 @@ trait RST_TryOpenBehaviors extends QueryTest {
// noinspection MapGetGet
def behaviors(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftXBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftXBehaviors.scala
index e997a5c87..eac5969a1 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftXBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftXBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_UpperLeftXBehaviors extends QueryTest {
def upperLeftXBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftYBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftYBehaviors.scala
index d98824f3d..cc2668716 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftYBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_UpperLeftYBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_UpperLeftYBehaviors extends QueryTest {
def upperLeftYBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WidthBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WidthBehaviors.scala
index f3fd3c416..5fbaa5cb6 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WidthBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WidthBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_WidthBehaviors extends QueryTest {
def widthBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordBehaviors.scala
index baef5fa90..8b3e6916f 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_WorldToRasterCoordBehaviors extends QueryTest {
def worldToRasterCoordBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordXBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordXBehaviors.scala
index f9fd82dff..720db8adf 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordXBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordXBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_WorldToRasterCoordXBehaviors extends QueryTest {
def worldToRasterCoordXBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordYBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordYBehaviors.scala
index bffba09c8..ebbf251f2 100644
--- a/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordYBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/expressions/raster/RST_WorldToRasterCoordYBehaviors.scala
@@ -10,7 +10,7 @@ import org.scalatest.matchers.should.Matchers._
trait RST_WorldToRasterCoordYBehaviors extends QueryTest {
def worldToRasterCoordYBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
diff --git a/src/test/scala/com/databricks/labs/mosaic/functions/MosaicContextBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/functions/MosaicContextBehaviors.scala
index bbcf44e6a..6c4d73db7 100644
--- a/src/test/scala/com/databricks/labs/mosaic/functions/MosaicContextBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/functions/MosaicContextBehaviors.scala
@@ -24,7 +24,7 @@ trait MosaicContextBehaviors extends MosaicSpatialQueryTest {
def creationOfContext(mosaicContext: MosaicContext): Unit = {
val indexSystem = mosaicContext.getIndexSystem
val geometryAPI = mosaicContext.getGeometryAPI
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
MosaicContext.reset()
an[Error] should be thrownBy MosaicContext.context
val mc = MosaicContext.build(indexSystem, geometryAPI)
@@ -39,7 +39,7 @@ trait MosaicContextBehaviors extends MosaicSpatialQueryTest {
}
def sqlRegistration(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
val indexSystem = mc.getIndexSystem
import mc.functions._
diff --git a/src/test/scala/com/databricks/labs/mosaic/models/knn/GridRingNeighboursBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/models/knn/GridRingNeighboursBehaviors.scala
index 1f7107f98..fbfcd3cfc 100644
--- a/src/test/scala/com/databricks/labs/mosaic/models/knn/GridRingNeighboursBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/models/knn/GridRingNeighboursBehaviors.scala
@@ -11,7 +11,7 @@ import org.scalatest.matchers.should.Matchers._
trait GridRingNeighboursBehaviors extends MosaicSpatialQueryTest {
def leftTransform(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
@@ -81,7 +81,7 @@ trait GridRingNeighboursBehaviors extends MosaicSpatialQueryTest {
}
def resultTransform(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
@@ -150,7 +150,7 @@ trait GridRingNeighboursBehaviors extends MosaicSpatialQueryTest {
}
def transform(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val sc = spark
import sc.implicits._
diff --git a/src/test/scala/com/databricks/labs/mosaic/models/knn/SpatialKNNTest.scala b/src/test/scala/com/databricks/labs/mosaic/models/knn/SpatialKNNTest.scala
index e3096ba84..3108891e2 100644
--- a/src/test/scala/com/databricks/labs/mosaic/models/knn/SpatialKNNTest.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/models/knn/SpatialKNNTest.scala
@@ -21,7 +21,7 @@ class SpatialKNNTest extends AnyFlatSpec with SpatialKNNBehaviors with SparkSuit
.set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
.set("spark.sql.parquet.compression.codec", "uncompressed")
var spark = withConf(conf)
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
it should behave like noApproximation(MosaicContext.build(H3IndexSystem, JTS), spark)
conf = new SparkConf(false)
@@ -32,7 +32,7 @@ class SpatialKNNTest extends AnyFlatSpec with SpatialKNNBehaviors with SparkSuit
.set("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
.set("spark.sql.parquet.compression.codec", "uncompressed")
spark = withConf(conf)
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
it should behave like noApproximation(MosaicContext.build(BNGIndexSystem, JTS), spark)
}
diff --git a/src/test/scala/com/databricks/labs/mosaic/sql/AnalyzerBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/sql/AnalyzerBehaviors.scala
index 09342a5f0..177c22ee9 100644
--- a/src/test/scala/com/databricks/labs/mosaic/sql/AnalyzerBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/sql/AnalyzerBehaviors.scala
@@ -9,7 +9,7 @@ import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
trait AnalyzerBehaviors extends MosaicSpatialQueryTest {
def behavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/sql/PrettifierBehaviors.scala b/src/test/scala/com/databricks/labs/mosaic/sql/PrettifierBehaviors.scala
index fc930e8ce..973a63d9b 100644
--- a/src/test/scala/com/databricks/labs/mosaic/sql/PrettifierBehaviors.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/sql/PrettifierBehaviors.scala
@@ -7,7 +7,7 @@ import org.scalatest.matchers.must.Matchers.{be, noException}
trait PrettifierBehaviors extends MosaicSpatialQueryTest {
def behavior(mosaicContext: MosaicContext): Unit = {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
val mc = mosaicContext
mc.register(spark)
diff --git a/src/test/scala/com/databricks/labs/mosaic/sql/extensions/TestSQLExtensions.scala b/src/test/scala/com/databricks/labs/mosaic/sql/extensions/TestSQLExtensions.scala
index 55bdcf1e2..32a695cb5 100644
--- a/src/test/scala/com/databricks/labs/mosaic/sql/extensions/TestSQLExtensions.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/sql/extensions/TestSQLExtensions.scala
@@ -18,7 +18,7 @@ class TestSQLExtensions extends AnyFlatSpec with SQLExtensionsBehaviors with Spa
.set(MOSAIC_RASTER_API, "GDAL")
.set("spark.sql.extensions", "com.databricks.labs.mosaic.sql.extensions.MosaicSQL")
var spark = withConf(conf)
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
it should behave like sqlRegister(MosaicContext.build(H3IndexSystem, JTS), spark)
conf = new SparkConf(false)
@@ -27,7 +27,7 @@ class TestSQLExtensions extends AnyFlatSpec with SQLExtensionsBehaviors with Spa
.set(MOSAIC_RASTER_API, "GDAL")
.set("spark.sql.extensions", "com.databricks.labs.mosaic.sql.extensions.MosaicSQL")
spark = withConf(conf)
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
it should behave like sqlRegister(MosaicContext.build(BNGIndexSystem, JTS), spark)
conf = new SparkConf(false)
@@ -36,7 +36,7 @@ class TestSQLExtensions extends AnyFlatSpec with SQLExtensionsBehaviors with Spa
.set(MOSAIC_RASTER_API, "GDAL")
.set("spark.sql.extensions", "com.databricks.labs.mosaic.sql.extensions.MosaicSQL")
spark = withConf(conf)
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
it should behave like {
an[Error] should be thrownBy spark.sql("""show functions""").collect()
}
@@ -44,7 +44,7 @@ class TestSQLExtensions extends AnyFlatSpec with SQLExtensionsBehaviors with Spa
conf = new SparkConf(false)
.set("spark.sql.extensions", "com.databricks.labs.mosaic.sql.extensions.MosaicSQLDefault")
spark = withConf(conf)
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
it should behave like sqlRegister(MosaicContext.build(H3IndexSystem, JTS), spark)
}
@@ -56,7 +56,7 @@ class TestSQLExtensions extends AnyFlatSpec with SQLExtensionsBehaviors with Spa
.set(MOSAIC_GDAL_NATIVE, "true")
.set("spark.sql.extensions", "com.databricks.labs.mosaic.sql.extensions.MosaicGDAL")
val spark = withConf(conf)
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
it should behave like mosaicGDAL(MosaicContext.build(H3IndexSystem, JTS), spark)
}
diff --git a/src/test/scala/com/databricks/labs/mosaic/test/MosaicSpatialQueryTest.scala b/src/test/scala/com/databricks/labs/mosaic/test/MosaicSpatialQueryTest.scala
index d9e5fd534..fecbd158f 100644
--- a/src/test/scala/com/databricks/labs/mosaic/test/MosaicSpatialQueryTest.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/test/MosaicSpatialQueryTest.scala
@@ -50,7 +50,7 @@ abstract class MosaicSpatialQueryTest extends PlanTest with MosaicHelper {
SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.CODEGEN_ONLY.toString,
"spark.sql.parquet.compression.codec" -> "uncompressed"
) {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
withMosaicContext(geom, is) {
testFun
}
@@ -73,7 +73,7 @@ abstract class MosaicSpatialQueryTest extends PlanTest with MosaicHelper {
SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString,
"spark.sql.parquet.compression.codec" -> "uncompressed"
) {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
withMosaicContext(geom, is) {
testFun
}
@@ -96,7 +96,7 @@ abstract class MosaicSpatialQueryTest extends PlanTest with MosaicHelper {
SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.CODEGEN_ONLY.toString,
"spark.sql.parquet.compression.codec" -> "uncompressed"
) {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
withMosaicContext(geom, is) {
testFun
}
@@ -119,7 +119,7 @@ abstract class MosaicSpatialQueryTest extends PlanTest with MosaicHelper {
SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString,
"spark.sql.parquet.compression.codec" -> "uncompressed"
) {
- spark.sparkContext.setLogLevel("FATAL")
+ spark.sparkContext.setLogLevel("ERROR")
withMosaicContext(geom, is) {
testFun
}
diff --git a/src/test/scala/com/databricks/labs/mosaic/test/SparkSuite.scala b/src/test/scala/com/databricks/labs/mosaic/test/SparkSuite.scala
index 2346ad05f..568ba3710 100644
--- a/src/test/scala/com/databricks/labs/mosaic/test/SparkSuite.scala
+++ b/src/test/scala/com/databricks/labs/mosaic/test/SparkSuite.scala
@@ -42,7 +42,7 @@ trait SparkSuite extends TestSuite with BeforeAndAfterAll {
private def startSpark(): Unit = {
_sc = new SparkContext("local[4]", "test", sparkConf)
- _sc.setLogLevel("FATAL")
+ _sc.setLogLevel("ERROR")
_spark = SparkSession.builder.config(sc.getConf).getOrCreate()
}
diff --git a/src/test/scala/org/apache/spark/sql/test/SharedSparkSessionGDAL.scala b/src/test/scala/org/apache/spark/sql/test/SharedSparkSessionGDAL.scala
index 9aa152cee..95f71978f 100644
--- a/src/test/scala/org/apache/spark/sql/test/SharedSparkSessionGDAL.scala
+++ b/src/test/scala/org/apache/spark/sql/test/SharedSparkSessionGDAL.scala
@@ -2,7 +2,7 @@ package org.apache.spark.sql.test
import com.databricks.labs.mosaic.gdal.MosaicGDAL
import com.databricks.labs.mosaic.utils.FileUtils
-import com.databricks.labs.mosaic.{MOSAIC_GDAL_NATIVE, MOSAIC_RASTER_CHECKPOINT, MOSAIC_TEST_MODE}
+import com.databricks.labs.mosaic.{MOSAIC_GDAL_NATIVE, MOSAIC_RASTER_CHECKPOINT, MOSAIC_RASTER_USE_CHECKPOINT, MOSAIC_RASTER_USE_CHECKPOINT_DEFAULT, MOSAIC_TEST_MODE}
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.gdal.gdal.gdal
@@ -23,7 +23,7 @@ trait SharedSparkSessionGDAL extends SharedSparkSession {
conf.set(MOSAIC_RASTER_CHECKPOINT, FileUtils.createMosaicTempDir(prefix = "/mnt/"))
SparkSession.cleanupAnyExistingSession()
val session = new MosaicTestSparkSession(conf)
- session.sparkContext.setLogLevel("FATAL")
+ session.sparkContext.setLogLevel("ERROR")
Try {
MosaicGDAL.enableGDAL(session)
}
@@ -32,6 +32,7 @@ trait SharedSparkSessionGDAL extends SharedSparkSession {
override def beforeEach(): Unit = {
super.beforeEach()
+ sparkConf.set(MOSAIC_RASTER_USE_CHECKPOINT, MOSAIC_RASTER_USE_CHECKPOINT_DEFAULT)
MosaicGDAL.enableGDAL(this.spark)
gdal.AllRegister()
}