diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b25672d0..3a9ff6687 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## v0.4.3 [DBR 13.3 LTS] +- Pyspark requirement removed from python setup.cfg as it is supplied by DBR - Python version limited to "<3.11,>=3.10" for DBR -- Pyspark version limited to 3.4.1 for DBR - iPython dependency limited to "<8.11,>=7.4.2" for both DBR and keplergl-jupyter - Expanded support for fuse-based checkpointing (persisted raster storage), managed through: - spark config 'spark.databricks.labs.mosaic.raster.use.checkpoint' in addition to 'spark.databricks.labs.mosaic.raster.checkpoint'. diff --git a/python/mosaic/api/__init__.py b/python/mosaic/api/__init__.py index 6377554d3..9c8ce7a81 100644 --- a/python/mosaic/api/__init__.py +++ b/python/mosaic/api/__init__.py @@ -1,7 +1,7 @@ from .accessors import * from .aggregators import * from .constructors import * -from .enable import enable_mosaic +from .enable import enable_mosaic, get_install_version, get_install_lib_dir from .functions import * from .fuse import * from .predicates import * diff --git a/python/mosaic/api/enable.py b/python/mosaic/api/enable.py index b2187b9ae..e7e66f974 100644 --- a/python/mosaic/api/enable.py +++ b/python/mosaic/api/enable.py @@ -1,3 +1,5 @@ +import importlib.metadata +import importlib.resources import warnings from IPython.core.getipython import get_ipython @@ -82,9 +84,10 @@ def enable_mosaic( config.log_info=True # Config global objects + # - add MosaicContext after MosaicLibraryHandler config.mosaic_spark = spark - config.mosaic_context = MosaicContext(spark) _ = MosaicLibraryHandler(spark, log_info=log_info) + config.mosaic_context = MosaicContext(spark) config.mosaic_context.jRegister(spark) _jcontext = config.mosaic_context.jContext() @@ -106,6 +109,26 @@ def enable_mosaic( config.ipython_hook.register_magics(MosaicKepler) +def get_install_version() -> str: + """ + :return: mosaic version installed + """ + return importlib.metadata.version("databricks-mosaic") + + +def get_install_lib_dir(override_jar_filename=None) -> str: + """ + This is looking for the library dir under site packages using the jar name. + :return: located library dir. + """ + v = get_install_version() + jar_filename = f"mosaic-{v}-jar-with-dependencies.jar" + if override_jar_filename: + jar_filename = override_jar_filename + with importlib.resources.path("mosaic.lib", jar_filename) as p: + return p.parent.as_posix() + + def refresh_context(): """ Refresh mosaic context, using previously configured information. diff --git a/python/setup.cfg b/python/setup.cfg index 6b4cb7de8..4ffd3dc94 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -22,7 +22,6 @@ install_requires = h3<4.0,>=3.7 ipython<8.11,>=7.4.2 keplergl==0.3.2 - pyspark==3.4.1 [options.package_data] mosaic =