library handling.

CHANGELOG.md

@@ -1,6 +1,6 @@
 ## v0.4.3 [DBR 13.3 LTS]
+- Pyspark requirement removed from python setup.cfg as it is supplied by DBR
 - Python version limited to "<3.11,>=3.10" for DBR
-- Pyspark version limited to 3.4.1 for DBR
 - iPython dependency limited to "<8.11,>=7.4.2" for both DBR and keplergl-jupyter 
 - Expanded support for fuse-based checkpointing (persisted raster storage), managed through:
   - spark config 'spark.databricks.labs.mosaic.raster.use.checkpoint' in addition to 'spark.databricks.labs.mosaic.raster.checkpoint'.

python/mosaic/api/__init__.py

@@ -1,7 +1,7 @@
 from .accessors import *
 from .aggregators import *
 from .constructors import *
-from .enable import enable_mosaic
+from .enable import enable_mosaic, get_install_version, get_install_lib_dir
 from .functions import *
 from .fuse import *
 from .predicates import *

python/mosaic/api/enable.py

@@ -1,3 +1,5 @@
+import importlib.metadata
+import importlib.resources
 import warnings
 
 from IPython.core.getipython import get_ipython
@@ -82,9 +84,10 @@ def enable_mosaic(
         config.log_info=True
 
     # Config global objects
+    # - add MosaicContext after MosaicLibraryHandler
     config.mosaic_spark = spark
-    config.mosaic_context = MosaicContext(spark)
     _ = MosaicLibraryHandler(spark, log_info=log_info)
+    config.mosaic_context = MosaicContext(spark)
     config.mosaic_context.jRegister(spark)
 
     _jcontext = config.mosaic_context.jContext()
@@ -106,6 +109,26 @@ def enable_mosaic(
         config.ipython_hook.register_magics(MosaicKepler)
 
 
+def get_install_version() -> str:
+    """
+    :return: mosaic version installed
+    """
+    return importlib.metadata.version("databricks-mosaic")
+
+
+def get_install_lib_dir(override_jar_filename=None) -> str:
+    """
+    This is looking for the library dir under site packages using the jar name.
+    :return: located library dir.
+    """
+    v = get_install_version()
+    jar_filename = f"mosaic-{v}-jar-with-dependencies.jar"
+    if override_jar_filename:
+        jar_filename = override_jar_filename
+    with importlib.resources.path("mosaic.lib", jar_filename) as p:
+        return p.parent.as_posix()
+
+
 def refresh_context():
     """
     Refresh mosaic context, using previously configured information.

python/setup.cfg

@@ -22,7 +22,6 @@ install_requires =
     h3<4.0,>=3.7
     ipython<8.11,>=7.4.2
     keplergl==0.3.2
-    pyspark==3.4.1
 
 [options.package_data]
 mosaic =