Merge branch 'master' into grpcio-1-62-2

Granulate · Oct 8, 2024 · aadebc3 · aadebc3
2 parents 5866ef2 + 1cb5b4a
commit aadebc3
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 1 deletion.
diff --git a/granulate_utils/metadata/bigdata/bigdatainfo.py b/granulate_utils/metadata/bigdata/bigdatainfo.py
@@ -1,7 +1,7 @@
 from typing import Optional
 
 from granulate_utils.metadata.bigdata.cloudera import get_cloudera_version
-from granulate_utils.metadata.bigdata.databricks import get_databricks_version
+from granulate_utils.metadata.bigdata.databricks import get_databricks_version, is_databricks
 from granulate_utils.metadata.bigdata.dataproc import get_dataproc_version
 from granulate_utils.metadata.bigdata.emr import get_emr_version
 from granulate_utils.metadata.bigdata.interfaces import BigDataInfo
@@ -15,6 +15,8 @@ def get_bigdata_info() -> Optional[BigDataInfo]:
         return BigDataInfo("emr", emr_version)
     elif databricks_version := get_databricks_version():
         return BigDataInfo("databricks", databricks_version)
+    elif is_databricks():
+        return BigDataInfo("databricks", "unknown")
     elif dataproc_version := get_dataproc_version():
         return BigDataInfo("dataproc", dataproc_version)
     elif cloudera_version := get_cloudera_version():

diff --git a/granulate_utils/metadata/bigdata/databricks.py b/granulate_utils/metadata/bigdata/databricks.py
@@ -1,11 +1,14 @@
 import logging
+import os
 from typing import TYPE_CHECKING, Optional, Union
 
 if TYPE_CHECKING:
     _LoggerAdapter = logging.LoggerAdapter[logging.Logger]
 else:
     _LoggerAdapter = logging.LoggerAdapter
 
+DATABRICKS_METRICS_PROP_PATH = "/databricks/spark/conf/metrics.properties"
+
 
 def get_databricks_version() -> Optional[str]:
     try:
@@ -15,6 +18,13 @@ def get_databricks_version() -> Optional[str]:
         return None
 
 
+def is_databricks() -> bool:
+    """
+    In some Databricks versions / images, /databricks/DBR_VERSION is missing but this file exists.
+    """
+    return os.path.exists(DATABRICKS_METRICS_PROP_PATH)
+
+
 def get_hadoop_version(logger: Optional[Union[logging.Logger, _LoggerAdapter]]) -> Optional[str]:
     try:
         with open("/databricks/spark/HADOOP_VERSION", "r") as f: