Merge pull request #25 from umccr/feature/databricks-hail-runtime

Bump spark to 3.4.1, install marksupsafe, install into right env
umccr · Dec 3, 2023 · f2dc417 · f2dc417
2 parents 7e39548 + 6b88ab1
commit f2dc417
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 9 deletions.
diff --git a/...ark-3.4.0-patch/0001-spark-to-3.4.0.patch → ...ark-3.4.1-patch/0001-spark-to-3.4.1.patch b/...ark-3.4.0-patch/0001-spark-to-3.4.0.patch → ...ark-3.4.1-patch/0001-spark-to-3.4.1.patch
@@ -1,7 +1,7 @@
 From ff5aa8bd5ae37d64ad9e37a45f76f6fcaced01c1 Mon Sep 17 00:00:00 2001
 From: Alexis Lucattini <[email protected]>
 Date: Fri, 1 Dec 2023 04:48:08 +0000
-Subject: [PATCH] spark-to-3.4.0
+Subject: [PATCH] spark-to-3.4.1
 
 ---
  hail/build.gradle                             | 2 +-
@@ -31,7 +31,7 @@ index 9f8492a2a..a4c11f8ea 100644
  plotly>=5.5.0,<6
  protobuf==3.20.2
 -pyspark>=3.3.0,<3.4
-+pyspark==3.4.0,<3.5
++pyspark==3.4.1,<3.5
  requests>=2.25.1,<3
  scipy>1.2,<1.12
 diff --git a/hail/src/main/scala/is/hail/HailContext.scala b/hail/src/main/scala/is/hail/HailContext.scala

diff --git a/...ail/0.2.126--spark-3.4.0-patch/Dockerfile → ...ail/0.2.126--spark-3.4.1-patch/Dockerfile b/...ail/0.2.126--spark-3.4.0-patch/Dockerfile → ...ail/0.2.126--spark-3.4.1-patch/Dockerfile
@@ -2,7 +2,7 @@ FROM databricksruntime/python:13.3-LTS
 
 ARG HAIL_COMMIT_ID="5b718e1"  # https://github.com/hail-is/hail/commit/5b718e1d3e1d72fad15cc57801601a6e35df41dd
 ARG SCALA_VERSION="2.12.0"
-ARG SPARK_VERSION="3.4.0"
+ARG SPARK_VERSION="3.4.1"
 ARG BREEZE_VERSION="2.1.0"
 
 RUN apt update -y && \
@@ -15,23 +15,26 @@ RUN apt update -y && \
       liblapack3 \
       git \
       rsync \
-      liblz4-dev && \ 
-      pip3 install ipython
+      liblz4-dev 
 
-COPY 0001-spark-to-3.4.0.patch /root/0001-spark-to-3.4.0.patch 
+COPY 0001-spark-to-3.4.1.patch /root/0001-spark-to-3.4.1.patch 
 
 RUN \
     git clone "https://github.com/hail-is/hail.git" && \
     ( \
       cd hail && \
       git checkout "${HAIL_COMMIT_ID}" && \
-      git apply /root/0001-spark-to-3.4.0.patch && \
+      git apply /root/0001-spark-to-3.4.1.patch && \
       ( \
         cd hail/ && \
+        . /databricks/python3/bin/activate && \
         make install \
           HAIL_COMPILE_NATIVES=1 \
           SCALA_VERSION="${SCALA_VERSION}" \
-          SPARK_VERSION="${SPARK_VERSION}"\
+          SPARK_VERSION="${SPARK_VERSION}"\ && \
+        pip install markupsafe==2.0.1 \
       ) \
     ) && \
-    rm /root/0001-spark-to-3.4.0.patch
+    rm -rf hail && \
+    rm /root/0001-spark-to-3.4.1.patch
+
diff --git a/...hail/0.2.126--spark-3.4.0-patch/Readme.md → ...hail/0.2.126--spark-3.4.1-patch/Readme.md b/...hail/0.2.126--spark-3.4.0-patch/Readme.md → ...hail/0.2.126--spark-3.4.1-patch/Readme.md