Merge pull request #13 from minrk/jupyter-fs

enable jupyter-fs

jupyterhub/gfts-hub/Chart.lock

@@ -1,7 +1,7 @@
 dependencies:
 - name: jupyterhub
   repository: https://jupyterhub.github.io/helm-chart
-  version: 3.3.2
+  version: 3.3.6
 - name: dask-gateway
   repository: https://helm.dask.org/
   version: 2024.1.0
@@ -14,5 +14,5 @@ dependencies:
 - name: grafana
   repository: https://grafana.github.io/helm-charts
   version: 7.0.14
-digest: sha256:0e76398b9b637b1a642450e916c5b13b1a259d5c1bdb746fb06a13b528177c83
-generated: "2024-03-21T13:17:37.982155+01:00"
+digest: sha256:f65f939aed209d2ebec4ea4534deef0d14d8b99ee1c5ee20ddee11d0a7115b20
+generated: "2024-04-03T14:18:11.909303+02:00"

jupyterhub/gfts-hub/Chart.yaml

@@ -6,7 +6,7 @@ kubeVersion: ">= 1.24.0-0"
 dependencies:
   # jupyterhub
   - name: jupyterhub
-    version: "3.3.2"
+    version: "3.3.6"
     repository: https://jupyterhub.github.io/helm-chart
 
   # dask-gateway

jupyterhub/images/user/Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/pangeo/pangeo-notebook:2024.01.23
+FROM quay.io/pangeo/pangeo-notebook:2024.03.22
 
 # install some extra things with requirements.txt
 COPY requirements.txt /tmp/requirements.txt

jupyterhub/images/user/jupyter_server_config.py

@@ -1,4 +1,12 @@
 import os
+from configparser import ConfigParser
+from pathlib import Path
+from urllib.parse import urlparse
+
+import fs.errors
+import fs.opener
+from fs_s3fs import S3FS
+from fs_s3fs.opener import S3FSOpener
 
 c = get_config()  # noqa
 
@@ -20,3 +28,74 @@
     c.MappingKernelManager.cull_connected = True
 
 c.ContentsManager.hide_globs.extend(["lost+found"])
+
+
+# workaround https://github.com/PyFilesystem/s3fs/issues/70
+# because our files weren't created with S3FS (aka fs-s3fs)
+# they were created with s3fs. Ha!
+class EnsureDirS3FS(S3FS):
+    def getinfo(self, path, *args, **kwargs):
+        try:
+            return super().getinfo(path, *args, **kwargs)
+        except fs.errors.ResourceNotFound as e:
+            # check if getinfo failed becuase it's a directory
+            # without an empty Object
+            # if so, create it
+            # S3FS.scandir and S3FS.getinfo don't work on missing directories
+            # but S3FS.listdir does
+            try:
+                self.listdir(path)
+            except fs.errors.ResourceNotFound:
+                # it actually doesn't exist, raise original error
+                raise e from None
+            else:
+                # it's a directory but the empty directory object doesn't exist
+                # create it then call getinfo
+                print(f"Making empty directory {path}")
+                self.makedir(path)
+                return super().getinfo(path, *args, **kwargs)
+
+
+# define custom opener for GFTS
+# loads GFTS S3 credentials from .aws/credentials [gfts] profile
+
+
+class GFTSOpener(S3FSOpener):
+    protocols = ["gfts-s3"]
+
+    def open_fs(self, fs_url, parse_result, *args, **kwargs):
+        bucket_name, _, dir_path = parse_result.resource.partition("/")
+        creds = ConfigParser()
+        creds.read(Path.home() / ".aws/credentials")
+        endpoint_url = creds.get(
+            "gfts",
+            "aws_endpoint_url",
+            fallback="https://s3.gra.perf.cloud.ovh.net",
+        )
+        # get 'gra' from 's3.gra.perf....'
+        region_name = urlparse(endpoint_url).hostname.split(".")[1]
+        return EnsureDirS3FS(
+            bucket_name,
+            dir_path=dir_path or "/",
+            aws_access_key_id=creds.get("gfts", "aws_access_key_id", fallback=None),
+            aws_secret_access_key=creds.get(
+                "gfts", "aws_secret_access_key", fallback=None
+            ),
+            endpoint_url=endpoint_url,
+            region=region_name,
+            acl=parse_result.params.get("acl", None),
+            cache_control=parse_result.params.get("cache_control", None),
+        )
+
+
+fs.opener.registry.install(GFTSOpener)
+
+
+c.JupyterFs.resources = [
+    {
+        "name": "gfts-data-lake",
+        "url": "gfts-s3://destine-gfts-data-lake/",
+    },
+]
+
+c.ServerApp.contents_manager_class = "jupyterfs.metamanager.MetaManager"

jupyterhub/images/user/requirements.txt

@@ -1 +1,2 @@
-jupyterhub==4.1.0
+jupyter-fs
+jupyterhub==4.1.4