migrate /etc/noteable to /var/run/noteable with symlink

Taskfile.yaml

@@ -6,6 +6,10 @@ includes:
   rust: ./Taskfile.rust.yaml
   deno: ./Taskfile.deno.yaml
 
+vars:
+  NBL_HOME_DIR: /srv/noteable
+  NBL_WORK_DIR: /var/run/noteable
+
 tasks:
   copy-root-files:
     desc: Copy files from the root of the repository to the base build directories
@@ -32,11 +36,15 @@ tasks:
       - >-
         docker buildx build {{.LANGUAGE}}/{{.IDENTIFIER}}/{{.NBL_LANGUAGE_VERSION}} {{.CLI_ARGS}} \
           --build-arg "NBL_LANGUAGE_VERSION={{.NBL_LANGUAGE_VERSION}}" \
+          --build-arg "NBL_HOME_DIR={{.NBL_HOME_DIR}}" \
+          --build-arg "NBL_WORK_DIR={{.NBL_WORK_DIR}}" \
           --target "{{.BUILD_TARGET}}" \
           --tag "local/kernel-{{.LANGUAGE}}-{{.NBL_LANGUAGE_VERSION}}-{{.IDENTIFIER}}{{.TAG_SUFFIX}}:{{.TAG}}"
     requires:
       vars: [LANGUAGE, IDENTIFIER, NBL_LANGUAGE_VERSION]
     vars:
       BUILD_TARGET: '{{default "base" .BUILD_TARGET}}'
+      NBL_HOME_DIR: '{{default "/srv/noteable" .NBL_HOME_DIR}}'
+      NBL_WORK_DIR: '{{default "/var/run/noteable" .NBL_WORK_DIR}}'
       TAG_SUFFIX: '{{default "" .TAG_SUFFIX}}'
       TAG: '{{default "dev" .TAG}}'

deno/base/1.37/Dockerfile

@@ -6,25 +6,32 @@
 # ---
 # Deno is installed from a PR that adds Jupyter support
 # Rust 1.72 is required
-ARG NBL_DENO_VERSION="1.37-dev"
 FROM rust:1.72 as base
 
+# Application code lives in NBL_HOME_DIR, project data lives in NBL_WORK_DIR
+ARG NBL_HOME_DIR=/srv/noteable
+ARG NBL_WORK_DIR=/var/run/noteable
+
 # User/group setup
 USER root
 
 ENV NB_USER="noteable" \
-    NB_UID=4004 \
-    NB_GID=4004
+  NB_UID=4004 \
+  NB_GID=4004
 
 RUN groupadd --gid 4004 noteable && \
-    useradd --uid 4004 \
-    --shell /bin/false \
-    --create-home \
-    --no-log-init \
-    --gid noteable noteable \
-    --home-dir /srv/noteable && \
-    chown --recursive noteable:noteable /srv/noteable && \
-    mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable
+  useradd --uid 4004 \
+  --shell /bin/false \
+  --create-home \
+  --no-log-init \
+  --gid noteable noteable \
+  --home-dir "${NBL_HOME_DIR}" && \
+  chown --recursive noteable:noteable "${NBL_HOME_DIR}" && \
+  mkdir -p "${NBL_WORK_DIR}" && chown --recursive noteable:noteable "${NBL_WORK_DIR}"
+
+# TEMPORARY: symlink the previous /etc/noteable to the new NBL_WORK_DIR (/var/run/noteable) dir
+# until all other services are migrated and the old /etc/noteable can be removed
+RUN ln -s "${NBL_WORK_DIR}" /etc/noteable
 
 WORKDIR /tmp
 
@@ -36,14 +43,14 @@ USER noteable
 
 # use the deno installer and upgrade to the specified (canary) build
 # https://docs.deno.com/runtime/manual/references/contributing/release_schedule
-ENV DENO_INSTALL=/srv/noteable
+ENV DENO_INSTALL=${NBL_HOME_DIR}
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN curl -fsSL https://deno.land/x/install/install.sh | sh
 # deno installs to `DENO_INSTALL/bin`, so we'll add to PATH for follow-on deno commands
 ENV PATH="${DENO_INSTALL}/bin/:${PATH}"
 RUN deno upgrade \
-    --canary \
-    --version=48e695a2c89edad6e4880e7decfdb36d524f8279
+  --canary \
+  --version=48e695a2c89edad6e4880e7decfdb36d524f8279
 
 # the kernel needs the deno kernelspec discoverable locally before it can start
 # hadolint ignore=DL3059
@@ -52,9 +59,9 @@ RUN deno --unstable jupyter --install
 COPY secrets_helper.sh /tmp/secrets_helper.sh
 COPY run.sh /usr/local/bin
 
-ENV HOME="/srv/noteable"
+ENV HOME="${NBL_HOME_DIR}"
 
-WORKDIR /etc/noteable/project
+WORKDIR ${NBL_WORK_DIR}/project
 EXPOSE 50001-50005
 
 ENTRYPOINT ["tini", "-g", "--"]

python/base-gpu/3.10/Dockerfile

@@ -2,6 +2,9 @@
 # hadolint ignore=DL3006
 FROM base AS main
 
+# Application code lives in NBL_HOME_DIR, project data lives in NBL_WORK_DIR
+ARG NBL_HOME_DIR=/srv/noteable
+ARG NBL_WORK_DIR=/var/run/noteable
 ARG NBL_PYTHON_VERSION=3.10
 
 USER root
@@ -52,9 +55,9 @@ RUN /usr/bin/apt-install Aptfile
 COPY gpu.Aptfile .
 RUN /usr/bin/apt-install gpu.Aptfile
 
-ENV PATH="/srv/noteable/.local/bin:${PATH}" \
-  HOME="/srv/noteable" \
-  XDG_CACHE_HOME="/srv/noteable/.cache/" \
+ENV PATH="${NBL_HOME_DIR}/.local/bin:${PATH}" \
+  HOME="${NBL_HOME_DIR}" \
+  XDG_CACHE_HOME="${NBL_HOME_DIR}/.cache/" \
   GOOGLE_APPLICATION_CREDENTIALS="/vault/secrets/gcp-credentials"
 
 RUN chown -R noteable:noteable "${CONDA_DIR}"
@@ -80,4 +83,4 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_DIR/lib/:$CUDNN_PATH/lib \
 # Overwrite the base run.sh to include `mamba` usage
 COPY run.sh /usr/local/bin
 
-WORKDIR /etc/noteable/project
+WORKDIR ${NBL_WORK_DIR}/project

python/base-gpu/3.11/Dockerfile

@@ -2,6 +2,9 @@
 # hadolint ignore=DL3006
 FROM base AS main
 
+# Application code lives in NBL_HOME_DIR, project data lives in NBL_WORK_DIR
+ARG NBL_HOME_DIR=/srv/noteable
+ARG NBL_WORK_DIR=/var/run/noteable
 ARG NBL_PYTHON_VERSION=3.11
 
 USER root
@@ -52,9 +55,9 @@ RUN /usr/bin/apt-install Aptfile
 COPY gpu.Aptfile .
 RUN /usr/bin/apt-install gpu.Aptfile
 
-ENV PATH="/srv/noteable/.local/bin:${PATH}" \
-  HOME="/srv/noteable" \
-  XDG_CACHE_HOME="/srv/noteable/.cache/" \
+ENV PATH="${NBL_HOME_DIR}/.local/bin:${PATH}" \
+  HOME="${NBL_HOME_DIR}" \
+  XDG_CACHE_HOME="${NBL_HOME_DIR}/.cache/" \
   GOOGLE_APPLICATION_CREDENTIALS="/vault/secrets/gcp-credentials"
 
 RUN chown -R noteable:noteable "${CONDA_DIR}"
@@ -80,4 +83,4 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_DIR/lib/:$CUDNN_PATH/lib \
 # Overwrite the base run.sh to include `mamba` usage
 COPY run.sh /usr/local/bin
 
-WORKDIR /etc/noteable/project
+WORKDIR ${NBL_WORK_DIR}/project

python/base-gpu/3.9/Dockerfile

@@ -2,6 +2,9 @@
 # hadolint ignore=DL3006
 FROM base AS main
 
+# Application code lives in NBL_HOME_DIR, project data lives in NBL_WORK_DIR
+ARG NBL_HOME_DIR=/srv/noteable
+ARG NBL_WORK_DIR=/var/run/noteable
 ARG NBL_PYTHON_VERSION=3.9
 
 USER root
@@ -52,9 +55,9 @@ RUN /usr/bin/apt-install Aptfile
 COPY gpu.Aptfile .
 RUN /usr/bin/apt-install gpu.Aptfile
 
-ENV PATH="/srv/noteable/.local/bin:${PATH}" \
-  HOME="/srv/noteable" \
-  XDG_CACHE_HOME="/srv/noteable/.cache/" \
+ENV PATH="${NBL_HOME_DIR}/.local/bin:${PATH}" \
+  HOME="${NBL_HOME_DIR}" \
+  XDG_CACHE_HOME="${NBL_HOME_DIR}/.cache/" \
   GOOGLE_APPLICATION_CREDENTIALS="/vault/secrets/gcp-credentials"
 
 RUN chown -R noteable:noteable "${CONDA_DIR}"
@@ -80,4 +83,4 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_DIR/lib/:$CUDNN_PATH/lib \
 # Overwrite the base run.sh to include `mamba` usage
 COPY run.sh /usr/local/bin
 
-WORKDIR /etc/noteable/project
+WORKDIR ${NBL_WORK_DIR}/project

python/base/3.10/Dockerfile

@@ -7,6 +7,11 @@
 ARG NBL_PYTHON_VERSION=3.10
 FROM python:${NBL_PYTHON_VERSION}-slim-bullseye as base
 
+# Application code lives in NBL_HOME_DIR, project data lives in NBL_WORK_DIR
+ARG NBL_HOME_DIR=/srv/noteable
+ARG NBL_WORK_DIR=/var/run/noteable
+ARG NBL_PYTHON_VERSION
+
 # User/group setup
 USER root
 
@@ -20,11 +25,15 @@ RUN groupadd --gid 4004 noteable && \
   --create-home \
   --no-log-init \
   --gid noteable noteable \
-  --home-dir /srv/noteable && \
-  chown --recursive noteable:noteable /srv/noteable && \
+  --home-dir "${NBL_HOME_DIR}" && \
+  chown --recursive noteable:noteable "${NBL_HOME_DIR}" && \
+  mkdir -p "${NBL_WORK_DIR}" && chown --recursive noteable:noteable "${NBL_WORK_DIR}" && \
   mkdir /opt/venv && chown noteable:noteable /opt/venv && \
-  mkdir /etc/ipython && chown noteable:noteable /etc/ipython && \
-  mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable
+  mkdir /etc/ipython && chown noteable:noteable /etc/ipython
+
+# TEMPORARY: symlink the previous /etc/noteable to the new NBL_HOME_DIR (/var/run/noteable) dir
+# until all other services are migrated and the old /etc/noteable can be removed
+RUN ln -s "${NBL_WORK_DIR}" /etc/noteable
 
 WORKDIR /tmp
 
@@ -49,12 +58,12 @@ RUN pip install --no-cache-dir -r /tmp/base_requirements.txt
 COPY secrets_helper.sh /tmp/secrets_helper.sh
 COPY run.sh /usr/local/bin
 
-ENV HOME="/srv/noteable" \
-  PYTHONPATH="/srv/noteable/.local/lib/python${NBL_PYTHON_VERSION}/site-packages:${PYTHONPATH}" \
-  XDG_CACHE_HOME="/srv/noteable/.cache/" \
+ENV HOME="${NBL_HOME_DIR}" \
+  PYTHONPATH="${NBL_HOME_DIR}/.local/lib/python${NBL_PYTHON_VERSION}/site-packages:${PYTHONPATH}" \
+  XDG_CACHE_HOME="${NBL_HOME_DIR}/.cache/" \
   GOOGLE_APPLICATION_CREDENTIALS="/vault/secrets/gcp-credentials"
 
-WORKDIR /etc/noteable/project
+WORKDIR ${NBL_WORK_DIR}/project
 EXPOSE 50001-50005
 
 ENTRYPOINT ["/tini", "-g", "--"]

python/base/3.11/Dockerfile

@@ -7,6 +7,11 @@
 ARG NBL_PYTHON_VERSION=3.11
 FROM python:${NBL_PYTHON_VERSION}-slim-bullseye as base
 
+# Application code lives in NBL_HOME_DIR, project data lives in NBL_WORK_DIR
+ARG NBL_HOME_DIR=/srv/noteable
+ARG NBL_WORK_DIR=/var/run/noteable
+ARG NBL_PYTHON_VERSION
+
 # User/group setup
 USER root
 
@@ -20,11 +25,15 @@ RUN groupadd --gid 4004 noteable && \
   --create-home \
   --no-log-init \
   --gid noteable noteable \
-  --home-dir /srv/noteable && \
-  chown --recursive noteable:noteable /srv/noteable && \
+  --home-dir "${NBL_HOME_DIR}" && \
+  chown --recursive noteable:noteable "${NBL_HOME_DIR}" && \
+  mkdir -p "${NBL_WORK_DIR}" && chown --recursive noteable:noteable "${NBL_WORK_DIR}" && \
   mkdir /opt/venv && chown noteable:noteable /opt/venv && \
-  mkdir /etc/ipython && chown noteable:noteable /etc/ipython && \
-  mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable
+  mkdir /etc/ipython && chown noteable:noteable /etc/ipython
+
+# TEMPORARY: symlink the previous /etc/noteable to the new NBL_WORK_DIR (/var/run/noteable) dir
+# until all other services are migrated and the old /etc/noteable can be removed
+RUN ln -s "${NBL_WORK_DIR}" /etc/noteable
 
 WORKDIR /tmp
 
@@ -49,12 +58,12 @@ RUN pip install --no-cache-dir -r /tmp/base_requirements.txt
 COPY secrets_helper.sh /tmp/secrets_helper.sh
 COPY run.sh /usr/local/bin
 
-ENV HOME="/srv/noteable" \
-  PYTHONPATH="/srv/noteable/.local/lib/python${NBL_PYTHON_VERSION}/site-packages:${PYTHONPATH}" \
-  XDG_CACHE_HOME="/srv/noteable/.cache/" \
+ENV HOME="${NBL_HOME_DIR}" \
+  PYTHONPATH="${NBL_HOME_DIR}/.local/lib/python${NBL_PYTHON_VERSION}/site-packages:${PYTHONPATH}" \
+  XDG_CACHE_HOME="${NBL_HOME_DIR}/.cache/" \
   GOOGLE_APPLICATION_CREDENTIALS="/vault/secrets/gcp-credentials"
 
-WORKDIR /etc/noteable/project
+WORKDIR ${NBL_WORK_DIR}/project
 EXPOSE 50001-50005
 
 ENTRYPOINT ["/tini", "-g", "--"]

python/base/3.9/Dockerfile

@@ -7,6 +7,11 @@
 ARG NBL_PYTHON_VERSION=3.9
 FROM python:${NBL_PYTHON_VERSION}-slim-bullseye as base
 
+# Application code lives in NBL_HOME_DIR, project data lives in NBL_WORK_DIR
+ARG NBL_HOME_DIR=/srv/noteable
+ARG NBL_WORK_DIR=/var/run/noteable
+ARG NBL_PYTHON_VERSION
+
 # User/group setup
 USER root
 
@@ -20,11 +25,15 @@ RUN groupadd --gid 4004 noteable && \
   --create-home \
   --no-log-init \
   --gid noteable noteable \
-  --home-dir /srv/noteable && \
-  chown --recursive noteable:noteable /srv/noteable && \
+  --home-dir "${NBL_HOME_DIR}" && \
+  chown --recursive noteable:noteable "${NBL_HOME_DIR}" && \
+  mkdir -p "${NBL_WORK_DIR}" && chown --recursive noteable:noteable "${NBL_WORK_DIR}" && \
   mkdir /opt/venv && chown noteable:noteable /opt/venv && \
-  mkdir /etc/ipython && chown noteable:noteable /etc/ipython && \
-  mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable
+  mkdir /etc/ipython && chown noteable:noteable /etc/ipython
+
+# TEMPORARY: symlink the previous /etc/noteable to the new NBL_WORK_DIR (/var/run/noteable) dir
+# until all other services are migrated and the old /etc/noteable can be removed
+RUN ln -s "${NBL_WORK_DIR}" /etc/noteable
 
 WORKDIR /tmp
 
@@ -49,12 +58,12 @@ RUN pip install --no-cache-dir -r /tmp/base_requirements.txt
 COPY secrets_helper.sh /tmp/secrets_helper.sh
 COPY run.sh /usr/local/bin
 
-ENV HOME="/srv/noteable" \
-  PYTHONPATH="/srv/noteable/.local/lib/python${NBL_PYTHON_VERSION}/site-packages:${PYTHONPATH}" \
-  XDG_CACHE_HOME="/srv/noteable/.cache/" \
+ENV HOME="${NBL_HOME_DIR}" \
+  PYTHONPATH="${NBL_HOME_DIR}/.local/lib/python${NBL_PYTHON_VERSION}/site-packages:${PYTHONPATH}" \
+  XDG_CACHE_HOME="${NBL_HOME_DIR}/.cache/" \
   GOOGLE_APPLICATION_CREDENTIALS="/vault/secrets/gcp-credentials"
 
-WORKDIR /etc/noteable/project
+WORKDIR ${NBL_WORK_DIR}/project
 EXPOSE 50001-50005
 
 ENTRYPOINT ["/tini", "-g", "--"]

python/noteable/3.10/Dockerfile

@@ -6,6 +6,7 @@
 # hadolint ignore=DL3006
 FROM base as main
 
+ARG NBL_HOME_DIR=/srv/noteable
 ARG NBL_PYTHON_VERSION=3.10
 
 USER root
@@ -32,7 +33,7 @@ RUN pip install --no-cache-dir -r /tmp/noteable_requirements.txt
 # hadolint ignore=SC1008,DL3059
 RUN python -c "import dx, noteable, psutil, sidecar_comms"
 
-COPY .pythonrc /srv/noteable/.
+COPY .pythonrc "${NBL_HOME_DIR}/."
 COPY ipython_config.py /etc/ipython
 COPY git_credential_helper.py /git_credential_helper.py
 COPY git-wrapper.sh /usr/local/bin/git

python/noteable/3.9/Dockerfile

@@ -6,6 +6,7 @@
 # hadolint ignore=DL3006
 FROM base as main
 
+ARG NBL_HOME_DIR=/srv/noteable
 ARG NBL_PYTHON_VERSION=3.9
 
 USER root
@@ -32,7 +33,7 @@ RUN pip install --no-cache-dir -r /tmp/noteable_requirements.txt
 # hadolint ignore=DL3059
 RUN python -c "import dx, noteable, psutil, sidecar_comms"
 
-COPY .pythonrc /srv/noteable/.
+COPY .pythonrc "${NBL_HOME_DIR}/."
 COPY ipython_config.py /etc/ipython
 COPY git_credential_helper.py /git_credential_helper.py
 COPY git-wrapper.sh /usr/local/bin/git

python/noteable/ipython_config.py

@@ -6,7 +6,7 @@
 
 c.SqlMagic.feedback = False
 c.SqlMagic.autopandas = True
-c.NTBLMagic.project_dir = "/etc/noteable/project"
+c.NTBLMagic.project_dir = "/var/run/noteable/project"
 c.NoteableDataLoaderMagic.return_head = False
 c.IPythonKernel._execute_sleep = 0.15
 # 10 minutes to support large files