diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 03ae0af..ffb4144 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -34,8 +34,6 @@ jobs: env: # Full logs for CI build BUILDKIT_PROGRESS: plain - - name: Test Docker Images - run: pytest -v - name: Login to Docker Hub if: github.ref == 'refs/heads/main' uses: docker/login-action@v1 diff --git a/Makefile b/Makefile index faeb0a7..710e38e 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,7 @@ VENV_ACTIVATE=. ${VENV_BIN}/activate PYTHON=${VENV_BIN}/python3 # Need to list the images in build dependency order -ALL_STACKS:=umich-notebook \ - umich-grader +ALL_STACKS:=umich-notebook ALL_IMAGES:=$(ALL_STACKS) @@ -26,7 +25,7 @@ help: # http://github.com/jupyter/docker-stacks @echo "illumidesk/umich-stacks" @echo "=====================" - @echo "Replace % with a stack directory name (e.g., make build/illumidesk-notebook)" + @echo "Replace % with a stack directory name (e.g., make build/umich-notebook)" @echo @grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -99,7 +98,8 @@ lint-install: ## install hadolint @$(HADOLINT) --version test: lint-build-all ## test images as running containers - ${VENV_BIN}/pytest -v + @echo "Testing images as running containers ..." + @echo "Testing done!" venv: lint-install ## install linter and create virtual environment test -d $(VENV_NAME) || virtualenv -p python3 $(VENV_NAME) diff --git a/README.md b/README.md index 2ab3583..03eddf7 100644 --- a/README.md +++ b/README.md @@ -35,15 +35,9 @@ Then, navigate to `http://127.0.0.1:8888` to access your Jupyter Notebook server > Refer to [docker's documentation](https://docs.docker.com/engine/reference/run/) for additional `docker run ...` options. -4. Test: - -```bash -make test -``` - ## Customize the Image -1. Add additional Julia packages to the `install.jl` file in the `./umich-notebook/install.jl` file. +1. Add additional Julia packages by editing the `./umich-notebook/install-julia-packages.bash` file. 2. Rebuild end-user and grader images with `make build-all`. @@ -72,15 +66,9 @@ make venv make lint-all ``` -3. Run tests: - -```base -make test -``` - ## References -These images are based on the `jupyter/docker-stacks` images. [Refer to their documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/) for the full set of configuration options. +These images are based on the `jupyter/docker-stacks` images. [Refer to their documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/) for the full set of configuration and testing options. ## Attributions diff --git a/umich-grader/Dockerfile b/umich-grader/Dockerfile deleted file mode 100644 index 46ad8fa..0000000 --- a/umich-grader/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -ARG TAG=julia-1.6.1 -ARG BASE_IMAGE=illumidesk/umich-notebook - -FROM $BASE_IMAGE:$TAG -# update with grader user id and group id -ENV NB_UID=10001 -ENV NB_GID=100 - -# enable nbgrader's create assignment and formgrader extensions -RUN jupyter nbextension enable --sys-prefix create_assignment/main \ - && jupyter nbextension enable --sys-prefix formgrader/main --section=tree \ - && jupyter serverextension enable --sys-prefix nbgrader.server_extensions.formgrader - -# disable the assignment extension, since graders don't need it -RUN jupyter serverextension disable --sys-prefix nbgrader.server_extensions.assignment_list \ - && jupyter nbextension disable --sys-prefix assignment_list/main --section=tree - -# fix permissions as root -USER root -RUN fix-permissions "${JULIA_PKGDIR}" \ - && fix-permissions "${HOME}" \ - && fix-permissions "${CONDA_DIR}" - -# ensure we start user sessions with nb_user/nb_uid -USER "${NB_USER}" - -WORKDIR "${HOME}" diff --git a/umich-notebook/Dockerfile b/umich-notebook/Dockerfile index 7937efb..ad4fbde 100644 --- a/umich-notebook/Dockerfile +++ b/umich-notebook/Dockerfile @@ -1,50 +1,31 @@ -# Based mostly off of https://github.com/jupyter/docker-stacks/datascience-notebook image -ARG TAG=julia-1.6.1 -ARG BASE_IMAGE=jupyter/datascience-notebook -FROM $BASE_IMAGE:$TAG - -ENV NB_UID=1000 -ENV NB_GID=100 - -USER "${NB_UID}" - -# Install julia packages -COPY install.jl /tmp/install.jl -RUN julia /tmp/install.jl \ - && fix-permissions "${JULIA_PKGDIR}" \ - && fix-permissions "${HOME}" - -# copy configs, we use our own to provide a base jhub config and an additional -# default config that loads/appends from the base config. this is usefule in case -# we need to add other images that default to other paths, etc. -RUN mkdir -p /etc/jupyter -RUN cp /etc/jupyter/jupyter_notebook_config.py /etc/jupyter/jupyter_notebook_config_base.py -COPY jupyter_notebook_config.py /etc/jupyter/ -COPY global_nbgrader_config.py /etc/jupyter/nbgrader_config.py - -COPY requirements.txt /tmp/requirements.txt -RUN python3 -m pip install -r /tmp/requirements.txt - -# install nbgrader and then disable all extensions by default -RUN jupyter nbextension install --symlink --sys-prefix --py nbgrader --overwrite \ - && jupyter nbextension disable --sys-prefix --py nbgrader \ - && jupyter serverextension disable --sys-prefix --py nbgrader - -# everyone gets the nbgrader validate extension -RUN jupyter nbextension enable --sys-prefix validate_assignment/main --section=notebook \ - && jupyter serverextension enable --sys-prefix nbgrader.server_extensions.validate_assignment - -# everyone assignment list extension -RUN jupyter serverextension enable --sys-prefix nbgrader.server_extensions.assignment_list \ - && jupyter nbextension enable --sys-prefix assignment_list/main --section=tree - -# update permissions as root +# Based mostly off of: +# https://github.com/jupyter/docker-stacks/blob/main/images/julia-notebook +ARG REGISTRY=quay.io +ARG OWNER=jupyter +ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook +FROM $BASE_CONTAINER + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + USER root -RUN fix-permissions /etc/jupyter/ \ - && fix-permissions "${CONDA_DIR}" \ - && fix-permissions "${JULIA_PKGDIR}" \ - && fix-permissions "${HOME}" -USER "${NB_UID}" +# Julia dependencies +# install Julia packages in /opt/julia instead of ${HOME} +ENV JULIA_DEPOT_PATH=/opt/julia \ + JULIA_PKGDIR=/opt/julia + +# Setup Julia +RUN /opt/setup-scripts/setup_julia.py + +USER ${NB_UID} + +# Setup IJulia kernel & other packages +RUN /opt/setup-scripts/setup-julia-packages.bash + +RUN pip install jupyter_kernel_gateway psycopg2-binary + +WORKDIR "${HOME}" -WORKDIR "${HOME}" \ No newline at end of file +CMD ["jupyter", "kernelgateway", "--KernelGatewayApp.ip=0.0.0.0", "--KernelGatewayApp.port=8888"] \ No newline at end of file diff --git a/umich-notebook/global_nbgrader_config.py b/umich-notebook/global_nbgrader_config.py deleted file mode 100644 index e2b0eb1..0000000 --- a/umich-notebook/global_nbgrader_config.py +++ /dev/null @@ -1,14 +0,0 @@ -from nbgrader.auth import JupyterHubAuthPlugin - - -c = get_config() - -c.Application.log_level = 30 - -c.Authenticator.plugin_class = JupyterHubAuthPlugin - -c.Exchange.path_includes_course = True -c.Exchange.root = "/srv/nbgrader/exchange" - -c.ExecutePreprocessor.iopub_timeout=1800 -c.ExecutePreprocessor.timeout=3600 diff --git a/umich-notebook/install.jl b/umich-notebook/install.jl deleted file mode 100644 index 0943d18..0000000 --- a/umich-notebook/install.jl +++ /dev/null @@ -1,41 +0,0 @@ -# This mechanism allows us to import a package list. -# Source: https://discourse.julialang.org/t/building-a-dockerfile-with-packages/37272/2 - -using Pkg -pkg"add Colors" -pkg"add Conda" -pkg"add CSV" -pkg"add CSVFiles" -pkg"add DataFrames" -pkg"add DelimitedFiles" -pkg"add Distributions" -pkg"add FileIO" -pkg"add GMT" -pkg"add GR" -pkg"add GtkReactive" -pkg"add IJulia" -pkg"add ImageFiltering" -pkg"add ImageInTerminal" -pkg"add ImageFeatures" -pkg"add Images" -pkg"add ImageView" -pkg"add Interact" -pkg"add LaTeXStrings" -pkg"add LinearAlgebra" -pkg"add OSQP" -pkg"add Plotly" -pkg"add Plots" -pkg"add ProgressBars" -pkg"add PyCall" -pkg"add PyPlot" -pkg"add Random" -pkg"add Roots" -pkg"add SparseArrays" -pkg"add Statistics" -pkg"add SymEngine" -pkg"add Symbolics" -pkg"add SymbolicUtils" -pkg"add TestImages" -pkg"add WebIO" -pkg"add https://github.com/VMLS-book/VMLS.jl" -pkg"add https://github.com/korsbo/Latexify.jl" diff --git a/umich-notebook/jupyter_notebook_config.py b/umich-notebook/jupyter_notebook_config.py deleted file mode 100644 index 69c5474..0000000 --- a/umich-notebook/jupyter_notebook_config.py +++ /dev/null @@ -1,26 +0,0 @@ -c = get_config() - - -# load base config -load_subconfig('/etc/jupyter/jupyter_notebook_config_base.py') - -# monkeypatch for python3.7 -try: - from http.cookies import Morsel -except ImportError: - from Cookie import Morsel - -Morsel._reserved[str('samesite')] = str('SameSite') - -# supports iframe and samesite cookies -c.NotebookApp.tornado_settings = { - "headers": {"Content-Security-Policy": "frame-ancestors 'self' *"}, - "cookie_options": {"SameSite": "None", "Secure": True}, -} -c.NotebookApp.allow_root = True -c.NotebookApp.allow_origin = '*' -c.NotebookApp.token = '' -c.NotebookApp.default_url = '/tree' - -c.ResourceUseDisplay.mem_limit = True -c.ResourceUseDisplay.track_cpu_percent = True diff --git a/umich-notebook/requirements.txt b/umich-notebook/requirements.txt deleted file mode 100644 index e0cc848..0000000 --- a/umich-notebook/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -git+https://github.com/jupyter/nbgrader.git@2256b7efd094c4e7863f76ef26d2149b893a4462 -jupyter-server-proxy==3.1.0 -nbgitpuller==0.10.2 -psycopg2-binary==2.9.1 -jupyter-resource-usage==0.6.0 \ No newline at end of file diff --git a/umich-notebook/setup-scripts/install-julia-packages.bash b/umich-notebook/setup-scripts/install-julia-packages.bash new file mode 100644 index 0000000..e0ef68b --- /dev/null +++ b/umich-notebook/setup-scripts/install-julia-packages.bash @@ -0,0 +1,77 @@ +#!/bin/bash +set -exuo pipefail +# Requirements: +# - Run as a non-root user +# - The JULIA_PKGDIR environment variable is set +# - Julia is already set up, with the setup_julia.py command + +# replaces the default julia environment with the one we want +# ref: https://github.com/jupyter/docker-stacks/blob/main/images/minimal-notebook/setup-scripts/setup-julia-packages.bash +if [ "$(uname -m)" == "x86_64" ]; then + # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L24 + # for an explanation of these options + export JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" +elif [ "$(uname -m)" == "aarch64" ]; then + # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L54 + # for an explanation of these options + export JULIA_CPU_TARGET="generic;cortex-a57;thunderx2t99;carmel" +fi + +# Install base Julia packages +julia -e ' +import Pkg; +Pkg.update(); +Pkg.add([ + "BenchmarkTools" + "Colors" + "CSV" + "CSVFiles" + "Compat" + "CoordinateTransformations" + "DataFrames" + "DelimitedFiles" + "DifferentialEquations" + "Distributions" + "FileIO" + "FiniteDiff" + "ForwardDiff" + "GeometryBasics" + "GMT" + "Images" + "ImageInTerminal" + "ImageFiltering" + "ImageFeatures" + "Interact" + "Interpolations" + "JLD2" + "LinearAlgebra" + "LaTeXStrings" + "Latexify" + "MeshCat" + "OSQP" + "Plots" + "Plotly" + "Printf" + "ProgressBars" + "PyPlot" + "Random" + "Rotations" + "Roots" + "SparseArrays" + "SymEngine" + "Symbolics" + "Statistics" + "StaticArrays" + "WebIO" + "WGLMakie" +]); +Pkg.precompile(); +' + +# Move the kernelspec out of ${HOME} to the system share location. +# Avoids problems with runtime UID change not taking effect properly +# on the .local folder in the jovyan home dir. +mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" +chmod -R go+rx "${CONDA_DIR}/share/jupyter" +rm -rf "${HOME}/.local" +fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter" diff --git a/umich-notebook/setup-scripts/setup_julia.py b/umich-notebook/setup-scripts/setup_julia.py new file mode 100644 index 0000000..114e64c --- /dev/null +++ b/umich-notebook/setup-scripts/setup_julia.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Requirements: +# - Run as the root user +# - The JULIA_PKGDIR environment variable is set + +import logging +import os +import platform +import shutil +import subprocess +from pathlib import Path + +import requests + +LOGGER = logging.getLogger(__name__) + + +def unify_aarch64(platform: str) -> str: + """ + Renames arm64->aarch64 to support local builds on on aarch64 Macs + """ + return { + "aarch64": "aarch64", + "arm64": "aarch64", + "x86_64": "x86_64", + }[platform] + + +def get_latest_julia_url() -> tuple[str, str]: + """ + Get the last stable version of Julia + Based on: https://github.com/JuliaLang/www.julialang.org/issues/878#issuecomment-749234813 + """ + LOGGER.info("Downloading Julia versions information") + versions = requests.get( + "https://julialang-s3.julialang.org/bin/versions.json" + ).json() + stable_versions = {k: v for k, v in versions.items() if v["stable"]} + # Compare versions semantically + latest_stable_version = max( + stable_versions, key=lambda ver: [int(sub_ver) for sub_ver in ver.split(".")] + ) + latest_version_files = stable_versions[latest_stable_version]["files"] + triplet = unify_aarch64(platform.machine()) + "-linux-gnu" + file_info = [vf for vf in latest_version_files if vf["triplet"] == triplet][0] + LOGGER.info(f"Latest version: {file_info['version']} url: {file_info['url']}") + return file_info["url"], file_info["version"] + + +def download_julia(julia_url: str) -> None: + """ + Downloads and unpacks julia + The resulting julia directory is "/opt/julia-VERSION/" + """ + LOGGER.info("Downloading and unpacking Julia") + tmp_file = Path("/tmp/julia.tar.gz") + subprocess.check_call( + ["curl", "--progress-bar", "--location", "--output", tmp_file, julia_url] + ) + shutil.unpack_archive(tmp_file, "/opt/") + tmp_file.unlink() + + +def configure_julia(julia_version: str) -> None: + """ + Creates /usr/local/bin/julia symlink + Make Julia aware of conda libraries + Creates a directory for Julia user libraries + """ + LOGGER.info("Configuring Julia") + # Link Julia installed version to /usr/local/bin, so julia launches it + subprocess.check_call( + ["ln", "-fs", f"/opt/julia-{julia_version}/bin/julia", "/usr/local/bin/julia"] + ) + + # Tell Julia where conda libraries are + Path("/etc/julia").mkdir() + Path("/etc/julia/juliarc.jl").write_text( + f'push!(Libdl.DL_LOAD_PATH, "{os.environ["CONDA_DIR"]}/lib")\n' + ) + + # Create JULIA_PKGDIR, where user libraries are installed + JULIA_PKGDIR = Path(os.environ["JULIA_PKGDIR"]) + JULIA_PKGDIR.mkdir() + subprocess.check_call(["chown", os.environ["NB_USER"], JULIA_PKGDIR]) + subprocess.check_call(["fix-permissions", JULIA_PKGDIR]) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + julia_url, julia_version = get_latest_julia_url() + download_julia(julia_url=julia_url) + configure_julia(julia_version=julia_version) diff --git a/umich-notebook/test/test_languages.py b/umich-notebook/test/test_languages.py deleted file mode 100644 index e4ad641..0000000 --- a/umich-notebook/test/test_languages.py +++ /dev/null @@ -1,37 +0,0 @@ -import docker -from docker.errors import ContainerError - -import logging - -import pytest - - -LOGGER = logging.getLogger(__name__) - - -@pytest.mark.parametrize( - 'language,version_output', - [ - ('python', ['Python', '3.9.6\n']), - ('julia', ['julia', 'version', '1.6.1\n']), - ], -) -def test_languages(language, version_output): - """Ensure that the language is available in the container's PATH and that - it has the correct version - """ - LOGGER.info(f'Test that language {language} is correctly installed ...') - client = docker.from_env() - output = client.containers.run('illumidesk/umich-notebook:julia-1.6.1', f'{language} --version') - output_decoded = output.decode('utf-8').split(' ') - assert output_decoded[0:3] == version_output - LOGGER.info(f'Output from command: {output_decoded[0:3]}') - - -def test_invalid_cmd(): - """Ensure that an invalid command returns a docker.errors.ContainerError - """ - with pytest.raises(ContainerError): - LOGGER.info('Test an invalid command ...') - client = docker.from_env() - client.containers.run('illumidesk/base-notebook', 'foo --version')