diff --git a/Makefile b/Makefile index 710e38e..3a0b59c 100644 --- a/Makefile +++ b/Makefile @@ -23,9 +23,9 @@ HADOLINT="${HOME}/hadolint" help: # http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html # http://github.com/jupyter/docker-stacks - @echo "illumidesk/umich-stacks" + @echo "illumidesk/illumidesk-stacks" @echo "=====================" - @echo "Replace % with a stack directory name (e.g., make build/umich-notebook)" + @echo "Replace % with a stack directory name (e.g., make build/python-notebook)" @echo @grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index 03eddf7..558cde7 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -[![Test Docker Image Status](https://github.com/illumidesk/umich-stacks/workflows/Test%20and%20Push/badge.svg)](https://github.com/illumidesk/umich-stacks/actions?query=branch%3Amain+workflow%3A%22Test+and+Push%22) +[![Test Docker Image Status](https://github.com/illumidesk/illumidesk-stacks/workflows/Test%20and%20Push/badge.svg)](https://github.com/illumidesk/illumidesk-stacks/actions?query=branch%3Amain+workflow%3A%22Test+and+Push%22) -# IllumiDesk Docker Stacks for the University of Michigan +# IllumiDesk Docker Stacks -This repo is used to manage University of Michigan's docker images for the IllumiDesk learning environment integrated with the Canvas LMS. +This repo is used to manage IllumiDesk's standard docker images for the IllumiDesk learning environment to power Jupyter Server backends. ## Pre Requisits @@ -28,7 +28,7 @@ make build-all Running the image standalone is helpful for testing: ```bash -docker run -p 8888:8888 illumidesk/umich-notebook:latest +docker run -p 8888:8888 illumidesk/python-notebook:latest ``` Then, navigate to `http://127.0.0.1:8888` to access your Jupyter Notebook server. @@ -37,19 +37,24 @@ Then, navigate to `http://127.0.0.1:8888` to access your Jupyter Notebook server ## Customize the Image -1. Add additional Julia packages by editing the `./umich-notebook/install-julia-packages.bash` file. +1. Add additional Python packages by editing the `./python-notebook/requirements.txt` file. 2. Rebuild end-user and grader images with `make build-all`. -3. (Optional) Push images to DockerHub +3. Push images to AWS ECR + +- [You must first authenticate](https://docs.aws.amazon.com/AmazonECR/latest/userguide/registry_auth.html) to push to AWS ECR repos using `docker` coomands. +- Run `docker push ` where `` represents the full path to your AWS ECR repository. + +1. (Optional) Push images to DockerHub This step requires creating an Organization account in DockerHub or other docker image compatible registry. The `docker push ...` command will push the image to the DockerHub registry by default. Please refer to the official Docker documentation if you would like to push another registry. -For example, assuming the DockerHub organization is `illumidesk`, the source files are in the `umich-notebook` folder, and the tag is `latest`, then the full namespace for the image would be `illumidesk/umich-notebook:latest`. Assuming the image has been built, push the image to DockerHub or any other docker registry with the `docker push :` command: +For example, assuming the DockerHub organization is `illumidesk`, the source files are in the `python-notebook` folder, and the tag is `latest`, then the full namespace for the image would be `illumidesk/python-notebook:latest`. Assuming the image has been built, push the image to DockerHub or any other docker registry with the `docker push :` command: ```bash docker login -docker push illumidesk/umich-notebook:latest +docker push illumidesk/python-notebook:latest ``` ## Development and Testing diff --git a/umich-notebook/Dockerfile b/python-notebook/Dockerfile similarity index 51% rename from umich-notebook/Dockerfile rename to python-notebook/Dockerfile index ad4fbde..a260481 100644 --- a/umich-notebook/Dockerfile +++ b/python-notebook/Dockerfile @@ -2,30 +2,21 @@ # https://github.com/jupyter/docker-stacks/blob/main/images/julia-notebook ARG REGISTRY=quay.io ARG OWNER=jupyter -ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook -FROM $BASE_CONTAINER +ARG BASE_CONTAINER=$REGISTRY/$OWNER/datascience-notebook +FROM ${BASE_CONTAINER} # Fix: https://github.com/hadolint/hadolint/wiki/DL4006 # Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 SHELL ["/bin/bash", "-o", "pipefail", "-c"] -USER root - -# Julia dependencies -# install Julia packages in /opt/julia instead of ${HOME} -ENV JULIA_DEPOT_PATH=/opt/julia \ - JULIA_PKGDIR=/opt/julia - -# Setup Julia -RUN /opt/setup-scripts/setup_julia.py - USER ${NB_UID} # Setup IJulia kernel & other packages -RUN /opt/setup-scripts/setup-julia-packages.bash +COPY --chown=${NB_UID}:${NB_GID} requirements.txt "${HOME}/requirements.txt" +WORKDIR "${HOME}" +RUN pip install -r requirements.txt -RUN pip install jupyter_kernel_gateway psycopg2-binary WORKDIR "${HOME}" -CMD ["jupyter", "kernelgateway", "--KernelGatewayApp.ip=0.0.0.0", "--KernelGatewayApp.port=8888"] \ No newline at end of file +CMD ["jupyter", "kernelgateway", "--KernelGatewayApp.ip=0.0.0.0", "--KernelGatewayApp.port=8888"] diff --git a/python-notebook/requirements.txt b/python-notebook/requirements.txt new file mode 100755 index 0000000..a051485 --- /dev/null +++ b/python-notebook/requirements.txt @@ -0,0 +1,18 @@ +psycopg2-binary +gpt4all +jupyter_kernel_gateway +langchain +langchain.tools +langchain_openai +langchain_text_splitters +llmlingua +networkx +nltk +openai +pdfplumber +pymupdf +pypdf +pdf2image +poppler-utils +reportlab +requests \ No newline at end of file diff --git a/umich-notebook/setup-scripts/install-julia-packages.bash b/umich-notebook/setup-scripts/install-julia-packages.bash deleted file mode 100644 index e0ef68b..0000000 --- a/umich-notebook/setup-scripts/install-julia-packages.bash +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -set -exuo pipefail -# Requirements: -# - Run as a non-root user -# - The JULIA_PKGDIR environment variable is set -# - Julia is already set up, with the setup_julia.py command - -# replaces the default julia environment with the one we want -# ref: https://github.com/jupyter/docker-stacks/blob/main/images/minimal-notebook/setup-scripts/setup-julia-packages.bash -if [ "$(uname -m)" == "x86_64" ]; then - # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L24 - # for an explanation of these options - export JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" -elif [ "$(uname -m)" == "aarch64" ]; then - # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L54 - # for an explanation of these options - export JULIA_CPU_TARGET="generic;cortex-a57;thunderx2t99;carmel" -fi - -# Install base Julia packages -julia -e ' -import Pkg; -Pkg.update(); -Pkg.add([ - "BenchmarkTools" - "Colors" - "CSV" - "CSVFiles" - "Compat" - "CoordinateTransformations" - "DataFrames" - "DelimitedFiles" - "DifferentialEquations" - "Distributions" - "FileIO" - "FiniteDiff" - "ForwardDiff" - "GeometryBasics" - "GMT" - "Images" - "ImageInTerminal" - "ImageFiltering" - "ImageFeatures" - "Interact" - "Interpolations" - "JLD2" - "LinearAlgebra" - "LaTeXStrings" - "Latexify" - "MeshCat" - "OSQP" - "Plots" - "Plotly" - "Printf" - "ProgressBars" - "PyPlot" - "Random" - "Rotations" - "Roots" - "SparseArrays" - "SymEngine" - "Symbolics" - "Statistics" - "StaticArrays" - "WebIO" - "WGLMakie" -]); -Pkg.precompile(); -' - -# Move the kernelspec out of ${HOME} to the system share location. -# Avoids problems with runtime UID change not taking effect properly -# on the .local folder in the jovyan home dir. -mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" -chmod -R go+rx "${CONDA_DIR}/share/jupyter" -rm -rf "${HOME}/.local" -fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter" diff --git a/umich-notebook/setup-scripts/setup_julia.py b/umich-notebook/setup-scripts/setup_julia.py deleted file mode 100644 index 114e64c..0000000 --- a/umich-notebook/setup-scripts/setup_julia.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -# Requirements: -# - Run as the root user -# - The JULIA_PKGDIR environment variable is set - -import logging -import os -import platform -import shutil -import subprocess -from pathlib import Path - -import requests - -LOGGER = logging.getLogger(__name__) - - -def unify_aarch64(platform: str) -> str: - """ - Renames arm64->aarch64 to support local builds on on aarch64 Macs - """ - return { - "aarch64": "aarch64", - "arm64": "aarch64", - "x86_64": "x86_64", - }[platform] - - -def get_latest_julia_url() -> tuple[str, str]: - """ - Get the last stable version of Julia - Based on: https://github.com/JuliaLang/www.julialang.org/issues/878#issuecomment-749234813 - """ - LOGGER.info("Downloading Julia versions information") - versions = requests.get( - "https://julialang-s3.julialang.org/bin/versions.json" - ).json() - stable_versions = {k: v for k, v in versions.items() if v["stable"]} - # Compare versions semantically - latest_stable_version = max( - stable_versions, key=lambda ver: [int(sub_ver) for sub_ver in ver.split(".")] - ) - latest_version_files = stable_versions[latest_stable_version]["files"] - triplet = unify_aarch64(platform.machine()) + "-linux-gnu" - file_info = [vf for vf in latest_version_files if vf["triplet"] == triplet][0] - LOGGER.info(f"Latest version: {file_info['version']} url: {file_info['url']}") - return file_info["url"], file_info["version"] - - -def download_julia(julia_url: str) -> None: - """ - Downloads and unpacks julia - The resulting julia directory is "/opt/julia-VERSION/" - """ - LOGGER.info("Downloading and unpacking Julia") - tmp_file = Path("/tmp/julia.tar.gz") - subprocess.check_call( - ["curl", "--progress-bar", "--location", "--output", tmp_file, julia_url] - ) - shutil.unpack_archive(tmp_file, "/opt/") - tmp_file.unlink() - - -def configure_julia(julia_version: str) -> None: - """ - Creates /usr/local/bin/julia symlink - Make Julia aware of conda libraries - Creates a directory for Julia user libraries - """ - LOGGER.info("Configuring Julia") - # Link Julia installed version to /usr/local/bin, so julia launches it - subprocess.check_call( - ["ln", "-fs", f"/opt/julia-{julia_version}/bin/julia", "/usr/local/bin/julia"] - ) - - # Tell Julia where conda libraries are - Path("/etc/julia").mkdir() - Path("/etc/julia/juliarc.jl").write_text( - f'push!(Libdl.DL_LOAD_PATH, "{os.environ["CONDA_DIR"]}/lib")\n' - ) - - # Create JULIA_PKGDIR, where user libraries are installed - JULIA_PKGDIR = Path(os.environ["JULIA_PKGDIR"]) - JULIA_PKGDIR.mkdir() - subprocess.check_call(["chown", os.environ["NB_USER"], JULIA_PKGDIR]) - subprocess.check_call(["fix-permissions", JULIA_PKGDIR]) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - - julia_url, julia_version = get_latest_julia_url() - download_julia(julia_url=julia_url) - configure_julia(julia_version=julia_version)