build!: migrate container images and deploy templates to dioptra v1

This commit migrates the project's Dockerfiles and the cookiecutter deployment template to Dioptra v1. It also adds pyparsing as a project dependency and regenerates the requirements lockfiles. BREAKING CHANGE: The migrated container images and template are not compatible with Dioptra v0. Co-authored-by: Keith Manville <[email protected]>
usnistgov · Jul 5, 2024 · 5071049 · 5071049
1 parent ae0b4a9
commit 5071049
Show file tree

Hide file tree

Showing 40 changed files with 2,279 additions and 1,731 deletions.
diff --git a/...ecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/config/nginx/http_restapi.conf b/...ecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/config/nginx/http_restapi.conf
@@ -13,8 +13,16 @@ server {
         root /frontend;
     }
 
+    # NOTE: This is a hacky workaround until the frontend Javascript client, which
+    # queries against /api (not /api/v1) for the backend API is patched
     location /api/ {
-        proxy_pass http://container-restapi/api/;
+        proxy_pass http://container-restapi/api/v1/;
+    }
+
+    # NOTE: This is a hacky workaround until the frontend Javascript client, which
+    # queries against /api (not /api/v1) for the backend API is patched
+    location /api/v1/ {
+        proxy_pass http://container-restapi/api/v1/;
     }
 
     location /api-docs/ {

diff --git a/...cutter-dioptra-deployment/{{cookiecutter.__project_slug}}/config/nginx/https_restapi.conf b/...cutter-dioptra-deployment/{{cookiecutter.__project_slug}}/config/nginx/https_restapi.conf
@@ -21,8 +21,16 @@ server {
         root /frontend;
     }
 
+    # NOTE: This is a hacky workaround until the frontend Javascript client, which
+    # queries against /api (not /api/v1) for the backend API is patched
     location /api/ {
-        proxy_pass http://container-restapi/api/;
+        proxy_pass http://container-restapi/api/v1/;
+    }
+
+    # NOTE: This is a hacky workaround until the frontend Javascript client, which
+    # queries against /api (not /api/v1) for the backend API is patched
+    location /api/v1/ {
+        proxy_pass http://container-restapi/api/v1/;
     }
 
     location /api-docs/ {

diff --git a/...oyment/{{cookiecutter.__project_slug}}/envs/{{ cookiecutter.__project_slug }}-restapi.env b/...oyment/{{cookiecutter.__project_slug}}/envs/{{ cookiecutter.__project_slug }}-restapi.env
@@ -1,6 +1,5 @@
 DIOPTRA_RESTAPI_ENV=prod
-DIOPTRA_RESTAPI_VERSION=v0
-DIOPTRA_LOGIN_DISABLED=true
+DIOPTRA_RESTAPI_VERSION=v1
 MLFLOW_S3_ENDPOINT_URL=http://{{ cookiecutter.__project_slug }}-minio:9000
 MLFLOW_TRACKING_URI=http://{{ cookiecutter.__project_slug }}-mlflow-tracking:5000
 RQ_REDIS_URI=redis://{{ cookiecutter.__project_slug }}-redis:6379/0
diff --git a/...loyment/{{cookiecutter.__project_slug}}/envs/{{ cookiecutter.__project_slug }}-worker.env b/...loyment/{{cookiecutter.__project_slug}}/envs/{{ cookiecutter.__project_slug }}-worker.env
@@ -1,7 +1,5 @@
-DIOPTRA_PLUGINS_S3_URI=s3://plugins/dioptra_builtins
-DIOPTRA_CUSTOM_PLUGINS_S3_URI=s3://plugins/dioptra_custom
 DIOPTRA_RESTAPI_ENV=prod
-DIOPTRA_LOGIN_DISABLED=true
 MLFLOW_TRACKING_URI=http://{{ cookiecutter.__project_slug }}-mlflow-tracking:5000
 MLFLOW_S3_ENDPOINT_URL=http://{{ cookiecutter.__project_slug }}-minio:9000
 RQ_REDIS_URI=redis://{{ cookiecutter.__project_slug }}-redis:6379/0
+DIOPTRA_API=http://{{ cookiecutter.__project_slug }}-restapi:5000
diff --git a/...dioptra-deployment/{{cookiecutter.__project_slug}}/scripts/generate_password_templates.py b/...dioptra-deployment/{{cookiecutter.__project_slug}}/scripts/generate_password_templates.py
@@ -30,6 +30,7 @@
         Path("systemd", "dioptra.service"),
     ),
 ]
+
 JINJA_ENV = Environment(loader=FileSystemLoader([str(BASE_DIRECTORY)]))
 
 
@@ -82,6 +83,11 @@ def generate_random_passwords(words_file: str | Path) -> dict[str, Any]:
             capitalize=False,
             delimiter="_",
         ),
+        dioptra_worker_password=_generate_random_password(
+            words,
+            min_words=3,
+            min_length=20,
+        ),
     )
 
 

diff --git a/.../cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/scripts/init-frontend.sh b/.../cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/scripts/init-frontend.sh
@@ -289,7 +289,10 @@ prepare_build_dir() {
     "public"
     "index.html"
     "package.json"
-    "vite.config.js"
+    "tsconfig.json"
+    "tsconfig.app.json"
+    "tsconfig.node.json"
+    "vite.config.ts"
   )
 
   log_info "Creating the build directory ${BUILD_DIR}"

diff --git a/...tes/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/scripts/init-minio.sh b/...tes/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/scripts/init-minio.sh
@@ -361,24 +361,6 @@ attach_minio_policies() {
 {% endfor -%}
 }
 
-###########################################################################################
-# Synchronize built-in plugins to plugins bucket
-#
-# Globals:
-#   INIT_REPOS_DIR
-#   MINIO_ENDPOINT_ALIAS
-# Arguments:
-#   None
-# Returns:
-#   None
-###########################################################################################
-
-sync_builtin_plugins() {
-  mc mirror --overwrite --remove \
-    "${INIT_REPOS_DIR}/dioptra/task-plugins/dioptra_builtins" \
-    "${MINIO_ENDPOINT_ALIAS}/plugins/dioptra_builtins"
-}
-
 ###########################################################################################
 # The top-level function in the script
 #
@@ -398,7 +380,6 @@ main() {
   create_minio_accounts
   create_minio_policies
   attach_minio_policies
-  sync_builtin_plugins
 }
 
 ###########################################################################################

diff --git a/...kiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/scripts/templates/dot-env.j2 b/...kiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/scripts/templates/dot-env.j2
@@ -1,5 +1,7 @@
 DIOPTRA_RESTAPI_DATABASE_URI=postgresql://dioptra:{{ "{{ postgres_user_dioptra_password }}" }}@{{ cookiecutter.__project_slug }}-db/restapi
 DIOPTRA_MLFLOW_TRACKING_DATABASE_URI=postgresql://dioptra:{{ "{{ postgres_user_dioptra_password }}" }}@{{ cookiecutter.__project_slug }}-db/mlflow-tracking
+DIOPTRA_WORKER_USERNAME=dioptra-worker
+DIOPTRA_WORKER_PASSWORD={{ "{{ dioptra_worker_password }}" }}
 MINIO_ROOT_USER=dioptra-admin
 MINIO_ROOT_PASSWORD={{ "{{ minio_root_password }}" }}
 MINIO_KMS_SECRET_KEY={{ cookiecutter.__project_slug }}-minio-key:{{ "{{ minio_kms_secret_key }}" }}

diff --git a/...ter-dioptra-deployment/{{cookiecutter.__project_slug}}/templates/macros/docker-compose.j2 b/...ter-dioptra-deployment/{{cookiecutter.__project_slug}}/templates/macros/docker-compose.j2
@@ -295,6 +295,8 @@ volumes:
     AWS_ACCESS_KEY_ID: "${WORKER_AWS_ACCESS_KEY_ID}"
     AWS_SECRET_ACCESS_KEY: "${WORKER_AWS_SECRET_ACCESS_KEY}"
     DIOPTRA_RESTAPI_DATABASE_URI: "${DIOPTRA_RESTAPI_DATABASE_URI}"
+    DIOPTRA_WORKER_USERNAME: "${DIOPTRA_WORKER_USERNAME}"
+    DIOPTRA_WORKER_PASSWORD: "${DIOPTRA_WORKER_PASSWORD}"
   command:
     - --wait-for
     - {{ cookiecutter.__project_slug }}-redis:6379
@@ -332,6 +334,8 @@ volumes:
     AWS_ACCESS_KEY_ID: "${WORKER_AWS_ACCESS_KEY_ID}"
     AWS_SECRET_ACCESS_KEY: "${WORKER_AWS_SECRET_ACCESS_KEY}"
     DIOPTRA_RESTAPI_DATABASE_URI: "${DIOPTRA_RESTAPI_DATABASE_URI}"
+    DIOPTRA_WORKER_USERNAME: "${DIOPTRA_WORKER_USERNAME}"
+    DIOPTRA_WORKER_PASSWORD: "${DIOPTRA_WORKER_PASSWORD}"
     NVIDIA_VISIBLE_DEVICES: {{ gpu_device_id }}
   command:
     - --wait-for
@@ -369,6 +373,8 @@ volumes:
     AWS_ACCESS_KEY_ID: "${WORKER_AWS_ACCESS_KEY_ID}"
     AWS_SECRET_ACCESS_KEY: "${WORKER_AWS_SECRET_ACCESS_KEY}"
     DIOPTRA_RESTAPI_DATABASE_URI: "${DIOPTRA_RESTAPI_DATABASE_URI}"
+    DIOPTRA_WORKER_USERNAME: "${DIOPTRA_WORKER_USERNAME}"
+    DIOPTRA_WORKER_PASSWORD: "${DIOPTRA_WORKER_PASSWORD}"
   command:
     - --wait-for
     - {{ cookiecutter.__project_slug }}-redis:6379
@@ -406,6 +412,8 @@ volumes:
     AWS_ACCESS_KEY_ID: "${WORKER_AWS_ACCESS_KEY_ID}"
     AWS_SECRET_ACCESS_KEY: "${WORKER_AWS_SECRET_ACCESS_KEY}"
     DIOPTRA_RESTAPI_DATABASE_URI: "${DIOPTRA_RESTAPI_DATABASE_URI}"
+    DIOPTRA_WORKER_USERNAME: "${DIOPTRA_WORKER_USERNAME}"
+    DIOPTRA_WORKER_PASSWORD: "${DIOPTRA_WORKER_PASSWORD}"
     NVIDIA_VISIBLE_DEVICES: {{ gpu_device_id }}
   command:
     - --wait-for

diff --git a/docker/Dockerfile.pytorch-cpu b/docker/Dockerfile.pytorch-cpu
@@ -251,7 +251,6 @@ COPY --from=build-venv --chown=${DIOPTRA_UID}:${DIOPTRA_GID} ${VIRTUAL_ENV} ${VI
 ENV DIOPTRA_USER ${DIOPTRA_USER}
 ENV DIOPTRA_UID ${DIOPTRA_UID}
 ENV DIOPTRA_GID ${DIOPTRA_GID}
-ENV DIOPTRA_PLUGIN_DIR ${DIOPTRA_WORKDIR}/plugins
 ENV DIOPTRA_RESTAPI_ENV prod
 ENV DIOPTRA_WORKDIR ${DIOPTRA_WORKDIR}
 ENV PATH ${VIRTUAL_ENV}/bin${PATH:+:${PATH}}

diff --git a/docker/Dockerfile.pytorch-gpu b/docker/Dockerfile.pytorch-gpu
@@ -251,7 +251,6 @@ COPY --from=build-venv --chown=${DIOPTRA_UID}:${DIOPTRA_GID} ${VIRTUAL_ENV} ${VI
 ENV DIOPTRA_USER ${DIOPTRA_USER}
 ENV DIOPTRA_UID ${DIOPTRA_UID}
 ENV DIOPTRA_GID ${DIOPTRA_GID}
-ENV DIOPTRA_PLUGIN_DIR ${DIOPTRA_WORKDIR}/plugins
 ENV DIOPTRA_RESTAPI_ENV prod
 ENV DIOPTRA_WORKDIR ${DIOPTRA_WORKDIR}
 ENV PATH ${VIRTUAL_ENV}/bin${PATH:+:${PATH}}

diff --git a/docker/Dockerfile.tensorflow2-cpu b/docker/Dockerfile.tensorflow2-cpu
@@ -253,7 +253,6 @@ COPY --from=build-venv --chown=${DIOPTRA_UID}:${DIOPTRA_GID} ${VIRTUAL_ENV} ${VI
 ENV DIOPTRA_USER ${DIOPTRA_USER}
 ENV DIOPTRA_UID ${DIOPTRA_UID}
 ENV DIOPTRA_GID ${DIOPTRA_GID}
-ENV DIOPTRA_PLUGIN_DIR ${DIOPTRA_WORKDIR}/plugins
 ENV DIOPTRA_RESTAPI_ENV prod
 ENV DIOPTRA_WORKDIR ${DIOPTRA_WORKDIR}
 ENV PATH ${VIRTUAL_ENV}/bin${PATH:+:${PATH}}

diff --git a/docker/Dockerfile.tensorflow2-gpu b/docker/Dockerfile.tensorflow2-gpu
@@ -253,7 +253,6 @@ COPY --from=build-venv --chown=${DIOPTRA_UID}:${DIOPTRA_GID} ${VIRTUAL_ENV} ${VI
 ENV DIOPTRA_USER ${DIOPTRA_USER}
 ENV DIOPTRA_UID ${DIOPTRA_UID}
 ENV DIOPTRA_GID ${DIOPTRA_GID}
-ENV DIOPTRA_PLUGIN_DIR ${DIOPTRA_WORKDIR}/plugins
 ENV DIOPTRA_RESTAPI_ENV prod
 ENV DIOPTRA_WORKDIR ${DIOPTRA_WORKDIR}
 ENV PATH ${VIRTUAL_ENV}/bin${PATH:+:${PATH}}

diff --git a/docker/requirements/linux-amd64-py3.11-mlflow-tracking-requirements.txt b/docker/requirements/linux-amd64-py3.11-mlflow-tracking-requirements.txt
@@ -4,19 +4,21 @@
 #
 #    pip-compile --output-file=venvs/linux-amd64-py3.11-mlflow-tracking-requirements.txt docker/pip-tools/mlflow-tracking-requirements.in
 #
-alembic==1.13.1
+alembic==1.13.2
     # via mlflow
 aniso8601==9.0.1
     # via graphene
-blinker==1.8.1
+blinker==1.8.2
     # via flask
-boto3==1.34.96
+boto3==1.34.139
     # via -r docker/pip-tools/mlflow-tracking-requirements.in
-botocore==1.34.96
+botocore==1.34.139
     # via
     #   boto3
     #   s3transfer
-certifi==2024.2.2
+cachetools==5.3.3
+    # via mlflow
+certifi==2024.7.4
     # via requests
 charset-normalizer==3.3.2
     # via requests
@@ -30,13 +32,15 @@ contourpy==1.2.1
     # via matplotlib
 cycler==0.12.1
     # via matplotlib
-docker==7.0.0
+deprecated==1.2.14
+    # via opentelemetry-api
+docker==7.1.0
     # via mlflow
 entrypoints==0.4
     # via mlflow
 flask==3.0.3
     # via mlflow
-fonttools==4.51.0
+fonttools==4.53.1
     # via matplotlib
 gitdb==4.0.11
     # via gitpython
@@ -52,15 +56,17 @@ graphql-relay==3.2.0
     # via graphene
 greenlet==3.0.3
     # via sqlalchemy
-gunicorn==21.2.0
+gunicorn==22.0.0
     # via mlflow
 idna==3.7
     # via requests
 importlib-metadata==7.1.0
-    # via mlflow
+    # via
+    #   mlflow
+    #   opentelemetry-api
 itsdangerous==2.2.0
     # via flask
-jinja2==3.1.3
+jinja2==3.1.4
     # via
     #   flask
     #   mlflow
@@ -72,7 +78,7 @@ joblib==1.4.2
     # via scikit-learn
 kiwisolver==1.4.5
     # via matplotlib
-mako==1.3.3
+mako==1.3.5
     # via alembic
 markdown==3.6
     # via mlflow
@@ -81,9 +87,9 @@ markupsafe==2.1.5
     #   jinja2
     #   mako
     #   werkzeug
-matplotlib==3.8.4
+matplotlib==3.9.1
     # via mlflow
-mlflow==2.12.1
+mlflow==2.14.2
     # via -r docker/pip-tools/mlflow-tracking-requirements.in
 numpy==1.26.4
     # via
@@ -94,17 +100,25 @@ numpy==1.26.4
     #   pyarrow
     #   scikit-learn
     #   scipy
-packaging==24.0
+opentelemetry-api==1.25.0
+    # via
+    #   mlflow
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+opentelemetry-sdk==1.25.0
+    # via mlflow
+opentelemetry-semantic-conventions==0.46b0
+    # via opentelemetry-sdk
+packaging==24.1
     # via
-    #   docker
     #   gunicorn
     #   matplotlib
     #   mlflow
 pandas==2.2.2
     # via mlflow
-pillow==10.3.0
+pillow==10.4.0
     # via matplotlib
-protobuf==5.26.1
+protobuf==4.25.3
     # via mlflow
 psycopg2-binary==2.9.9
     # via -r docker/pip-tools/mlflow-tracking-requirements.in
@@ -125,15 +139,15 @@ pyyaml==6.0.1
     # via mlflow
 querystring-parser==1.2.4
     # via mlflow
-requests==2.31.0
+requests==2.32.3
     # via
     #   docker
     #   mlflow
-s3transfer==0.10.1
+s3transfer==0.10.2
     # via boto3
-scikit-learn==1.4.2
+scikit-learn==1.5.1
     # via mlflow
-scipy==1.13.0
+scipy==1.14.0
     # via
     #   mlflow
     #   scikit-learn
@@ -145,26 +159,29 @@ six==1.16.0
     #   querystring-parser
 smmap==5.0.1
     # via gitdb
-sqlalchemy==2.0.29
+sqlalchemy==2.0.31
     # via
     #   alembic
     #   mlflow
 sqlparse==0.5.0
     # via mlflow
 threadpoolctl==3.5.0
     # via scikit-learn
-typing-extensions==4.11.0
+typing-extensions==4.12.2
     # via
     #   alembic
+    #   opentelemetry-sdk
     #   sqlalchemy
 tzdata==2024.1
     # via pandas
-urllib3==2.2.1
+urllib3==2.2.2
     # via
     #   botocore
     #   docker
     #   requests
-werkzeug==3.0.2
+werkzeug==3.0.3
     # via flask
-zipp==3.18.1
+wrapt==1.16.0
+    # via deprecated
+zipp==3.19.2
     # via importlib-metadata