diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/cookiecutter.json b/cookiecutter-templates/cookiecutter-dioptra-deployment/cookiecutter.json index 19801c053..deac16494 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/cookiecutter.json +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/cookiecutter.json @@ -18,6 +18,15 @@ "num_tensorflow_gpu_workers": "0", "num_pytorch_cpu_workers": "1", "num_pytorch_gpu_workers": "0", + "__base_directory": "", + "__pgadmin_default_password": "", + "__postgres_user_dioptra_password": "", + "__postgres_admin_password": "", + "__minio_root_password": "", + "__minio_mlflow_tracking_password": "", + "__minio_restapi_password": "", + "__minio_worker_password": "", + "__minio_kms_secret_key": "", "__containers": { "networks": ["dioptra"], "nginx": { diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/hooks/pre_gen_project.py b/cookiecutter-templates/cookiecutter-dioptra-deployment/hooks/pre_gen_project.py deleted file mode 100644 index 9fb76db17..000000000 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/hooks/pre_gen_project.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python - -import logging - - -logging.basicConfig(level=logging.DEBUG) -logger = logging.getLogger("pre_gen_project") diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/README.md b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/README.md index b7645038b..c20032b2e 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/README.md +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/README.md @@ -23,7 +23,7 @@ Open a terminal and run the following to start Dioptra. ```sh # Move into the new folder created by the cookiecutter template. -cd $((BASEDIR))/{{ cookiecutter.__project_slug }} +cd {{ cookiecutter.__base_directory }}/{{ cookiecutter.__project_slug }} # Initialize Dioptra using the init-deployment.sh script. ./init-deployment.sh @@ -32,7 +32,7 @@ cd $((BASEDIR))/{{ cookiecutter.__project_slug }} docker compose up -d ``` -To stop Dioptra, navigate back to the configuration folder `$((BASEDIR))/{{ cookiecutter.__project_slug }}` in the terminal and run, +To stop Dioptra, navigate back to the configuration folder `{{ cookiecutter.__base_directory }}/{{ cookiecutter.__project_slug }}` in the terminal and run, ```sh # Stop Dioptra diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/postgres-passwd.env b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/postgres-passwd.env index 2a491c347..5554990b3 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/postgres-passwd.env +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/postgres-passwd.env @@ -1,4 +1,4 @@ -POSTGRES_PASSWORD={{ "{{ postgres_admin_password }}" }} +POSTGRES_PASSWORD={{ cookiecutter.__postgres_admin_password }} POSTGRES_USER={{ cookiecutter.__db_admin_username }} POSTGRES_DB={{ cookiecutter.__db_admin_database }} -{{ "{{ '' }}" }} +{{ '' }} diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-db.env b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-db.env index 6df815a2e..558ad6aa3 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-db.env +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-db.env @@ -1,2 +1,2 @@ -POSTGRES_USER_DIOPTRA_PASSWORD={{ "{{ postgres_user_dioptra_password }}" }} -{{ "{{ '' }}" }} +POSTGRES_USER_DIOPTRA_PASSWORD={{ cookiecutter.__postgres_user_dioptra_password }} +{{ '' }} diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-dbadmin.env b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-dbadmin.env index a1776ab8c..3c5fd9dc9 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-dbadmin.env +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-dbadmin.env @@ -1,2 +1,2 @@ -PGADMIN_DEFAULT_PASSWORD={{ "{{ pgadmin_default_password }}" }} -{{ "{{ '' }}" }} +PGADMIN_DEFAULT_PASSWORD={{ cookiecutter.__pgadmin_default_password }} +{{ '' }} diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-minio-accounts.env b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-minio-accounts.env index 176404103..4f4d2c7c4 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-minio-accounts.env +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-minio-accounts.env @@ -1,12 +1,12 @@ MINIO_ROOT_USER=dioptra-admin -MINIO_ROOT_PASSWORD={{ "{{ minio_root_password }}" }} +MINIO_ROOT_PASSWORD={{ cookiecutter.__minio_root_password }} MINIO_MLFLOW_TRACKING_USER=dioptra-mlflow-tracking -MINIO_MLFLOW_TRACKING_PASSWORD={{ "{{ minio_mlflow_tracking_password }}" }} +MINIO_MLFLOW_TRACKING_PASSWORD={{ cookiecutter.__minio_mlflow_tracking_password }} MINIO_MLFLOW_TRACKING_POLICIES=mlflow-tracking-readwrite MINIO_RESTAPI_USER=dioptra-restapi -MINIO_RESTAPI_PASSWORD={{ "{{ minio_restapi_password }}" }} +MINIO_RESTAPI_PASSWORD={{ cookiecutter.__minio_restapi_password }} MINIO_RESTAPI_POLICIES=builtin-plugins-readwrite,custom-plugins-readwrite,workflow-uploadonly MINIO_WORKER_USER=dioptra-worker -MINIO_WORKER_PASSWORD={{ "{{ minio_worker_password }}" }} +MINIO_WORKER_PASSWORD={{ cookiecutter.__minio_worker_password }} MINIO_WORKER_POLICIES=builtin-plugins-readonly,custom-plugins-readonly,mlflow-tracking-readwrite,workflow-downloadonly -{{ "{{ '' }}" }} +{{ '' }} diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-minio.env b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-minio.env index 64428e7bf..76d099789 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-minio.env +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-minio.env @@ -1,4 +1,4 @@ MINIO_ROOT_USER=dioptra-admin -MINIO_ROOT_PASSWORD={{ "{{ minio_root_password }}" }} -MINIO_KMS_SECRET_KEY={{ cookiecutter.__project_slug }}-minio-key:{{ "{{ minio_kms_secret_key }}" }} -{{ "{{ '' }}" }} +MINIO_ROOT_PASSWORD={{ cookiecutter.__minio_root_password }} +MINIO_KMS_SECRET_KEY={{ cookiecutter.__project_slug }}-minio-key:{{ cookiecutter.__minio_kms_secret_key }} +{{ '' }} diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-mlflow-tracking-database-uri.env b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-mlflow-tracking-database-uri.env index 29cf1b1f9..875fcf009 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-mlflow-tracking-database-uri.env +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-mlflow-tracking-database-uri.env @@ -1,2 +1,2 @@ -DIOPTRA_MLFLOW_TRACKING_DATABASE_URI=postgresql://dioptra:{{ "{{ postgres_user_dioptra_password }}" }}@{{ cookiecutter.__project_slug }}-db/mlflow-tracking -{{ "{{ '' }}" }} +DIOPTRA_MLFLOW_TRACKING_DATABASE_URI=postgresql://dioptra:{{ cookiecutter.__postgres_user_dioptra_password }}@{{ cookiecutter.__project_slug }}-db/mlflow-tracking +{{ '' }} diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-mlflow-tracking.env b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-mlflow-tracking.env index 71de15417..79d7b898b 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-mlflow-tracking.env +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-mlflow-tracking.env @@ -1,3 +1,3 @@ AWS_ACCESS_KEY_ID=dioptra-mlflow-tracking -AWS_SECRET_ACCESS_KEY={{ "{{ minio_mlflow_tracking_password }}" }} -{{ "{{ '' }}" }} +AWS_SECRET_ACCESS_KEY={{ cookiecutter.__minio_mlflow_tracking_password }} +{{ '' }} diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-restapi-database-uri.env b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-restapi-database-uri.env index 2cab36d3f..c0f5dc174 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-restapi-database-uri.env +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-restapi-database-uri.env @@ -1,2 +1,2 @@ -DIOPTRA_RESTAPI_DATABASE_URI=postgresql://dioptra:{{ "{{ postgres_user_dioptra_password }}" }}@{{ cookiecutter.__project_slug }}-db/restapi -{{ "{{ '' }}" }} +DIOPTRA_RESTAPI_DATABASE_URI=postgresql://dioptra:{{ cookiecutter.__postgres_user_dioptra_password }}@{{ cookiecutter.__project_slug }}-db/restapi +{{ '' }} diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-restapi.env b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-restapi.env index aa290a2e7..37ac5ee47 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-restapi.env +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-restapi.env @@ -1,3 +1,3 @@ AWS_ACCESS_KEY_ID=dioptra-restapi -AWS_SECRET_ACCESS_KEY={{ "{{ minio_restapi_password }}" }} -{{ "{{ '' }}" }} +AWS_SECRET_ACCESS_KEY={{ cookiecutter.__minio_restapi_password }} +{{ '' }} diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-worker.env b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-worker.env index 25bc13567..971a03a6f 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-worker.env +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-worker.env @@ -1,3 +1,3 @@ AWS_ACCESS_KEY_ID=dioptra-worker -AWS_SECRET_ACCESS_KEY={{ "{{ minio_worker_password }}" }} -{{ "{{ '' }}" }} +AWS_SECRET_ACCESS_KEY={{ cookiecutter.__minio_worker_password }} +{{ '' }} diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/systemd/dioptra.service b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/systemd/dioptra.service index ba3d7910d..f50109afd 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/systemd/dioptra.service +++ b/cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/systemd/dioptra.service @@ -7,7 +7,7 @@ Wants=syslog.service [Service] Type=oneshot User=root -WorkingDirectory=$((BASEDIR)) +WorkingDirectory={{ cookiecutter.__base_directory }} ExecStart={{ cookiecutter.docker_compose_path }} up -d --remove-orphans --force-recreate ExecReload={{ cookiecutter.docker_compose_path }} restart ExecStop={{ cookiecutter.docker_compose_path }} down --remove-orphans diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/hooks/post_gen_project.py b/cookiecutter-templates/run_cookiecutter.py similarity index 52% rename from cookiecutter-templates/cookiecutter-dioptra-deployment/hooks/post_gen_project.py rename to cookiecutter-templates/run_cookiecutter.py index 9fd9c3e96..f08a1008c 100644 --- a/cookiecutter-templates/cookiecutter-dioptra-deployment/hooks/post_gen_project.py +++ b/cookiecutter-templates/run_cookiecutter.py @@ -4,82 +4,53 @@ import logging import os import random -import shutil import string +import sys import unicodedata +from cookiecutter.main import cookiecutter from pathlib import Path -from jinja2 import Environment, FileSystemLoader - WORDS_FILE = Path("/usr/share/dict/words") TEMP_DIRS = ["templates"] -TEMP_FILES = [] BASE_DIRECTORY = Path.cwd() -BASE_DIRECTORY_SYMBOL = "$((BASEDIR))" -PASSWORD_FILES = [ - Path("secrets") / "postgres-passwd.env", - Path("secrets") / "{{ cookiecutter.__project_slug }}-db.env", - Path("secrets") / "{{ cookiecutter.__project_slug }}-dbadmin.env", - Path("secrets") / "{{ cookiecutter.__project_slug }}-minio-accounts.env", - Path("secrets") / "{{ cookiecutter.__project_slug }}-minio.env", - Path("secrets") / "{{ cookiecutter.__project_slug }}-mlflow-tracking-database-uri.env", - Path("secrets") / "{{ cookiecutter.__project_slug }}-mlflow-tracking.env", - Path("secrets") / "{{ cookiecutter.__project_slug }}-restapi-database-uri.env", - Path("secrets") / "{{ cookiecutter.__project_slug }}-restapi.env", - Path("secrets") / "{{ cookiecutter.__project_slug }}-worker.env", -] -JINJA_ENV = Environment(loader=FileSystemLoader([str(BASE_DIRECTORY)])) - logging.basicConfig(level=logging.DEBUG) -logger = logging.getLogger("post_gen_project") - - -def remove_temp_dirs(temp_dirs): - for temp_dir_name in temp_dirs: - logger.info("Removing temporary directory: %s", temp_dir_name) - shutil.rmtree(temp_dir_name) - +logger = logging.getLogger("dioptra-deploy") -def remove_temp_files(temp_files): - for temp_file_name in temp_files: - logger.info("Removing temporary file: %s", str(temp_file_name)) - temp_file_name.unlink() - -def insert_random_passwords(password_files, words_file, env): +def get_random_passwords(words_file): logger.info("Generating \"Correct Horse Battery Staple\" passwords") words = _populate_words(words_file) variables = dict( - minio_mlflow_tracking_password=_generate_random_password( + __minio_mlflow_tracking_password=_generate_random_password( words, capitalize=False, delimiter="_", ), - minio_kms_secret_key=_generate_random_kms_secret_key(size=32), - minio_root_password=_generate_random_password(words), - minio_restapi_password=_generate_random_password( + __minio_kms_secret_key=_generate_random_kms_secret_key(size=32), + __minio_root_password=_generate_random_password(words), + __minio_restapi_password=_generate_random_password( words, capitalize=False, delimiter="_", ), - minio_worker_password=_generate_random_password( + __minio_worker_password=_generate_random_password( words, capitalize=False, delimiter="_", ), - pgadmin_default_password=_generate_random_password( + __pgadmin_default_password=_generate_random_password( words, capitalize=False, delimiter="_", ), - postgres_admin_password=_generate_random_password( + __postgres_admin_password=_generate_random_password( words, min_words=3, min_length=20, ), - postgres_user_dioptra_password=_generate_random_password( + __postgres_user_dioptra_password=_generate_random_password( words, min_words=3, min_length=20, @@ -88,38 +59,7 @@ def insert_random_passwords(password_files, words_file, env): ), ) - for filepath in password_files: - logger.info("Inserting generated passwords in file: %s", str(filepath)) - - content = _render_template( - env=env, - # Jinja2 requires forward slashes in the template name. - template_name=str(filepath.as_posix()), - variables=variables, - ) - - with (BASE_DIRECTORY / filepath).open("wt") as f: - f.write(content) - - -def render_absolute_path_to_base_directory(): - logger.info( - "Scanning files and replacing the \"%s\" symbol with %s", - BASE_DIRECTORY_SYMBOL, - str(BASE_DIRECTORY), - ) - - for dirpath, dirnames, filenames in os.walk(BASE_DIRECTORY): - for filename in filenames: - filepath = Path(dirpath) / filename - - with filepath.open("rt", encoding="utf-8") as f: - data = f.read() - - data = data.replace(BASE_DIRECTORY_SYMBOL, str(BASE_DIRECTORY)) - - with filepath.open("wt", encoding="utf-8") as f: - f.write(data) + return variables def _render_template(env, template_name, variables): @@ -198,9 +138,16 @@ def _populate_words(words_file, source_encoding="utf-8", unicode_normalize_form= if __name__ == "__main__": + script_path = Path(sys.argv[0]) + template_path = script_path.with_name("cookiecutter-dioptra-deployment") + logger.debug("Current working directory: %s", str(BASE_DIRECTORY)) + logger.debug("Template directory: %s", str(template_path)) + + extra_context = { + "__base_directory": str(BASE_DIRECTORY) + } + passwords = get_random_passwords(WORDS_FILE) + extra_context.update(passwords) - render_absolute_path_to_base_directory() - insert_random_passwords(PASSWORD_FILES, WORDS_FILE, JINJA_ENV) - remove_temp_dirs(TEMP_DIRS) - remove_temp_files(TEMP_FILES) + cookiecutter(str(template_path), extra_context=extra_context)