refactor: change cookiecutter to preserve passwords

cookiecutter-templates/cookiecutter-dioptra-deployment/cookiecutter.json

@@ -18,6 +18,15 @@
     "num_tensorflow_gpu_workers": "0",
     "num_pytorch_cpu_workers": "1",
     "num_pytorch_gpu_workers": "0",
+    "__base_directory": "",
+    "__pgadmin_default_password": "",
+    "__postgres_user_dioptra_password": "",
+    "__postgres_admin_password": "",
+    "__minio_root_password": "",
+    "__minio_mlflow_tracking_password": "",
+    "__minio_restapi_password": "",
+    "__minio_worker_password": "",
+    "__minio_kms_secret_key": "",
     "__containers": {
         "networks": ["dioptra"],
         "nginx": {

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/README.md

@@ -23,7 +23,7 @@ Open a terminal and run the following to start Dioptra.
 
 ```sh
 # Move into the new folder created by the cookiecutter template.
-cd $((BASEDIR))/{{ cookiecutter.__project_slug }}
+cd {{ cookiecutter.__base_directory }}/{{ cookiecutter.__project_slug }}
 
 # Initialize Dioptra using the init-deployment.sh script.
 ./init-deployment.sh
@@ -32,7 +32,7 @@ cd $((BASEDIR))/{{ cookiecutter.__project_slug }}
 docker compose up -d
 ```
 
-To stop Dioptra, navigate back to the configuration folder `$((BASEDIR))/{{ cookiecutter.__project_slug }}` in the terminal and run,
+To stop Dioptra, navigate back to the configuration folder `{{ cookiecutter.__base_directory }}/{{ cookiecutter.__project_slug }}` in the terminal and run,
 
 ```sh
 # Stop Dioptra

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/postgres-passwd.env

@@ -1,4 +1,4 @@
-POSTGRES_PASSWORD={{ "{{ postgres_admin_password }}" }}
+POSTGRES_PASSWORD={{ cookiecutter.__postgres_admin_password }}
 POSTGRES_USER={{ cookiecutter.__db_admin_username }}
 POSTGRES_DB={{ cookiecutter.__db_admin_database }}
-{{ "{{ '' }}" }}
+{{ '' }}

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-db.env

@@ -1,2 +1,2 @@
-POSTGRES_USER_DIOPTRA_PASSWORD={{ "{{ postgres_user_dioptra_password }}" }}
-{{ "{{ '' }}" }}
+POSTGRES_USER_DIOPTRA_PASSWORD={{ cookiecutter.__postgres_user_dioptra_password }}
+{{ '' }}

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-dbadmin.env

@@ -1,2 +1,2 @@
-PGADMIN_DEFAULT_PASSWORD={{ "{{ pgadmin_default_password }}" }}
-{{ "{{ '' }}" }}
+PGADMIN_DEFAULT_PASSWORD={{ cookiecutter.__pgadmin_default_password }}
+{{ '' }}

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-minio-accounts.env

@@ -1,12 +1,12 @@
 MINIO_ROOT_USER=dioptra-admin
-MINIO_ROOT_PASSWORD={{ "{{ minio_root_password }}" }}
+MINIO_ROOT_PASSWORD={{ cookiecutter.__minio_root_password }}
 MINIO_MLFLOW_TRACKING_USER=dioptra-mlflow-tracking
-MINIO_MLFLOW_TRACKING_PASSWORD={{ "{{ minio_mlflow_tracking_password }}" }}
+MINIO_MLFLOW_TRACKING_PASSWORD={{ cookiecutter.__minio_mlflow_tracking_password }}
 MINIO_MLFLOW_TRACKING_POLICIES=mlflow-tracking-readwrite
 MINIO_RESTAPI_USER=dioptra-restapi
-MINIO_RESTAPI_PASSWORD={{ "{{ minio_restapi_password }}" }}
+MINIO_RESTAPI_PASSWORD={{ cookiecutter.__minio_restapi_password }}
 MINIO_RESTAPI_POLICIES=builtin-plugins-readwrite,custom-plugins-readwrite,workflow-uploadonly
 MINIO_WORKER_USER=dioptra-worker
-MINIO_WORKER_PASSWORD={{ "{{ minio_worker_password }}" }}
+MINIO_WORKER_PASSWORD={{ cookiecutter.__minio_worker_password }}
 MINIO_WORKER_POLICIES=builtin-plugins-readonly,custom-plugins-readonly,mlflow-tracking-readwrite,workflow-downloadonly
-{{ "{{ '' }}" }}
+{{ '' }}

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-minio.env

@@ -1,4 +1,4 @@
 MINIO_ROOT_USER=dioptra-admin
-MINIO_ROOT_PASSWORD={{ "{{ minio_root_password }}" }}
-MINIO_KMS_SECRET_KEY={{ cookiecutter.__project_slug }}-minio-key:{{ "{{ minio_kms_secret_key }}" }}
-{{ "{{ '' }}" }}
+MINIO_ROOT_PASSWORD={{ cookiecutter.__minio_root_password }}
+MINIO_KMS_SECRET_KEY={{ cookiecutter.__project_slug }}-minio-key:{{ cookiecutter.__minio_kms_secret_key }}
+{{ '' }}

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-mlflow-tracking-database-uri.env

@@ -1,2 +1,2 @@
-DIOPTRA_MLFLOW_TRACKING_DATABASE_URI=postgresql://dioptra:{{ "{{ postgres_user_dioptra_password }}" }}@{{ cookiecutter.__project_slug }}-db/mlflow-tracking
-{{ "{{ '' }}" }}
+DIOPTRA_MLFLOW_TRACKING_DATABASE_URI=postgresql://dioptra:{{ cookiecutter.__postgres_user_dioptra_password }}@{{ cookiecutter.__project_slug }}-db/mlflow-tracking
+{{ '' }}

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-mlflow-tracking.env

@@ -1,3 +1,3 @@
 AWS_ACCESS_KEY_ID=dioptra-mlflow-tracking
-AWS_SECRET_ACCESS_KEY={{ "{{ minio_mlflow_tracking_password }}" }}
-{{ "{{ '' }}" }}
+AWS_SECRET_ACCESS_KEY={{ cookiecutter.__minio_mlflow_tracking_password }}
+{{ '' }}

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-restapi-database-uri.env

@@ -1,2 +1,2 @@
-DIOPTRA_RESTAPI_DATABASE_URI=postgresql://dioptra:{{ "{{ postgres_user_dioptra_password }}" }}@{{ cookiecutter.__project_slug }}-db/restapi
-{{ "{{ '' }}" }}
+DIOPTRA_RESTAPI_DATABASE_URI=postgresql://dioptra:{{ cookiecutter.__postgres_user_dioptra_password }}@{{ cookiecutter.__project_slug }}-db/restapi
+{{ '' }}

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-restapi.env

@@ -1,3 +1,3 @@
 AWS_ACCESS_KEY_ID=dioptra-restapi
-AWS_SECRET_ACCESS_KEY={{ "{{ minio_restapi_password }}" }}
-{{ "{{ '' }}" }}
+AWS_SECRET_ACCESS_KEY={{ cookiecutter.__minio_restapi_password }}
+{{ '' }}

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/secrets/{{ cookiecutter.__project_slug }}-worker.env

@@ -1,3 +1,3 @@
 AWS_ACCESS_KEY_ID=dioptra-worker
-AWS_SECRET_ACCESS_KEY={{ "{{ minio_worker_password }}" }}
-{{ "{{ '' }}" }}
+AWS_SECRET_ACCESS_KEY={{ cookiecutter.__minio_worker_password }}
+{{ '' }}

cookiecutter-templates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/systemd/dioptra.service

@@ -7,7 +7,7 @@ Wants=syslog.service
 [Service]
 Type=oneshot
 User=root
-WorkingDirectory=$((BASEDIR))
+WorkingDirectory={{ cookiecutter.__base_directory }}
 ExecStart={{ cookiecutter.docker_compose_path }} up -d --remove-orphans --force-recreate
 ExecReload={{ cookiecutter.docker_compose_path }} restart
 ExecStop={{ cookiecutter.docker_compose_path }} down --remove-orphans

cookiecutter-templates/cookiecutter-dioptra-deployment/hooks/post_gen_project.py → cookiecutter-templates/run_cookiecutter.py

@@ -4,82 +4,53 @@
 import logging
 import os
 import random
-import shutil
 import string
+import sys
 import unicodedata
+from cookiecutter.main import cookiecutter
 from pathlib import Path
 
-from jinja2 import Environment, FileSystemLoader
-
 WORDS_FILE = Path("/usr/share/dict/words")
 TEMP_DIRS = ["templates"]
-TEMP_FILES = []
 BASE_DIRECTORY = Path.cwd()
-BASE_DIRECTORY_SYMBOL = "$((BASEDIR))"
-PASSWORD_FILES = [
-    Path("secrets") / "postgres-passwd.env",
-    Path("secrets") / "{{ cookiecutter.__project_slug }}-db.env",
-    Path("secrets") / "{{ cookiecutter.__project_slug }}-dbadmin.env",
-    Path("secrets") / "{{ cookiecutter.__project_slug }}-minio-accounts.env",
-    Path("secrets") / "{{ cookiecutter.__project_slug }}-minio.env",
-    Path("secrets") / "{{ cookiecutter.__project_slug }}-mlflow-tracking-database-uri.env",
-    Path("secrets") / "{{ cookiecutter.__project_slug }}-mlflow-tracking.env",
-    Path("secrets") / "{{ cookiecutter.__project_slug }}-restapi-database-uri.env",
-    Path("secrets") / "{{ cookiecutter.__project_slug }}-restapi.env",
-    Path("secrets") / "{{ cookiecutter.__project_slug }}-worker.env",
-]
-JINJA_ENV = Environment(loader=FileSystemLoader([str(BASE_DIRECTORY)]))
-
 
 logging.basicConfig(level=logging.DEBUG)
-logger = logging.getLogger("post_gen_project")
-
-
-def remove_temp_dirs(temp_dirs):
-    for temp_dir_name in temp_dirs:
-        logger.info("Removing temporary directory: %s", temp_dir_name)
-        shutil.rmtree(temp_dir_name)
-
+logger = logging.getLogger("dioptra-deploy")
 
-def remove_temp_files(temp_files):
-    for temp_file_name in temp_files:
-        logger.info("Removing temporary file: %s", str(temp_file_name))
-        temp_file_name.unlink()
 
-
-def insert_random_passwords(password_files, words_file, env):
+def get_random_passwords(words_file):
     logger.info("Generating \"Correct Horse Battery Staple\" passwords")
 
     words = _populate_words(words_file)
     variables = dict(
-        minio_mlflow_tracking_password=_generate_random_password(
+        __minio_mlflow_tracking_password=_generate_random_password(
             words,
             capitalize=False,
             delimiter="_",
         ),
-        minio_kms_secret_key=_generate_random_kms_secret_key(size=32),
-        minio_root_password=_generate_random_password(words),
-        minio_restapi_password=_generate_random_password(
+        __minio_kms_secret_key=_generate_random_kms_secret_key(size=32),
+        __minio_root_password=_generate_random_password(words),
+        __minio_restapi_password=_generate_random_password(
             words,
             capitalize=False,
             delimiter="_",
         ),
-        minio_worker_password=_generate_random_password(
+        __minio_worker_password=_generate_random_password(
             words,
             capitalize=False,
             delimiter="_",
         ),
-        pgadmin_default_password=_generate_random_password(
+        __pgadmin_default_password=_generate_random_password(
             words,
             capitalize=False,
             delimiter="_",
         ),
-        postgres_admin_password=_generate_random_password(
+        __postgres_admin_password=_generate_random_password(
             words,
             min_words=3,
             min_length=20,
         ),
-        postgres_user_dioptra_password=_generate_random_password(
+        __postgres_user_dioptra_password=_generate_random_password(
             words,
             min_words=3,
             min_length=20,
@@ -88,38 +59,7 @@ def insert_random_passwords(password_files, words_file, env):
         ),
     )
 
-    for filepath in password_files:
-        logger.info("Inserting generated passwords in file: %s", str(filepath))
-
-        content = _render_template(
-            env=env,
-            # Jinja2 requires forward slashes in the template name.
-            template_name=str(filepath.as_posix()),
-            variables=variables,
-        )
-
-        with (BASE_DIRECTORY / filepath).open("wt") as f:
-            f.write(content)
-
-
-def render_absolute_path_to_base_directory():
-    logger.info(
-        "Scanning files and replacing the \"%s\" symbol with %s",
-        BASE_DIRECTORY_SYMBOL,
-        str(BASE_DIRECTORY),
-    )
-
-    for dirpath, dirnames, filenames in os.walk(BASE_DIRECTORY):
-        for filename in filenames:
-            filepath = Path(dirpath) / filename
-
-            with filepath.open("rt", encoding="utf-8") as f:
-                data = f.read()
-
-            data = data.replace(BASE_DIRECTORY_SYMBOL, str(BASE_DIRECTORY))
-
-            with filepath.open("wt", encoding="utf-8") as f:
-                f.write(data)
+    return variables
 
 
 def _render_template(env, template_name, variables):
@@ -198,9 +138,16 @@ def _populate_words(words_file, source_encoding="utf-8", unicode_normalize_form=
 
 
 if __name__ == "__main__":
+    script_path = Path(sys.argv[0])
+    template_path = script_path.with_name("cookiecutter-dioptra-deployment")
+
     logger.debug("Current working directory: %s", str(BASE_DIRECTORY))
+    logger.debug("Template directory: %s", str(template_path))
+
+    extra_context = {
+        "__base_directory": str(BASE_DIRECTORY)
+    }
+    passwords = get_random_passwords(WORDS_FILE)
+    extra_context.update(passwords)
 
-    render_absolute_path_to_base_directory()
-    insert_random_passwords(PASSWORD_FILES, WORDS_FILE, JINJA_ENV)
-    remove_temp_dirs(TEMP_DIRS)
-    remove_temp_files(TEMP_FILES)
+    cookiecutter(str(template_path), extra_context=extra_context)