From 7ebbdc85a8351280388a35e3f28ca7b8d3874b1d Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Mon, 2 Dec 2024 18:54:47 +0530 Subject: [PATCH 01/15] changing user name to ov_user --- api/Dockerfile | 10 +++++----- catalog/Dockerfile | 4 ++-- indexer_worker/Dockerfile | 8 ++++---- ingestion_server/Dockerfile | 8 ++++---- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index bd2a984a1a5..093f8873c4d 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -86,16 +86,16 @@ RUN apt-get update \ RUN mkdir /static # Create a non-root user, and make it the owner of the static dir created above -RUN useradd --create-home opener \ - && chown -R opener /static -USER opener +RUN useradd --create-home ov_user \ + && chown -R ov_user /static +USER ov_user # Copy subpackages from additional build-context 'packages' # hadolint ignore=DL3022 -COPY --chown=opener --from=packages openverse-attribution /packages/python/openverse-attribution/ +COPY --chown=ov_user --from=packages openverse-attribution /packages/python/openverse-attribution/ # Copy code into the final image -COPY --chown=opener . /api/ +COPY --chown=ov_user . /api/ # Collect static assets, these are used by the next stage, `nginx` RUN env \ diff --git a/catalog/Dockerfile b/catalog/Dockerfile index e7a7bc3d537..0b4261dbe5c 100644 --- a/catalog/Dockerfile +++ b/catalog/Dockerfile @@ -52,9 +52,9 @@ RUN apt-get update \ # These directories are not needed in production but it's easier to perform # this step here while we're doing it for the output directory RUN mkdir -p ${OUTPUT_DIR} /home/airflow/.cache && \ - chown airflow ${OUTPUT_DIR} /home/airflow/.cache + chown ov_user ${OUTPUT_DIR} /home/airflow/.cache -USER airflow +USER ov_user WORKDIR ${AIRFLOW_HOME} # Always add the prod req because the dev reqs depend on it for deduplication diff --git a/indexer_worker/Dockerfile b/indexer_worker/Dockerfile index 95eefba0f1b..17fed0efc4a 100644 --- a/indexer_worker/Dockerfile +++ b/indexer_worker/Dockerfile @@ -70,13 +70,13 @@ RUN apt-get update \ && mkdir /worker_state # Create a non-root user -RUN useradd ingestionu \ - && chown ingestionu /worker_state +RUN useradd ov_user \ + && chown ov_user /worker_state -USER ingestionu +USER ov_user # Copy code into the final image -COPY --chown=ingestionu . /indexer_worker/ +COPY --chown=ov_user . /indexer_worker/ # Expose Gunicorn server to indexer worker Falcon app EXPOSE 8003 diff --git a/ingestion_server/Dockerfile b/ingestion_server/Dockerfile index e28c3366f85..2d4630954bf 100644 --- a/ingestion_server/Dockerfile +++ b/ingestion_server/Dockerfile @@ -72,13 +72,13 @@ RUN apt-get update \ && mkdir /worker_state # Create a non-root user -RUN useradd ingestionu \ - && chown ingestionu /worker_state +RUN useradd ov_user \ + && chown ov_user /worker_state -USER ingestionu +USER ov_user # Copy code into the final image -COPY --chown=ingestionu . /ingestion_server/ +COPY --chown=ov_user . /ingestion_server/ # Exposes # - 8001: Gunicorn server for `ingestion_server` Falcon app From 2d9473ddccb02d6e38115982d578a140375235d1 Mon Sep 17 00:00:00 2001 From: Tejas Warathe <37204061+tejaswarathe@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:04:02 +0530 Subject: [PATCH 02/15] changing directory name to user Co-authored-by: Dhruv Bhanushali --- catalog/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog/Dockerfile b/catalog/Dockerfile index 0b4261dbe5c..49296bac327 100644 --- a/catalog/Dockerfile +++ b/catalog/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update \ # Create and set the ownership of the cache directory so airflow user can write to them # These directories are not needed in production but it's easier to perform # this step here while we're doing it for the output directory -RUN mkdir -p ${OUTPUT_DIR} /home/airflow/.cache && \ +RUN mkdir -p ${OUTPUT_DIR} /home/ov_user/.cache && \ chown ov_user ${OUTPUT_DIR} /home/airflow/.cache USER ov_user From fe89c6b0bd1faad0d7d5db79d68c8a8b9cd15cac Mon Sep 17 00:00:00 2001 From: Tejas Warathe <37204061+tejaswarathe@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:04:12 +0530 Subject: [PATCH 03/15] changing directory name to user Co-authored-by: Dhruv Bhanushali --- catalog/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog/Dockerfile b/catalog/Dockerfile index 49296bac327..38c6c3f3cd2 100644 --- a/catalog/Dockerfile +++ b/catalog/Dockerfile @@ -52,7 +52,7 @@ RUN apt-get update \ # These directories are not needed in production but it's easier to perform # this step here while we're doing it for the output directory RUN mkdir -p ${OUTPUT_DIR} /home/ov_user/.cache && \ - chown ov_user ${OUTPUT_DIR} /home/airflow/.cache + chown ov_user ${OUTPUT_DIR} /home/ov_user/.cache USER ov_user From 1002b39c031eb7dc1a351fa3fdc828161377c3c3 Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Tue, 3 Dec 2024 12:42:23 +0530 Subject: [PATCH 04/15] Updated user directories and adding new user --- catalog/Dockerfile | 3 ++- catalog/compose.yml | 2 +- catalog/pytest.ini | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/catalog/Dockerfile b/catalog/Dockerfile index 38c6c3f3cd2..ec026ff9637 100644 --- a/catalog/Dockerfile +++ b/catalog/Dockerfile @@ -18,7 +18,7 @@ ENV AIRFLOW_HOME=/opt/airflow ENV DAGS_FOLDER=${AIRFLOW_HOME}/catalog/dags ENV PYTHONPATH=${DAGS_FOLDER} ENV PATH=${AIRFLOW_HOME}/.local/bin:$PATH -ENV IPYTHONDIR=/home/airflow/.cache/ipython +ENV IPYTHONDIR=/home/ov_user/.cache/ipython # Container optimizations ENV PYTHONUNBUFFERED=1 @@ -51,6 +51,7 @@ RUN apt-get update \ # Create and set the ownership of the cache directory so airflow user can write to them # These directories are not needed in production but it's easier to perform # this step here while we're doing it for the output directory +RUN useradd -m ov_user RUN mkdir -p ${OUTPUT_DIR} /home/ov_user/.cache && \ chown ov_user ${OUTPUT_DIR} /home/ov_user/.cache diff --git a/catalog/compose.yml b/catalog/compose.yml index 224d3ff790d..7bfbf3b4913 100644 --- a/catalog/compose.yml +++ b/catalog/compose.yml @@ -19,7 +19,7 @@ x-airflow-common: &airflow-common - CATALOG_AIRFLOW_VERSION volumes: - .:/opt/airflow/catalog:z - - catalog-cache:/home/airflow/.cache + - catalog-cache:/home/ov_user/.cache services: # Dev changes for the scheduler diff --git a/catalog/pytest.ini b/catalog/pytest.ini index ddf36bfee9e..2fb1b565e6b 100644 --- a/catalog/pytest.ini +++ b/catalog/pytest.ini @@ -51,7 +51,7 @@ filterwarnings= # Change the pytest cache location since Docker cannot write within the module file # structure due to permissions issues # https://stackoverflow.com/a/56493748/3277713 -cache_dir=/home/airflow/.cache/pytest +cache_dir=/home/ov_user/.cache/pytest env = AIRFLOW_VAR_API_KEY_BROOKLYN_MUSEUM=apikey From b2728162993aee762e20835875daa626dd281cbe Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Wed, 4 Dec 2024 20:52:41 +0530 Subject: [PATCH 05/15] reverting changes --- catalog/Dockerfile | 10 +++++----- indexer_worker/Dockerfile | 8 ++++---- ingestion_server/Dockerfile | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/catalog/Dockerfile b/catalog/Dockerfile index ec026ff9637..88c7fbf2fe5 100644 --- a/catalog/Dockerfile +++ b/catalog/Dockerfile @@ -18,7 +18,7 @@ ENV AIRFLOW_HOME=/opt/airflow ENV DAGS_FOLDER=${AIRFLOW_HOME}/catalog/dags ENV PYTHONPATH=${DAGS_FOLDER} ENV PATH=${AIRFLOW_HOME}/.local/bin:$PATH -ENV IPYTHONDIR=/home/ov_user/.cache/ipython +ENV IPYTHONDIR=/home/airflow/.cache/ipython # Container optimizations ENV PYTHONUNBUFFERED=1 @@ -51,11 +51,11 @@ RUN apt-get update \ # Create and set the ownership of the cache directory so airflow user can write to them # These directories are not needed in production but it's easier to perform # this step here while we're doing it for the output directory -RUN useradd -m ov_user -RUN mkdir -p ${OUTPUT_DIR} /home/ov_user/.cache && \ - chown ov_user ${OUTPUT_DIR} /home/ov_user/.cache +# RUN useradd -m ov_user +RUN mkdir -p ${OUTPUT_DIR} /home/airflow/.cache && \ + chown airflow ${OUTPUT_DIR} /home/airflow/.cache -USER ov_user +USER airflow WORKDIR ${AIRFLOW_HOME} # Always add the prod req because the dev reqs depend on it for deduplication diff --git a/indexer_worker/Dockerfile b/indexer_worker/Dockerfile index 17fed0efc4a..95eefba0f1b 100644 --- a/indexer_worker/Dockerfile +++ b/indexer_worker/Dockerfile @@ -70,13 +70,13 @@ RUN apt-get update \ && mkdir /worker_state # Create a non-root user -RUN useradd ov_user \ - && chown ov_user /worker_state +RUN useradd ingestionu \ + && chown ingestionu /worker_state -USER ov_user +USER ingestionu # Copy code into the final image -COPY --chown=ov_user . /indexer_worker/ +COPY --chown=ingestionu . /indexer_worker/ # Expose Gunicorn server to indexer worker Falcon app EXPOSE 8003 diff --git a/ingestion_server/Dockerfile b/ingestion_server/Dockerfile index 2d4630954bf..e28c3366f85 100644 --- a/ingestion_server/Dockerfile +++ b/ingestion_server/Dockerfile @@ -72,13 +72,13 @@ RUN apt-get update \ && mkdir /worker_state # Create a non-root user -RUN useradd ov_user \ - && chown ov_user /worker_state +RUN useradd ingestionu \ + && chown ingestionu /worker_state -USER ov_user +USER ingestionu # Copy code into the final image -COPY --chown=ov_user . /ingestion_server/ +COPY --chown=ingestionu . /ingestion_server/ # Exposes # - 8001: Gunicorn server for `ingestion_server` Falcon app From 28efdf8db32914f3685553b935c30afee58c0e8d Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Wed, 4 Dec 2024 20:53:45 +0530 Subject: [PATCH 06/15] revert changes --- catalog/compose.yml | 2 +- catalog/pytest.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/catalog/compose.yml b/catalog/compose.yml index 7bfbf3b4913..224d3ff790d 100644 --- a/catalog/compose.yml +++ b/catalog/compose.yml @@ -19,7 +19,7 @@ x-airflow-common: &airflow-common - CATALOG_AIRFLOW_VERSION volumes: - .:/opt/airflow/catalog:z - - catalog-cache:/home/ov_user/.cache + - catalog-cache:/home/airflow/.cache services: # Dev changes for the scheduler diff --git a/catalog/pytest.ini b/catalog/pytest.ini index 2fb1b565e6b..ddf36bfee9e 100644 --- a/catalog/pytest.ini +++ b/catalog/pytest.ini @@ -51,7 +51,7 @@ filterwarnings= # Change the pytest cache location since Docker cannot write within the module file # structure due to permissions issues # https://stackoverflow.com/a/56493748/3277713 -cache_dir=/home/ov_user/.cache/pytest +cache_dir=/home/airflow/.cache/pytest env = AIRFLOW_VAR_API_KEY_BROOKLYN_MUSEUM=apikey From bc1d93d4e79e144603da5323f28876da9fc620d3 Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Wed, 4 Dec 2024 21:41:00 +0530 Subject: [PATCH 07/15] Changing DC_USER --- api/justfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/justfile b/api/justfile index d724387bb6f..b9c97931f09 100644 --- a/api/justfile +++ b/api/justfile @@ -98,7 +98,7 @@ recreate: # Launch a `pgcli` shell in the web container pgcli db_user_pass="deploy" db_name="openledger": up - env DC_USER="opener" just ../_pgcli web {{ db_user_pass }} {{ db_name }} db + env DC_USER="ov_user" just ../_pgcli web {{ db_user_pass }} {{ db_name }} db ######################### # Django administration # @@ -112,7 +112,7 @@ dj-local *args: # Run Django administrative commands inside the Docker container [positional-arguments] dj *args: wait-up - env DC_USER="{{ env_var_or_default("DC_USER", "opener") }}" just ../exec web python manage.py "$@" + env DC_USER="{{ env_var_or_default("DC_USER", "ov_user") }}" just ../exec web python manage.py "$@" # Get IPython shell inside the Docker container ipython: @@ -158,7 +158,7 @@ generate-docs doc="media-props" fail_on_diff="true": # Run API tests inside the Docker container [positional-arguments] test *args: wait-up - env DC_USER="opener" just ../exec web pytest "$@" + env DC_USER="ov_user" just ../exec web pytest "$@" # Run API tests locally [positional-arguments] From 1b26e14ebbd05ae526cee5173207ce1bcaedd700 Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Wed, 4 Dec 2024 23:41:07 +0530 Subject: [PATCH 08/15] Change of user name for catalog --- catalog/Dockerfile | 4 ++-- catalog/justfile | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/catalog/Dockerfile b/catalog/Dockerfile index 88c7fbf2fe5..5aa3b5a0d19 100644 --- a/catalog/Dockerfile +++ b/catalog/Dockerfile @@ -53,9 +53,9 @@ RUN apt-get update \ # this step here while we're doing it for the output directory # RUN useradd -m ov_user RUN mkdir -p ${OUTPUT_DIR} /home/airflow/.cache && \ - chown airflow ${OUTPUT_DIR} /home/airflow/.cache + chown ov_user ${OUTPUT_DIR} /home/airflow/.cache -USER airflow +USER ov_user WORKDIR ${AIRFLOW_HOME} # Always add the prod req because the dev reqs depend on it for deduplication diff --git a/catalog/justfile b/catalog/justfile index 0859b4ecd8e..788fc95e541 100644 --- a/catalog/justfile +++ b/catalog/justfile @@ -81,17 +81,17 @@ recreate: # Launch a Bash shell in an existing container under `SERVICE` shell: - env DC_USER="airflow" just ../exec {{ SERVICE }} /bin/bash + env DC_USER="ov_user" just ../exec {{ SERVICE }} /bin/bash # Run an Airflow CLI command and then exit [positional-arguments] cli *args: - env DC_USER="airflow" just ../exec {{ SERVICE }} "$@" + env DC_USER="ov_user" just ../exec {{ SERVICE }} "$@" # Launch an IPython shell in a new container under `SERVICE` [positional-arguments] ipython *args: up-deps - env DC_USER="airflow" just ../run \ + env DC_USER="ov_user" just ../run \ --workdir /opt/airflow/catalog/dags \ {{ SERVICE }} \ bash -c "ipython ${@:2}" From 33158bee845d2aa742cf02dcd286071d01cb76da Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Thu, 5 Dec 2024 12:03:38 +0530 Subject: [PATCH 09/15] reverting changes --- catalog/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/catalog/Dockerfile b/catalog/Dockerfile index 5aa3b5a0d19..e7a7bc3d537 100644 --- a/catalog/Dockerfile +++ b/catalog/Dockerfile @@ -51,11 +51,10 @@ RUN apt-get update \ # Create and set the ownership of the cache directory so airflow user can write to them # These directories are not needed in production but it's easier to perform # this step here while we're doing it for the output directory -# RUN useradd -m ov_user RUN mkdir -p ${OUTPUT_DIR} /home/airflow/.cache && \ - chown ov_user ${OUTPUT_DIR} /home/airflow/.cache + chown airflow ${OUTPUT_DIR} /home/airflow/.cache -USER ov_user +USER airflow WORKDIR ${AIRFLOW_HOME} # Always add the prod req because the dev reqs depend on it for deduplication From 4d7ce551d0ed1f5b3bd96587776462105a5c141c Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Thu, 5 Dec 2024 12:04:04 +0530 Subject: [PATCH 10/15] reverting changes --- catalog/justfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/catalog/justfile b/catalog/justfile index 788fc95e541..0859b4ecd8e 100644 --- a/catalog/justfile +++ b/catalog/justfile @@ -81,17 +81,17 @@ recreate: # Launch a Bash shell in an existing container under `SERVICE` shell: - env DC_USER="ov_user" just ../exec {{ SERVICE }} /bin/bash + env DC_USER="airflow" just ../exec {{ SERVICE }} /bin/bash # Run an Airflow CLI command and then exit [positional-arguments] cli *args: - env DC_USER="ov_user" just ../exec {{ SERVICE }} "$@" + env DC_USER="airflow" just ../exec {{ SERVICE }} "$@" # Launch an IPython shell in a new container under `SERVICE` [positional-arguments] ipython *args: up-deps - env DC_USER="ov_user" just ../run \ + env DC_USER="airflow" just ../run \ --workdir /opt/airflow/catalog/dags \ {{ SERVICE }} \ bash -c "ipython ${@:2}" From 68558361f71bb06d97bfe1e5826f8206bc7182f5 Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Thu, 5 Dec 2024 12:26:21 +0530 Subject: [PATCH 11/15] changing username fro ingestion_worker --- indexer_worker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/indexer_worker/Dockerfile b/indexer_worker/Dockerfile index 95eefba0f1b..17fed0efc4a 100644 --- a/indexer_worker/Dockerfile +++ b/indexer_worker/Dockerfile @@ -70,13 +70,13 @@ RUN apt-get update \ && mkdir /worker_state # Create a non-root user -RUN useradd ingestionu \ - && chown ingestionu /worker_state +RUN useradd ov_user \ + && chown ov_user /worker_state -USER ingestionu +USER ov_user # Copy code into the final image -COPY --chown=ingestionu . /indexer_worker/ +COPY --chown=ov_user . /indexer_worker/ # Expose Gunicorn server to indexer worker Falcon app EXPOSE 8003 From f1698b6b259456362da0a40666a26341eab7dba6 Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Thu, 5 Dec 2024 12:39:12 +0530 Subject: [PATCH 12/15] changing username for ingestion_server --- ingestion_server/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ingestion_server/Dockerfile b/ingestion_server/Dockerfile index e28c3366f85..2d4630954bf 100644 --- a/ingestion_server/Dockerfile +++ b/ingestion_server/Dockerfile @@ -72,13 +72,13 @@ RUN apt-get update \ && mkdir /worker_state # Create a non-root user -RUN useradd ingestionu \ - && chown ingestionu /worker_state +RUN useradd ov_user \ + && chown ov_user /worker_state -USER ingestionu +USER ov_user # Copy code into the final image -COPY --chown=ingestionu . /ingestion_server/ +COPY --chown=ov_user . /ingestion_server/ # Exposes # - 8001: Gunicorn server for `ingestion_server` Falcon app From 86b9083e18822a9843222f29d771f62dbc9ea4d9 Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Thu, 5 Dec 2024 13:55:07 +0530 Subject: [PATCH 13/15] revert indexer_worker username --- indexer_worker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/indexer_worker/Dockerfile b/indexer_worker/Dockerfile index 17fed0efc4a..95eefba0f1b 100644 --- a/indexer_worker/Dockerfile +++ b/indexer_worker/Dockerfile @@ -70,13 +70,13 @@ RUN apt-get update \ && mkdir /worker_state # Create a non-root user -RUN useradd ov_user \ - && chown ov_user /worker_state +RUN useradd ingestionu \ + && chown ingestionu /worker_state -USER ov_user +USER ingestionu # Copy code into the final image -COPY --chown=ov_user . /indexer_worker/ +COPY --chown=ingestionu . /indexer_worker/ # Expose Gunicorn server to indexer worker Falcon app EXPOSE 8003 From 91cff55bc743bdf7cead400cee7f081381db6839 Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Thu, 5 Dec 2024 13:55:55 +0530 Subject: [PATCH 14/15] change indexer worker username --- indexer_worker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/indexer_worker/Dockerfile b/indexer_worker/Dockerfile index 95eefba0f1b..17fed0efc4a 100644 --- a/indexer_worker/Dockerfile +++ b/indexer_worker/Dockerfile @@ -70,13 +70,13 @@ RUN apt-get update \ && mkdir /worker_state # Create a non-root user -RUN useradd ingestionu \ - && chown ingestionu /worker_state +RUN useradd ov_user \ + && chown ov_user /worker_state -USER ingestionu +USER ov_user # Copy code into the final image -COPY --chown=ingestionu . /indexer_worker/ +COPY --chown=ov_user . /indexer_worker/ # Expose Gunicorn server to indexer worker Falcon app EXPOSE 8003 From 4bdc47ba28b77b6e8872f5c07d3a1cc641f2fbec Mon Sep 17 00:00:00 2001 From: tejaswarathe Date: Fri, 6 Dec 2024 15:49:27 +0530 Subject: [PATCH 15/15] changes to DC_USER --- env.template | 2 +- justfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/env.template b/env.template index fa8dfa849c0..419481a488b 100644 --- a/env.template +++ b/env.template @@ -1,3 +1,3 @@ -# DC_USER=opener +# DC_USER=ov_user # SKIP_DOTENV_FILES_RECREATION=true diff --git a/justfile b/justfile index a1b43276426..d0c041a84e8 100644 --- a/justfile +++ b/justfile @@ -5,7 +5,7 @@ set dotenv-load := true # _ - Private recipes (https://github.com/casey/just#private-recipes) IS_CI := env_var_or_default("CI", "") -DC_USER := env_var_or_default("DC_USER", "opener") +DC_USER := env_var_or_default("DC_USER", "ov_user") # Show all available recipes, also recurses inside nested justfiles @_default: