From 6e33010e610991e0ffb5454045d3c72a38e26da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafn=20=C3=81rnason?= Date: Wed, 11 Dec 2024 11:15:09 +0000 Subject: [PATCH 1/8] fix get-secrets and env cleanup --- apps/services/user-notification/README.md | 3 ++- apps/services/user-notification/project.json | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/services/user-notification/README.md b/apps/services/user-notification/README.md index e7a51dd79013..ceb03b9a0284 100644 --- a/apps/services/user-notification/README.md +++ b/apps/services/user-notification/README.md @@ -40,6 +40,8 @@ Get secrets: yarn get-secrets user-notification ``` +> **Important**: For local development, the `FIREBASE_CREDENTIALS` environment variable needs to be cleaned of newlines. After running `get-secrets`, make sure to remove any newline characters from the `FIREBASE_CREDENTIALS` value in your environment. + Initalize dependencies: ```sh @@ -68,4 +70,3 @@ Start a cleanup worker with this command: ```sh yarn nx run services-user-notification:cleanup -``` diff --git a/apps/services/user-notification/project.json b/apps/services/user-notification/project.json index c55dca8e3ba9..220fe98318f0 100644 --- a/apps/services/user-notification/project.json +++ b/apps/services/user-notification/project.json @@ -126,7 +126,6 @@ "executor": "nx:run-commands", "options": { "commands": [ - "yarn get-secrets user-notification", "yarn nx run services-user-notification:dev-services", "yarn nx run services-user-notification:migrate", "yarn nx run services-user-notification:seed" From 09c583cd7633f8d4e495831d87a1017409e294d7 Mon Sep 17 00:00:00 2001 From: andes-it Date: Wed, 11 Dec 2024 11:30:56 +0000 Subject: [PATCH 2/8] chore: nx format:write update dirty files --- apps/services/user-notification/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/services/user-notification/README.md b/apps/services/user-notification/README.md index ceb03b9a0284..e2530dd5d49d 100644 --- a/apps/services/user-notification/README.md +++ b/apps/services/user-notification/README.md @@ -70,3 +70,4 @@ Start a cleanup worker with this command: ```sh yarn nx run services-user-notification:cleanup +``` From 1128ee549d1ad5381a274b79af8f641e3b23054e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafn=20=C3=81rnason?= Date: Wed, 11 Dec 2024 14:47:08 +0000 Subject: [PATCH 3/8] string cleaner for local --- apps/services/user-notification/README.md | 1 - apps/services/user-notification/project.json | 1 + apps/services/user-notification/src/config.ts | 17 +++++++++++++++-- libs/shared/utils/src/lib/environment.ts | 1 + 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/apps/services/user-notification/README.md b/apps/services/user-notification/README.md index ceb03b9a0284..aea7373bb9d7 100644 --- a/apps/services/user-notification/README.md +++ b/apps/services/user-notification/README.md @@ -40,7 +40,6 @@ Get secrets: yarn get-secrets user-notification ``` -> **Important**: For local development, the `FIREBASE_CREDENTIALS` environment variable needs to be cleaned of newlines. After running `get-secrets`, make sure to remove any newline characters from the `FIREBASE_CREDENTIALS` value in your environment. Initalize dependencies: diff --git a/apps/services/user-notification/project.json b/apps/services/user-notification/project.json index 220fe98318f0..c55dca8e3ba9 100644 --- a/apps/services/user-notification/project.json +++ b/apps/services/user-notification/project.json @@ -126,6 +126,7 @@ "executor": "nx:run-commands", "options": { "commands": [ + "yarn get-secrets user-notification", "yarn nx run services-user-notification:dev-services", "yarn nx run services-user-notification:migrate", "yarn nx run services-user-notification:seed" diff --git a/apps/services/user-notification/src/config.ts b/apps/services/user-notification/src/config.ts index 0ba2d9017bcb..6acb678609f7 100644 --- a/apps/services/user-notification/src/config.ts +++ b/apps/services/user-notification/src/config.ts @@ -2,6 +2,7 @@ import { z } from 'zod' import { processJob } from '@island.is/infra-nest-server' import { defineConfig } from '@island.is/nest/config' +import { isRunningOnEnvironment } from '@island.is/shared/utils' // Exported for testing purposes export const schema = z.object({ @@ -17,14 +18,26 @@ export const schema = z.object({ }), }) + + export const UserNotificationsConfig = defineConfig({ name: 'UserNotificationsApi', schema, - load(env) { + load: (env) => { const isWorker = processJob() === 'worker' + + let firebaseCredentials = env.required('FIREBASE_CREDENTIALS', '') + + // Clean the credentials string for local development + if (isRunningOnEnvironment('local')) { + firebaseCredentials = firebaseCredentials + .replace(/\\n/g, '') // clean newlines + .replace(/\\/g, '') // clean backslashes + } + return { isWorker, - firebaseCredentials: env.required('FIREBASE_CREDENTIALS', ''), + firebaseCredentials, servicePortalClickActionUrl: env.optional('SERVICE_PORTAL_CLICK_ACTION_URL') ?? 'https://island.is/minarsidur', diff --git a/libs/shared/utils/src/lib/environment.ts b/libs/shared/utils/src/lib/environment.ts index edce4ecbee01..d7ca4fab5fc7 100644 --- a/libs/shared/utils/src/lib/environment.ts +++ b/libs/shared/utils/src/lib/environment.ts @@ -37,5 +37,6 @@ if (isServer) { * @param environment */ export const isRunningOnEnvironment = (environment: ActiveEnvironment) => { + console.log('isRunningOnEnvironment', environment, activeEnvironment) return environment === activeEnvironment } From a81371c48b24a7eb5d6010f6568260db6d60daaa Mon Sep 17 00:00:00 2001 From: andes-it Date: Wed, 11 Dec 2024 15:21:45 +0000 Subject: [PATCH 4/8] chore: nx format:write update dirty files --- apps/services/user-notification/README.md | 1 - apps/services/user-notification/src/config.ts | 2 -- 2 files changed, 3 deletions(-) diff --git a/apps/services/user-notification/README.md b/apps/services/user-notification/README.md index 24a30bae0e2d..e7a51dd79013 100644 --- a/apps/services/user-notification/README.md +++ b/apps/services/user-notification/README.md @@ -40,7 +40,6 @@ Get secrets: yarn get-secrets user-notification ``` - Initalize dependencies: ```sh diff --git a/apps/services/user-notification/src/config.ts b/apps/services/user-notification/src/config.ts index 6acb678609f7..723331da9162 100644 --- a/apps/services/user-notification/src/config.ts +++ b/apps/services/user-notification/src/config.ts @@ -18,8 +18,6 @@ export const schema = z.object({ }), }) - - export const UserNotificationsConfig = defineConfig({ name: 'UserNotificationsApi', schema, From 450397573c060aa3ac1c2ff6143f65c96ee2570e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafn=20=C3=81rnason?= Date: Wed, 11 Dec 2024 15:30:45 +0000 Subject: [PATCH 5/8] update get-secrets --- apps/services/user-notification/src/config.ts | 17 ++----------- scripts/get-secrets.sh | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/apps/services/user-notification/src/config.ts b/apps/services/user-notification/src/config.ts index 6acb678609f7..0ba2d9017bcb 100644 --- a/apps/services/user-notification/src/config.ts +++ b/apps/services/user-notification/src/config.ts @@ -2,7 +2,6 @@ import { z } from 'zod' import { processJob } from '@island.is/infra-nest-server' import { defineConfig } from '@island.is/nest/config' -import { isRunningOnEnvironment } from '@island.is/shared/utils' // Exported for testing purposes export const schema = z.object({ @@ -18,26 +17,14 @@ export const schema = z.object({ }), }) - - export const UserNotificationsConfig = defineConfig({ name: 'UserNotificationsApi', schema, - load: (env) => { + load(env) { const isWorker = processJob() === 'worker' - - let firebaseCredentials = env.required('FIREBASE_CREDENTIALS', '') - - // Clean the credentials string for local development - if (isRunningOnEnvironment('local')) { - firebaseCredentials = firebaseCredentials - .replace(/\\n/g, '') // clean newlines - .replace(/\\/g, '') // clean backslashes - } - return { isWorker, - firebaseCredentials, + firebaseCredentials: env.required('FIREBASE_CREDENTIALS', ''), servicePortalClickActionUrl: env.optional('SERVICE_PORTAL_CLICK_ACTION_URL') ?? 'https://island.is/minarsidur', diff --git a/scripts/get-secrets.sh b/scripts/get-secrets.sh index 542ba01d277c..8d5a0ae01c66 100755 --- a/scripts/get-secrets.sh +++ b/scripts/get-secrets.sh @@ -31,6 +31,30 @@ function get-secrets { fi } +function get-secrets { + echo "Fetching secret environment variables for '$*'" + + pre=$(wc -l "$env_secret_file" | awk '{print $1}') + debug "Project '$*' has $pre secrets before render-secrets" + + # Capture output of ts-node command + output=$(ts-node --dir "$ROOT"/infra "$ROOT"/infra/src/cli/cli render-secrets --service="$*") + + # Process each line of output + echo "$output" | while IFS= read -r line; do + # Clean each line: remove newlines and backslashes within the line but keep JSON format intact + cleaned_line=$(echo "$line" | tr -d '\n' | sed 's/\\n/ /g' | sed 's/\\//g') + echo "$cleaned_line" >> "$env_secret_file" + done + + post=$(wc -l "$env_secret_file" | awk '{print $1}') + debug "Project '$*' has $post secrets after render-secrets" + + if [ "$pre" == "$post" ]; then + echo "No secrets found for project '$*'" + fi +} + function aws-check { if ! aws sts get-caller-identity &>/dev/null; then echo "You must be logged in to AWS to fetch secrets" >&2 From d32d5bad541cf7492f658943ad9d106fb66c1e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafn=20=C3=81rnason?= Date: Wed, 11 Dec 2024 15:40:59 +0000 Subject: [PATCH 6/8] cleanup --- libs/shared/utils/src/lib/environment.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/shared/utils/src/lib/environment.ts b/libs/shared/utils/src/lib/environment.ts index d7ca4fab5fc7..edce4ecbee01 100644 --- a/libs/shared/utils/src/lib/environment.ts +++ b/libs/shared/utils/src/lib/environment.ts @@ -37,6 +37,5 @@ if (isServer) { * @param environment */ export const isRunningOnEnvironment = (environment: ActiveEnvironment) => { - console.log('isRunningOnEnvironment', environment, activeEnvironment) return environment === activeEnvironment } From 6b9c861100506b2e35bd439d9e6492642c5e53d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafn=20=C3=81rnason?= Date: Wed, 11 Dec 2024 15:41:29 +0000 Subject: [PATCH 7/8] cleanup --- scripts/get-secrets.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/scripts/get-secrets.sh b/scripts/get-secrets.sh index 8d5a0ae01c66..fdd3d9ee6953 100755 --- a/scripts/get-secrets.sh +++ b/scripts/get-secrets.sh @@ -17,19 +17,6 @@ function show-help() { echo "" } -function get-secrets { - echo "Fetching secret environment variables for '$*'" - - pre=$(wc -l "$env_secret_file") - debug "Project '$*' has $pre secrets before render-secrets" - ts-node --dir "$ROOT"/infra "$ROOT"/infra/src/cli/cli render-secrets --service="$*" >>"$env_secret_file" - post=$(wc -l "$env_secret_file") - debug "Project '$*' has $post secrets after render-secrets" - - if [ "$pre" == "$post" ]; then - echo "No secrets found for project '$*'" - fi -} function get-secrets { echo "Fetching secret environment variables for '$*'" From 8a9876aca9b0c3e78ceff0ea29ed85c93866e593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafn=20=C3=81rnason?= Date: Thu, 12 Dec 2024 10:53:57 +0000 Subject: [PATCH 8/8] move logic to render-secrets --- infra/src/cli/render-secrets.ts | 4 +--- scripts/get-secrets.sh | 17 +++-------------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/infra/src/cli/render-secrets.ts b/infra/src/cli/render-secrets.ts index 9c859e2afba8..0ba1ba705b36 100644 --- a/infra/src/cli/render-secrets.ts +++ b/infra/src/cli/render-secrets.ts @@ -68,9 +68,7 @@ export const renderSecrets = async (service: string) => { logger.debug('env when rendering', { envMap }) Object.entries(envMap).forEach(([key, value]) => { - const escapedValue = (value ?? '') - .replace(/\n/g, '\\n') - .replace(/"/g, '\\"') + const escapedValue = (value ?? '').replace(/'/g, "\\'") console.log(`export ${key}='${escapedValue}'`) }) return envMap diff --git a/scripts/get-secrets.sh b/scripts/get-secrets.sh index fdd3d9ee6953..542ba01d277c 100755 --- a/scripts/get-secrets.sh +++ b/scripts/get-secrets.sh @@ -17,24 +17,13 @@ function show-help() { echo "" } - function get-secrets { echo "Fetching secret environment variables for '$*'" - pre=$(wc -l "$env_secret_file" | awk '{print $1}') + pre=$(wc -l "$env_secret_file") debug "Project '$*' has $pre secrets before render-secrets" - - # Capture output of ts-node command - output=$(ts-node --dir "$ROOT"/infra "$ROOT"/infra/src/cli/cli render-secrets --service="$*") - - # Process each line of output - echo "$output" | while IFS= read -r line; do - # Clean each line: remove newlines and backslashes within the line but keep JSON format intact - cleaned_line=$(echo "$line" | tr -d '\n' | sed 's/\\n/ /g' | sed 's/\\//g') - echo "$cleaned_line" >> "$env_secret_file" - done - - post=$(wc -l "$env_secret_file" | awk '{print $1}') + ts-node --dir "$ROOT"/infra "$ROOT"/infra/src/cli/cli render-secrets --service="$*" >>"$env_secret_file" + post=$(wc -l "$env_secret_file") debug "Project '$*' has $post secrets after render-secrets" if [ "$pre" == "$post" ]; then