Create integrated config (#2846)

* add init scripts for aws and azure

* create constant s3Client to be shared

* If there are 0 pages show 1 page

* add all services to orchestration's docker-compose.yml

* create single integrated.json for orchestration -> ecr-viewer

* Revert "If there are 0 pages show 1 page"

This reverts commit 1d2ac53.

* remove init script for azure

* seperate docker compose db and seed scripts

* Remove save source from ecr-viewer-config to allow it to be set by user

* install dotenvx

* remove quotes in environment variables :(

* remove unused check-sourc-env.mjs

* replace ecr viewer's docker compose

* replace seed script docker-compose

* update orchestration config to extend main docker-compose

* update ECR_VIEWER_URL in orchestration and update paths

* add .env.sample

* fix ecr viewer in orchestration docker compose

* add profiles to aws and azure

* remove .env.e2e

* fix lighthouse app

* rename step to be start app

* skip lighthouse since currently it doesn't do anything anyways

* update readme

* update dotenv command

* fix design review

* add all profiles to ecr-viewer and some to postgres

* Update containers/orchestration/docker-compose.yml

* move entrypoint to seed-scripts

* fix the down commands in convert-seed-data

* update orchestration env file location

* update run command to just use docker

.github/workflows/container-ecr-viewer.yaml

@@ -111,8 +111,8 @@ jobs:
         with:
           node-version: ${{env.NODE_VERSION}}
 
-      - name: Build app
-        run: docker compose --env-file .env.test up -d
+      - name: Start app
+        run: docker compose --env-file .env.test --profile postgres --profile ecr-viewer up -d
         working-directory: ./containers/${{env.CONTAINER}}
 
       - name: Wait for Application to be ready
@@ -122,11 +122,11 @@ jobs:
             sleep 5
           done
 
-      - name: Run Lighthouse
-        uses: treosh/lighthouse-ci-action@v12
-        with:
-          configPath: "./lighthouserc.yaml"
-          uploadArtifacts: true
+      # - name: Run Lighthouse
+      #   uses: treosh/lighthouse-ci-action@v12
+      #   with:
+      #     configPath: "./lighthouserc.yaml"
+      #     uploadArtifacts: true
 
   build-container:
     runs-on: ubuntu-latest

containers/ecr-viewer/.env.sample

@@ -0,0 +1,24 @@
+APP_ENV=test
+NEXT_PUBLIC_NON_INTEGRATED_VIEWER=true
+SOURCE=postgres
+
+# Azure Connection
+AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;"
+AZURE_CONTAINER_NAME=ecr-viewer-files
+
+# AWS Connection
+AWS_REGION=us-east-1
+AWS_CUSTOM_ENDPOINT=http://localhost:4566
+ECR_BUCKET_NAME=ecr-viewer-files
+AWS_ACCESS_KEY_ID=123
+AWS_SECRET_ACCESS_KEY=xyz
+
+# Metadata
+DATABASE_URL=postgres://postgres:pw@localhost:5432/ecr_viewer_db
+METADATA_DATABASE_TYPE=postgres
+METADATA_DATABASE_SCHEMA=core
+
+# SQL Server
+SQL_SERVER_USER=sa
+SQL_SERVER_PASSWORD=Password1!
+SQL_SERVER_SERVER=sqlserver

containers/ecr-viewer/README.md

@@ -73,7 +73,7 @@ The eCR Viewer is primailly deveoped on Mac silicon machines, See this [integrea
 Sample eICRs are included in `containers/ecr-viewer/seed-scripts/baseECR/`. If you ever need to update the eCRs or add new eCRs you can regenerate the data by:
 
 1. Run `npm run convert-seed-data` to re-run the FHIR conversion of the seed eCRs
-2. Delete the current volume used by your DB: `docker compose -f ./seed-scripts/docker-compose.yml --profile "*" down -v`
+2. Delete the current volume used by your DB: `docker compose -f ./docker-compose.yml --profile "*" down -v`
 3. Run `npm run local-dev` to re-run the eCR Viewer with the newly converted data.
 
 ### Developer Commands

containers/ecr-viewer/design-review/design-review.sh

@@ -56,6 +56,11 @@ if ! command_exists docker; then
     brew install --cask docker
 fi
 
+# Install NPM if it's not already installed
+if ! command_exists npm; then
+    brew install node
+fi
+
 # Start Docker
 open /Applications/Docker.app
 echo "Waiting for Docker to launch..."
@@ -80,23 +85,23 @@ cd ./containers/ecr-viewer
 # Checkout the specified branch
 git checkout $BRANCH_NAME
 
+npm i
+
 # Write env vars to .env.local
-echo "APP_ENV=test" > .env.local
-echo "DATABASE_URL=postgres://postgres:pw@db:5432/ecr_viewer_db" >> .env.local
+npm run setup-local-env
 echo "NEXT_PUBLIC_NON_INTEGRATED_VIEWER=$IS_NON_INTEGRATED" >> .env.local
 
 # Run FHIR conversion on seed data
 if [ "$CONVERT_SEED_DATA" = true ]; then
   echo "Running seed data FHIR conversion..."
-
-  docker compose -f ./seed-scripts/docker-compose.yml --profile design-review down -v
-  docker compose -f ./seed-scripts/docker-compose.yml --profile seed-postgres --env-file .env.local up --abort-on-container-exit
+  docker compose -f ./docker-compose.yml --profile "*" down -v
+  npm run convert-seed-data:build
 else
   echo "Skipping seed data FHIR conversion..."
 fi
 
-# Build and run docker compose
-docker compose -f ./seed-scripts/docker-compose.yml --profile design-review --env-file .env.local up -d --build
+# Run ecr viewer
+npx @dotenvx/dotenvx run -f .env.local -- sh -c 'docker compose -f ./docker-compose.yml --profile ${SOURCE} --profile ${METADATA_DATABASE_TYPE} --profile ecr-viewer up -d'
 
 # Wait for eCR Viewer to be available
 URL="http://localhost:3000/ecr-viewer"
@@ -110,5 +115,4 @@ open http://localhost:3000/ecr-viewer
 
 # Prompt to end review session
 read -p "Press enter to end review"
-docker compose -f ./seed-scripts/docker-compose.yml --profile design-review down
-docker compose -f ./seed-scripts/docker-compose.yml --profile seed-postgres down
+docker compose -f ./docker-compose.yml --profile "*" down

containers/ecr-viewer/docker-compose.yml

@@ -1,12 +1,36 @@
 services:
+  ecr-viewer:
+    build:
+      context: ../../
+      dockerfile: ./containers/ecr-viewer/Dockerfile
+    ports:
+      - "3000:3000"
+    environment:
+      - APP_ENV=${APP_ENV:-test}
+      - METADATA_DATABASE_TYPE=${METADATA_DATABASE_TYPE:-postgres}
+      - SOURCE=${SOURCE:-postgres}
+      - DATABASE_URL=postgres://postgres:pw@postgres:5432/ecr_viewer_db
+      - AWS_CUSTOM_ENDPOINT=http://aws-storage:4566
+      - AWS_REGION=${AWS_REGION:-us-east-1}
+      - ECR_BUCKET_NAME=${ECR_BUCKET_NAME:-ecr-viewer-files}
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-123}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-xyz}
+      - AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-storage:10000/devstoreaccount1;
+      - AZURE_CONTAINER_NAME=${AZURE_CONTAINER_NAME:-ecr-viewer-files}
+      - NEXT_PUBLIC_NON_INTEGRATED_VIEWER=${NEXT_PUBLIC_NON_INTEGRATED_VIEWER:-false}
+      - NEXT_PUBLIC_BASEPATH=${NEXT_PUBLIC_BASEPATH:-/ecr-viewer}
+    profiles:
+      - ecr-viewer
+
   postgres:
     image: "postgres:alpine"
     ports:
       - "5432:5432"
     volumes:
-      - ./sql/core.sql:/docker-entrypoint-initdb.d/core.sql
       - ./seed-scripts/sql/01-init.sql:/docker-entrypoint-initdb.d/01-init.sql
+      - ./sql/core.sql:/docker-entrypoint-initdb.d/core.sql
       - ./seed-scripts/sql/.pgpass/:/usr/local/lib/.pgpass
+      - db:/var/lib/postgresql/data
     environment:
       - POSTGRES_USER=postgres
       - PGUSER=postgres
@@ -17,6 +41,9 @@ services:
       interval: 2s
       timeout: 5s
       retries: 20
+    profiles:
+      - postgres
+
   sqlserver:
     image: mcr.microsoft.com/mssql/server:2022-latest
     environment:
@@ -25,7 +52,7 @@ services:
     ports:
       - "1433:1433"
     volumes:
-      - ./sql:/var/opt/mssql/scripts
+      - ./seed-scripts/sql:/var/opt/mssql/scripts
     entrypoint: /var/opt/mssql/scripts/entrypoint.sh
     healthcheck:
       test:
@@ -36,60 +63,52 @@ services:
       interval: 20s
       timeout: 10s
       retries: 5
-  # Next.js app
-  ecr-viewer:
-    build:
-      context: ../../
-      dockerfile: ./containers/ecr-viewer/Dockerfile
-    ports:
-      - "3000:3000"
-    environment:
-      - DATABASE_URL=${DATABASE_URL:-postgres://postgres:pw@db:5432/ecr_viewer_db}
-      - APP_ENV=${APP_ENV:-prod}
+    profiles:
+      - sqlserver
 
-  jaeger:
-    image: jaegertracing/all-in-one:latest
-    container_name: "jaeger"
-    volumes:
-      - "./jaeger-ui.json:/etc/jaeger/jaeger-ui.json"
-    command: --query.ui-config /etc/jaeger/jaeger-ui.json
-    ports:
-      - "16686:16686" # serves frontend and queries
-      - "14268"
-      - "14250"
-  prometheus:
-    image: prom/prometheus:latest
+  aws-storage:
+    image: gresau/localstack-persist:3
     ports:
-      - "9090:9090"
+      - "4566:4566"
     volumes:
-      - "./prometheus.yml:/etc/prometheus/prometheus.yml"
-      - "prom_data:/prometheus"
-  otel-collector:
-    image: otel/opentelemetry-collector-contrib:latest
-    container_name: "otel-collector"
+      - aws-data:/persisted-data
+      - ./seed-scripts/aws:/etc/localstack/init/ready.d
+    profiles:
+      - s3
+
+  azure-storage:
+    image: mcr.microsoft.com/azure-storage/azurite
+    hostname: azurite
+    restart: always
+    command: "azurite --blobHost 0.0.0.0 --blobPort 10000 -l data --debug /data/debug.log"
     volumes:
-      - "./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml"
-    command: ["--config=/etc/otelcol-contrib/config.yaml"]
-    ports:
-      - "8888:8888" # Prometheus metrics exposed by collector
-      - "8889:8889" # Endpoint prometheus will scrape
-      - "4317:4317" # gRPC OTLP receiver
-      - "4318:4318" # HTTP OTLP receiver
-    depends_on:
-      - jaeger
-      - prometheus
-  grafana:
-    image: grafana/grafana-oss
+      - azure-data:/opt/azurite/data
+    healthcheck:
+      test: nc 127.0.0.1 10000 -z
+      interval: 1s
+      retries: 30
     ports:
-      - "4000:3000"
-    volumes:
-      - ./grafana.ini:/etc/grafana/grafana.ini
-      - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
-      - ./grafana/datasources/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml
-      - grafana_data:/var/lib/grafana
+      - "10000:10000"
+    profiles:
+      - azure
+
+  azure-storage-init:
+    image: mcr.microsoft.com/azure-cli:latest
+    command:
+      - /bin/sh
+      - -c
+      - |
+        az storage container create --name ecr-viewer-files
     depends_on:
-      - prometheus
+      azure-storage:
+        condition: service_healthy
+    environment:
+      # https://github.com/Azure/Azurite/blob/main/README.md#usage-with-azure-storage-sdks-or-tools
+      AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure-storage:10000/devstoreaccount1;
+    profiles:
+      - azure
 
 volumes:
-  prom_data:
-  grafana_data:
+  db:
+  azure-data:
+  aws-data:

containers/ecr-viewer/package.json

@@ -5,14 +5,15 @@
   "private": true,
   "scripts": {
     "dev": "next dev",
-    "local-dev": "docker compose -f ./seed-scripts/docker-compose.yml --profile local-dev-$(node ./check-source-env.mjs) up -d && npm run dev",
+    "local-dev": "npx @dotenvx/dotenvx run -f .env.local -- sh -c 'docker compose -f ./docker-compose.yml --profile ${SOURCE} --profile ${METADATA_DATABASE_TYPE} up -d && npm run dev'",
     "setup-local-env": "./setup-env.sh",
     "build": "next build",
     "start": "next start",
     "lint": "next lint",
     "test": "TZ=America/New_York jest",
     "test:watch": "TZ=America/New_York jest --watch",
-    "convert-seed-data": ". ./.env.local && docker compose -f ./seed-scripts/docker-compose.yml --profile seed-$(node ./check-source-env.mjs) up --abort-on-container-exit",
+    "convert-seed-data": "npx @dotenvx/dotenvx run -f .env.local -- sh -c 'docker compose -f ./docker-compose.yml --profile ${SOURCE} --profile ${METADATA_DATABASE_TYPE} --profile ecr-viewer up -d && docker compose -f ./seed-scripts/docker-compose-seed.yml up --abort-on-container-exit && docker compose -f ./docker-compose.yml --profile \"*\" down && docker compose -f ./seed-scripts/docker-compose-seed.yml --profile \"*\" down'",
+    "convert-seed-data:build": "npx @dotenvx/dotenvx run -f .env.local -- sh -c 'docker compose -f ./docker-compose.yml --profile ${SOURCE} --profile ${METADATA_DATABASE_TYPE} --profile ecr-viewer up -d --build && docker compose -f ./seed-scripts/docker-compose-seed.yml up --abort-on-container-exit --build && docker compose -f ./docker-compose.yml --profile \"*\" down && docker compose -f ./seed-scripts/docker-compose-seed.yml --profile \"*\" down'",
     "update-cypress-data": "docker compose -f ./seed-scripts/docker-compose-create-sql.yml up --build --abort-on-container-exit",
     "cypress:open": "cypress open",
     "cypress:run": "cypress run",
@@ -63,6 +64,7 @@
     "sanitize-html": "^2.13.1"
   },
   "devDependencies": {
+    "@dotenvx/dotenvx": "^1.22.0",
     "@next/eslint-plugin-next": "^14.2.5",
     "@smithy/util-stream": "^2.1.2",
     "@testing-library/jest-dom": "^6.1.5",
@@ -81,7 +83,6 @@
     "aws-sdk-client-mock": "^3.1.0",
     "concurrently": "^9.0.1",
     "cypress": "^13.6.6",
-    "dotenv": "^16.4.5",
     "eslint": "^8.56.0",
     "eslint-config-next": "14.0.3",
     "eslint-config-prettier": "^9.1.0",

containers/ecr-viewer/seed-scripts/aws/init.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+awslocal s3 mb s3://ecr-viewer-files

containers/ecr-viewer/seed-scripts/docker-compose-seed.yml

@@ -0,0 +1,70 @@
+services:
+  fhir-converter-service:
+    build:
+      context: ../../fhir-converter/
+    ports:
+      - "8082:8080"
+
+  ingestion-service:
+    platform: linux/amd64
+    build:
+      context: ../../ingestion
+    ports:
+      - "8083:8080"
+    logging:
+      driver: "json-file"
+    environment:
+      SMARTY_AUTH_ID: ${SMARTY_AUTH_ID}
+      SMARTY_AUTH_TOKEN: ${SMARTY_AUTH_TOKEN}
+
+  message-parser-service:
+    platform: linux/amd64
+    build:
+      context: ../../message-parser
+    ports:
+      - "8085:8080"
+    logging:
+      driver: "json-file"
+
+  trigger-code-reference-service:
+    platform: linux/amd64
+    build:
+      context: ../../trigger-code-reference
+    ports:
+      - "8086:8080"
+    logging:
+      driver: "json-file"
+
+  orchestration-service:
+    platform: linux/amd64
+    build:
+      context: ../../orchestration/
+    depends_on:
+      - fhir-converter-service
+      - ingestion-service
+      - message-parser-service
+    ports:
+      - "8080:8080"
+    logging:
+      driver: "json-file"
+    env_file:
+      - ../../orchestration/.env
+    environment:
+      - OTEL_TRACES_EXPORTER=none
+      - OTEL_METRICS_EXPORTER=none
+      - ECR_VIEWER_URL=http://host.docker.internal:3000/ecr-viewer
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://orchestration-service:8080/"]
+      interval: 10s
+      timeout: 20s
+      retries: 5
+
+  convert-data:
+    build: .
+    depends_on:
+      orchestration-service:
+        condition: service_healthy
+    ports:
+      - "8081:8081"
+    volumes:
+      - ./baseECR:/code/baseECR