build(dev): docker compose using a prebuilt image with latest gems and node packages

.devcontainer/devcontainer.json

@@ -2,7 +2,7 @@
 {
   "name": "Chemotion Dockerfile",
   "dockerComposeFile": [
-    "../docker-compose.dev.yml"
+    "docker-compose.dev.yml"
   ],
   "service": "app",
   "workspaceFolder": "/home/chemotion-dev/app",
@@ -51,14 +51,10 @@
     "/bin/bash",
     ".devcontainer/pre_create.sh"
   ],
-  "postCreateCommand": [
-    "/bin/bash",
-    "prepare-ruby-dev.sh"
-  ],
   "remoteUser": "chemotion-dev", // see https: //aka.ms/vscode-remote/containers/non-root
   "shutdownAction": "stopCompose", // stop compose when quitting
   "overrideCommand": true, // The 'app' container (which vscode is using as the devontainer) dies, as soon as we are killing the rails server that is started in the 'app' container command. To prevent this we override the 'app' container command from the the compose file.
   "containerEnv": {
     "RAILS_ENV": "development"
   },
-}
+}

.devcontainer/pre_create.sh

@@ -1,7 +1,43 @@
 #!/bin/bash
 
+if [ -f .env ]; then
+  echo ".env already exists"
+else
+  if [ -f .env.example ]; then
+    cp .env.example .env
+  else
+    echo "No .env.example file found"
+    touch .env
+  fi
+fi
+
+if [ -f .dockerenv ]; then
+  echo "Using .dockerenv to create .devcontainer/.env"
+  cp .dockerenv .devcontainer/.env
+elif [ -f ./.dockerenv.example ]; then
+  echo "Using .dockerenv.example to create .devcontainer/.env"
+  cp .dockerenv.example .devcontainer/.env
+else
+  echo "Neither .dockerenv nor .dockerenv.example found. Exiting."
+  exit 1
+fi
+
+# Step 2: Check if ./.env exists and append its contents to .devcontainer/.env
+if [ -f .env ]; then
+  echo "Appending .env contents to .devcontainer/.env"
+  cat .env >> .devcontainer/.env
+else
+  echo "No .env file found in the current directory."
+fi
+
+cat .env.development >> .devcontainer/.env
+
+echo ".devcontainer/.env created successfully."
+
+cp docker-compose.dev.yml .devcontainer/docker-compose.dev.yml
+cp Dockerfile.chemotion-dev .devcontainer/Dockerfile.chemotion-dev
+
 # enable configuration files
-cp .env.development .env
 cp public/welcome-message-sample.md public/welcome-message.md
 cp config/datacollectors.yml.example config/datacollectors.yml
 cp config/storage.yml.example config/storage.yml

.dockerenv.example

@@ -0,0 +1,18 @@
+## Environment variables for docker-compose.dev.yml
+## Copy this file to .dockerenv and adjust the values
+## Do not commit .dockerenv to the repository
+## `docker compose --env-file .dockerenv -f docker-compose.dev.yml up`
+##  will use the values from this file
+## Latest available version: https://hub.docker.com/u/complat/dev/tags
+## app image with preinstalled asdf plugins(ruby, nodejs), gems and nodejs packages
+DOCKER_DEV_IMAGE=complat/dev:v1.10.3-37-ga95534401
+#DOCKER_PG_IMAGE=postgres:16
+## Use another named volume for homedir (asdf, gems, etc) or database
+#VOLUME_NAME_HOMEDIR=chemotion_eln_homedir2
+#VOLUME_NAME_DB=chemotion_eln_database2
+
+## Run db migration (rake db:migrate) when starting the app container {always, once, never}
+# always: run db migration on every start
+# once: run db migration only once after the db is created
+# never: never run db migration on start
+RAKE_DB_MIGRATE=once

.gitignore

@@ -32,6 +32,9 @@
 
 .env
 .env.test
+.dockerenv
+/.devcontainer/Dockerfile*
+/.devcontainer/docker-compose*
 
 /config/matrices.json
 /config/mailcollector.yml

.tool-versions

@@ -1,2 +1,2 @@
-nodejs 18.20.4 18.20.3
+nodejs 18.20.6 18.20.5
 ruby 2.7.8

.vscode/tasks.json

@@ -53,6 +53,18 @@
       "label": "mocha - unit tests current file",
       "type": "shell",
       "command": "NODE_PATH=./spec/javascripts:./app/javascript yarn mocha --watch --exit --require '@babel/register' './spec/javascripts/helper/setup.js' '${file}'"
+    },
+    {
+      "label": "Dev Docker Compose Up",
+      "type": "shell",
+      "command": "docker",
+      "args": ["compose", "-f", "docker-compose.dev.yml", "--env-file", ".dockerenv", "up", "-d"]
+    },
+    {
+      "label": "Dev Docker Compose Down",
+      "type": "shell",
+      "command": "docker",
+      "args": ["compose", "-f", "docker-compose.dev.yml", "--env-file", ".dockerenv", "down", "--remove-orphans"]
     }
   ]
 }

Dockerfile.chemotion-dev

@@ -6,6 +6,8 @@
 FROM --platform=linux/amd64 ubuntu:jammy
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG FULL_BUILD=false
+ARG BRANCH=main
 
 RUN set -xe  && apt-get update -yqqq --fix-missing && apt-get upgrade -y
 RUN apt update && apt-get install -yqq --fix-missing bash ca-certificates wget apt-transport-https git gpg\
@@ -39,3 +41,29 @@ SHELL ["/bin/bash", "-c"]
 # Even if asdf and the related tools are only installed by running run-ruby-dev.sh, we set the PATH variables here, so when we enter the container via docker exec, we have the path set correctly
 ENV ASDF_DIR=/home/chemotion-dev/.asdf
 ENV PATH=/home/chemotion-dev/.asdf/shims:/home/chemotion-dev/.asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV NODE_PATH=/home/chemotion-dev/node_modules
+
+RUN if [ "$FULL_BUILD" = "true" ]; then \
+   git clone --single-branch --branch=$BRANCH https://github.com/complat/chemotion_ELN app; \
+  fi
+
+# Install asdf if FULL_BUILD is set to true
+WORKDIR /home/chemotion-dev/app
+RUN if [ "$FULL_BUILD" = "true" ]; then \
+      ./prepare-asdf.sh; \
+    fi
+
+# Install ruby and nodejs if FULL_BUILD is set to true
+RUN if [ "$FULL_BUILD" = "true" ]; then \
+      ./prepare-rubygems.sh; \
+    fi
+
+# Install nodejs packages if FULL_BUILD is set to true
+RUN if [ "$FULL_BUILD" = "true" ]; then \
+      echo -e "--modules-folder ${NODE_PATH}\n" > /home/chemotion-dev/app/.yarnrc; \
+      ./prepare-nodejs.sh; \
+      ./prepare-nodejspkg.sh; \
+    fi
+
+WORKDIR /home/chemotion-dev
+RUN rm -rf /home/chemotion-dev/app

docker-compose.dev.yml

@@ -9,55 +9,68 @@
 # - Run tests:
 #     bundle exec rspec
 #
+# - see .dockerenv.example for environment variables usage
+
+x-common-volumes:
+  - &vhome homedir:/home/chemotion-dev
+  - &vapp .:/home/chemotion-dev/app
+
 services:
   postgres:
-    image: 'postgres:14'
+    image: ${DOCKER_PG_IMAGE:-postgres:14}
     environment:
       - 'POSTGRES_HOST_AUTH_METHOD=trust'
     expose: # expose port to app container
       - '5432'
     ports: # expose port to host machine in case we want to use external db gui tools
       - '5432:5432'
     volumes:
-      - 'database:/var/lib/postgresql/data'
+      - database:/var/lib/postgresql/data
 
   app:
     build:
       context: '.'
       dockerfile: 'Dockerfile.chemotion-dev'
+    image: ${DOCKER_DEV_IMAGE:-}
     depends_on:
       - 'postgres'
     healthcheck:
       test: ["CMD", "bundle", "check"] # exit 0 if all gems from Gemfile are installed, otherwise exit 1
       interval: 30s
       timeout: 10s
+    env_file:
+      - ./.env
     environment:
       - 'SHAKAPACKER_DEV_SERVER_HOST=webpacker'
       - 'SHAKAPACKER_DEV_SERVER_PORT=3035'
       - 'THOR_SILENCE_DEPRECATION=true'
+      - RAKE_DB_MIGRATE=${RAKE_DB_MIGRATE:-never}
     ports: # expose default rails port to host machine
       - "3000:3000"
     volumes:
-      - 'homedir:/home/chemotion-dev/'
-      - '.:/home/chemotion-dev/app'
+      - *vhome
+      - *vapp
     working_dir: "/home/chemotion-dev/app"
     command: "./run-ruby-dev.sh"
 
   webpacker:
     build:
       context: '.'
       dockerfile: 'Dockerfile.chemotion-dev'
+    image: ${DOCKER_DEV_IMAGE:-}
     depends_on:
       app:
         condition: service_healthy
     environment:
       - 'NODE_ENV=development'
       - 'SHAKAPACKER_DEV_SERVER_HOST=webpacker'
       - 'SHAKAPACKER_DEV_SERVER_PORT=3035'
-    env_file: ./.env
+    env_file:
+      - ./.env
+      #- ./.env.development
     volumes:
-      - 'homedir:/home/chemotion-dev/'
-      - '.:/home/chemotion-dev/app'
+      - *vhome
+      - *vapp
     ports: # expose webpacker dev server port to app container
       - '3035:3035'
     expose:
@@ -67,4 +80,6 @@ services:
 
 volumes:
   database:
+    name: ${VOLUME_NAME_DB:-chemotion_eln_database}
   homedir:
+    name: ${VOLUME_NAME_HOMEDIR:-chemotion_eln_homedir}

prepare-nodejs.sh

@@ -7,6 +7,7 @@
 
 set -e
 
+echo '>>> check nodejs version as set in package.json: install if mismatch, and correct .tool-versions'
 # Get the currently installed Node.js version using asdf
 CURRENT_NODE_VERSION=$(asdf current nodejs 2>/dev/null | awk '{print $2}')
 

prepare-nodejspkg.sh

@@ -24,5 +24,4 @@ fi
 
 echo '>>> Installing JS packages...'
 yarn install --production=false
-yarn install
 

prepare-ruby-dev.sh

@@ -7,29 +7,68 @@ export ASDF_BRANCH=v0.14.0
 echo '>>> checking asdf installation'
 ./prepare-asdf.sh
 
-# check nodejs version as set in package.json: install if mismatch, and correct .tool-versions'
-echo '>>> check nodejs version as set in package.json: install if mismatch, and correct .tool-versions'
-./prepare-nodejs.sh
-
 # ruby gems installation
 ./prepare-rubygems.sh
 
 # node packages installation
-./prepare-nodejspkg.sh
+# ./prepare-nodejspkg.sh
 
 # prepare rails server
 rm -f tmp/pids/server.pid
 
-if [ "$( psql -h postgres -U postgres -XtAc "SELECT 1 FROM pg_database WHERE datname='chemotion_dev'" )" = '1' ]
+# assume default database configuration
+DATABASE_NAME=${DATABASE_NAME:-chemotion_dev}
+DATABASE_USER=${DATABASE_USER:-postgres}
+DATABASE_HOST=${DATABASE_HOST:-postgres}
+DATABASE_PORT=${DATABASE_PORT:-5432}
+
+# check if yq is installed
+# if yq is installed parse config/database.yml file for the actual values
+# if yq is not installed, then keep the set values default values
+if command -v yq &> /dev/null
 then
+  DATABASE_NAME=$(yq -r .development.database config/database.yml)
+  DATABASE_USER=$(yq -r .development.username config/database.yml)
+  DATABASE_HOST=$(yq -r .development.host config/database.yml)
+  DATABASE_PORT=$(yq -r .development.port config/database.yml)
+fi
+echo "DATABASE_NAME: $DATABASE_NAME"
+echo "DATABASE_USER: $DATABASE_USER"
+echo "DATABASE_HOST: $DATABASE_HOST"
+echo "DATABASE_PORT: $DATABASE_PORT"
+
+# check if the database for the given environment configuration exists
+db_exists=$( psql -h $DATABASE_HOST -U $DATABASE_USER -p $DATABASE_PORT -XtAc "SELECT 1 FROM pg_database WHERE datname='$DATABASE_NAME'" )
+echo "Database exists: $db_exists"
+if [ "$db_exists" = '1' ]
+then
+  echo "==================================================="
+  echo "Database already exists, skipping Database creation"
+  echo "==================================================="
+  if [ "$RAKE_DB_MIGRATE" = "always" ]
+  then
     echo "================================================"
-    echo "Database already exists, skipping Database setup"
+    echo "Running 'rake db:migrate'"
     echo "================================================"
+      bundle exec rake db:migrate
+  fi
 else
+  # if RAKE_DB_MIGRATE is set to always or once, run rake db:setup
+  if [ "$RAKE_DB_MIGRATE" = "always" ] || [ "$RAKE_DB_MIGRATE" = "once" ]
+  then
+    echo "================================================"
+    echo "Database does not exist"
+    echo "running 'rake db:create/migrate/seed'"
+    echo "================================================"
+    bundle exec rake db:create
+    bundle exec rake db:migrate
+    bundle exec rake db:seed
+  else
     echo "================================================"
     echo "Database does not exist, running 'rake db:setup'"
     echo "================================================"
     bundle exec rake db:setup
+  fi
 fi
 
 

run-js-dev.sh

@@ -1,14 +1,20 @@
 #!/bin/bash
 
-if command -v yarn; then
-    echo '>>> yarn is installed -> continue'
+./prepare-nodejs.sh
+
+# if NODE_PATH is set, add --modules-folder option to .yarnrc and create a symlink to the node_modules folder
+if [ -z "$NODE_PATH" ]; then
+    echo ">>> NODE_PATH is not set"
 else
-    echo '>>> Missing yarn. Installing...'
-    npm install -g yarn
+    echo ">>> NODE_PATH is set to $NODE_PATH"
+#    echo ">>> Adding --modules-folder option to .yarnrc"
+#    echo -e "--modules-folder $NODE_PATH" > .yarnrc
+# create a symlink unless it already exists
+    [ -L "${HOME}/app/node_modules" ] || ln -s  "${NODE_PATH}/" "${HOME}/app/node_modules" 
 fi
 
-echo '>>> Installing JS packages...'
-yarn install
+
+./prepare-nodejspkg.sh
 
 echo "=========================================================================================================="
 echo "THIS WILL FAIL UNTIL THE RUBY GEMS ARE INSTALLED BY run-ruby-dev.sh. JUST TRY AGAIN AFTER INSTALLING THEM."