fix: development environment setup during the docker build process

.gitattributes

@@ -0,0 +1,6 @@
+# Ensure all shell and ruby scripts have LF line endings
+*.sh text eol=lf
+*.rb text eol=lf
+
+# Ensure .tool-versions has LF line endings
+.tool-versions text eol=lf

.tool-versions

@@ -1,2 +1,2 @@
-nodejs 18.20.3
+nodejs 20.18.0
 ruby 2.7.8

Dockerfile.chemotion-dev

@@ -7,38 +7,57 @@ FROM --platform=linux/amd64 ubuntu:jammy
 
 ARG DEBIAN_FRONTEND=noninteractive
 
-RUN set -xe  && apt-get update -yqqq --fix-missing && apt-get upgrade -y
-RUN apt update && apt-get install -yqq --fix-missing bash ca-certificates wget apt-transport-https git gpg\
-      imagemagick libmagic-dev libmagickcore-dev libmagickwand-dev curl gnupg2 \
-      build-essential sudo postgresql-client libappindicator1 swig \
-      gconf-service libasound2 libgconf-2-4 cmake \
-      libnspr4 libnss3 libpango1.0-0 libxss1 xdg-utils tzdata libpq-dev \
-      gtk2-engines-pixbuf \
-      libssl-dev libreadline-dev\
-      unzip openssh-client \
-      libsqlite3-dev libboost-all-dev p7zip-full \
-      xfonts-cyrillic xfonts-100dpi xfonts-75dpi xfonts-base xfonts-scalable \
-      fonts-crosextra-caladea fonts-crosextra-carlito \
-      fonts-dejavu fonts-dejavu-core fonts-dejavu-extra fonts-liberation2 fonts-liberation \
-      fonts-linuxlibertine fonts-noto-core fonts-noto-extra fonts-noto-ui-core \
-      fonts-opensymbol fonts-sil-gentium fonts-sil-gentium-basic inkscape \
-      libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libxtst6 xauth xvfb
+RUN set -xe \
+    && apt-get update -yqqq --fix-missing \
+    && apt-get upgrade -y
+
+# Install basic dependencies
+RUN apt-get install -yqq --fix-missing \
+        bash ca-certificates wget apt-transport-https git gpg \
+        imagemagick libmagic-dev libmagickcore-dev libmagickwand-dev \
+        curl gnupg2 build-essential sudo postgresql-client \
+        libappindicator1 swig gconf-service libasound2 \
+        libgconf-2-4 cmake libnspr4 libnss3 libpango1.0-0 \
+        libxss1 xdg-utils tzdata libpq-dev gtk2-engines-pixbuf \
+        libssl-dev libreadline-dev unzip openssh-client \
+        libsqlite3-dev libboost-all-dev p7zip-full \
+        xfonts-cyrillic xfonts-100dpi xfonts-75dpi xfonts-base \
+        xfonts-scalable fonts-crosextra-caladea fonts-crosextra-carlito \
+        fonts-dejavu fonts-dejavu-core fonts-dejavu-extra fonts-liberation2 \
+        fonts-liberation fonts-linuxlibertine fonts-noto-core \
+        fonts-noto-extra fonts-noto-ui-core fonts-opensymbol \
+        fonts-sil-gentium fonts-sil-gentium-basic inkscape \
+        libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libxtst6 xauth xvfb jq
+
+# Set up environment variables for gem installation
+ENV GEM_HOME=/home/chemotion-dev/.gem
+ENV PATH=$GEM_HOME/bin:$PATH:/usr/local/bin:/usr/bin:/bin
+
+# Install Ruby and Bundler
+RUN apt-get install -y ruby-full \
+    && gem install bundler --user-install
+
+# Clean up
 RUN apt-get clean \
     && rm -rf /var/lib/apt/lists/*
-RUN useradd -ms /bin/bash chemotion-dev \
-    && echo "chemotion-dev ALL=NOPASSWD: ALL" >> /etc/sudoers
 
-USER chemotion-dev
-WORKDIR /home/chemotion-dev
+# Create a user and set permissions
+RUN useradd -ms /bin/bash chemotion-dev && \
+       echo "chemotion-dev ALL=NOPASSWD: ALL" >> /etc/sudoers
 
 # Create node modules folder OUTSIDE of application directory
-RUN mkdir /home/chemotion-dev/node_modules
+RUN mkdir /home/chemotion-dev/node_modules && \
+    chown -R chemotion-dev:chemotion-dev /home/chemotion-dev
+
+# Switch to the new user
+USER chemotion-dev
+WORKDIR /home/chemotion-dev
 
 SHELL ["/bin/bash", "-c"]
 
 # Even if asdf and the related tools are only installed by running run-ruby-dev.sh, we set the PATH variables here, so when we enter the container via docker exec, we have the path set correctly
 ENV ASDF_DIR=/home/chemotion-dev/.asdf
-ENV PATH=/home/chemotion-dev/.asdf/shims:/home/chemotion-dev/.asdf/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV PATH=/home/chemotion-dev/.asdf/shims:/home/chemotion-dev/.asdf/bin:$GEM_HOME/bin:$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
 RUN echo 'network-timeout 600000' > /home/chemotion-dev/.yarnrc
 # use node modules from outside the application directory

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2024_07_11_120833) do
+ActiveRecord::Schema.define(version: 2024_07_26_064022) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "hstore"
@@ -1778,4 +1778,4 @@
       users
     WHERE ((channels.id = messages.channel_id) AND (messages.id = notifications.message_id) AND (users.id = messages.created_by));
   SQL
-end
+end

docker-compose.dev.yml

@@ -42,7 +42,15 @@ services:
       - 'homedir:/home/chemotion-dev/'
       - '.:/home/chemotion-dev/app'
     working_dir: "/home/chemotion-dev/app"
-    command: "./run-ruby-dev.sh"
+    # Ensure correct LF ending for shell scripts expected by Docker
+    command: >
+      sh -c "
+      {
+        if ! grep -Iq '\r' run-ruby-dev.sh; then
+          echo 'Fixing CRLF line endings';
+          sed -i 's/\r$//' *.sh;
+        fi
+      } && { ./run-ruby-dev.sh; }"
 
   webpacker:
     build:
@@ -56,7 +64,7 @@ services:
       - 'WEBPACKER_DEV_SERVER_PORT=3035'
       - 'NODE_MODULES_PATH=/home/chemotion-dev/node_modules/' # required for the asset pipeline to find the node modules
       - 'WEBPACKER_NODE_MODULES_BIN_PATH=/node_modules/.bin' # required for webpacker to find the yarn binary
-    env_file: ./.env
+    env_file: ./.env.development
     volumes:
       - 'homedir:/home/chemotion-dev/'
       - '.:/home/chemotion-dev/app'

package.json

@@ -154,7 +154,7 @@
   },
   "license": "MIT",
   "engines": {
-    "node": "^18.19.1"
+    "node": "^20.18.0"
   },
   "browserslist": [
     "defaults"

prepare-nodejs.sh

@@ -44,7 +44,12 @@ else
   echo "Node.js updated to version: $REQUIRED_NODE_VERSION"
   asdf reshim nodejs
 fi
+
+# Update package.json engines field to enforce correct Node.js version for yarn installation
+echo ">>> Updating package.json engines field with required Node.js version: $REQUIRED_NODE_VERSION"
+jq ".engines.node = \"^$REQUIRED_NODE_VERSION\"" package.json > tmp.json && mv tmp.json package.json
+
 export REQUIRED_NODE_VERSION
-echo $REQUIRED_NODE_VERSION
+echo "Required Node Version: $REQUIRED_NODE_VERSION"
 
 

prepare-nodejspkg.sh

@@ -23,6 +23,6 @@ else
 fi
 
 echo '>>> Installing JS packages...'
-yarn install --production=false
-yarn install
+yarn install --production=false || { echo 'yarn install failed'; exit 1; }
+yarn install || { echo 'yarn install second run failed'; exit 1; }
 

prepare-ruby-dev.sh

@@ -20,6 +20,9 @@ echo '>>> check nodejs version as set in package.json: install if mismatch, and
 # prepare rails server
 rm -f tmp/pids/server.pid
 
+# prepare database
+source ./prepare_database.sh
+
 if [ "$( psql -h postgres -U postgres -XtAc "SELECT 1 FROM pg_database WHERE datname='chemotion_dev'" )" = '1' ]
 then
     echo "================================================"
@@ -29,7 +32,21 @@ else
     echo "================================================"
     echo "Database does not exist, running 'rake db:setup'"
     echo "================================================"
-    bundle exec rake db:setup
+
+    if rename_config_files; then
+        echo "Configuration DB files renamed successfully."
+        bundle exec rake db:setup
+    else
+        echo "Failed to rename configuration files. Skipping database setup."
+        exit 1
+    fi
 fi
 
+# Any pending migrations after setup?
+if pending_migrations; then
+    echo "Pending migrations found. Running db:migrate..."
+    bundle exec rails db:migrate RAILS_ENV=development
+else
+    echo "No pending migrations."
+fi
 

prepare_database.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Prepare database configuration files by renaming existing examples
+rename_config_files() {
+    local files_to_rename=(
+        "database.yml.example"
+        "shrine.yml.example"
+        "storage.yml.example"
+        "datacollectors.yml.example"
+        # Add any other related file to rename here
+    )
+
+    for file in "${files_to_rename[@]}"; do
+        correct_yml_file="./config/${file%.example}"
+
+        if [ -f "$correct_yml_file" ]; then
+            echo "$correct_yml_file already exists, skipping rename for $file"
+        else
+            if [ -f "./config/$file" ]; then
+                mv "./config/$file" "$correct_yml_file"  # Remove .example from the filename
+                echo "Renamed $file to ${file%.example}"
+            else
+                echo "$file does not exist, skipping rename"
+                return 1
+            fi
+        fi
+    done
+
+    return 0
+}
+
+# Check for pending migrations
+pending_migrations() {
+    if bundle exec rails db:migrate:status | grep -q "down"; then
+        return 0  # exists
+    else
+        return 1  # none
+    fi
+}

run-js-dev.sh

@@ -13,4 +13,11 @@ yarn install
 echo "=========================================================================================================="
 echo "THIS WILL FAIL UNTIL THE RUBY GEMS ARE INSTALLED BY run-ruby-dev.sh. JUST TRY AGAIN AFTER INSTALLING THEM."
 echo "=========================================================================================================="
+
+# Fix line endings for webpacker-dev-server, if exists
+if file ./bin/webpacker-dev-server | grep -q CRLF; then
+    echo "Fixing CRLF line endings in webpacker-dev-server"
+    sed -i 's/\r$//' ./bin/webpacker-dev-server
+fi
+
 ./bin/webpacker-dev-server

run-ruby-dev.sh

@@ -2,4 +2,5 @@
 
 ./prepare-ruby-dev.sh
 
+echo "Starting the Rails server..."
 bundle exec rails s -p 3000 -b 0.0.0.0