Merge pull request #674 from zurdi15/3.0-rc-7

Changes for RC7

DEVELOPER-SETUP.md

@@ -42,8 +42,7 @@ pipx install --suffix _npm git+https://github.com/radoering/poetry.git@non-packa
 
 More info: https://github.com/python-poetry/poetry/pull/8650
 
-
-Then creat the virtual environment
+Then create the virtual environment
 
 ```sh
 # Fix disable parallel installation stuck: $> poetry_npm config experimental.new-installer false

backend/config/__init__.py

@@ -26,9 +26,8 @@
 DB_NAME: Final = os.environ.get("DB_NAME", "romm")
 
 # REDIS
-REDIS_HOST: Final = os.environ.get("REDIS_HOST", "localhost")
-REDIS_PORT: Final = os.environ.get("REDIS_PORT", "6379")
-REDIS_PASSWORD: Final = os.environ.get("REDIS_PASSWORD")
+REDIS_HOST: Final = "127.0.0.1"
+REDIS_PORT: Final = 6379
 
 # IGDB
 IGDB_CLIENT_ID: Final = os.environ.get(

backend/endpoints/user.py

@@ -149,7 +149,7 @@ def update_user(
             "hashed_password"
         )
         if request.user.id == id and creds_updated:
-            auth_handler.clear_session(request)
+            request.session.clear()
 
     return db_user_handler.get_user(id)
 

backend/handler/auth_handler/__init__.py

@@ -77,15 +77,15 @@ async def get_current_active_user_from_session(self, conn: HTTPConnection):
         # Key exists therefore user is probably authenticated
         user = db_user_handler.get_user_by_username(username)
         if user is None:
-            conn.session.clear()
+            conn.session = {}
 
             raise HTTPException(
                 status_code=status.HTTP_403_FORBIDDEN,
                 detail="User not found",
             )
 
         if not user.enabled:
-            conn.session.clear()
+            conn.session = {}
 
             raise HTTPException(
                 status_code=status.HTTP_403_FORBIDDEN, detail="Inactive user"

backend/handler/redis_handler.py

@@ -1,7 +1,7 @@
 import sys
 from enum import Enum
 
-from config import REDIS_HOST, REDIS_PASSWORD, REDIS_PORT
+from config import REDIS_HOST, REDIS_PORT
 from logger.logger import log
 from redis import Redis
 from rq import Queue
@@ -39,15 +39,8 @@ def __str__(self) -> str:
         return repr(self)
 
 
-redis_client = Redis(
-    host=REDIS_HOST, port=int(REDIS_PORT), password=REDIS_PASSWORD, db=0
-)
-
-redis_url = (
-    f"redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}"
-    if REDIS_PASSWORD
-    else f"redis://{REDIS_HOST}:{REDIS_PORT}"
-)
+redis_client = Redis(host=REDIS_HOST, port=REDIS_PORT, db=0)
+redis_url = f"redis://{REDIS_HOST}:{REDIS_PORT}"
 
 high_prio_queue = Queue(name=QueuePrio.HIGH.value, connection=redis_client)
 default_queue = Queue(name=QueuePrio.DEFAULT.value, connection=redis_client)
@@ -60,8 +53,7 @@ def __str__(self) -> str:
     # A seperate client that auto-decodes responses is needed
     cache = Redis(
         host=REDIS_HOST,
-        port=int(REDIS_PORT),
-        password=REDIS_PASSWORD,
+        port=REDIS_PORT,
         db=0,
         decode_responses=True,
     )

docker-compose.yml

@@ -17,6 +17,6 @@ services:
     container_name: redis
     restart: unless-stopped
     ports:
-      - ${REDIS_PORT}:6379
+      - $REDIS_PORT:6379
     env_file:
       - .env

docker/Dockerfile

@@ -23,7 +23,8 @@ RUN apk add --upgrade \
         libffi \
         mariadb-connector-c \
         netcat-openbsd \
-        python3
+        python3 \
+        tzdata
 
 # Install additional build dependencies
 RUN apk add --upgrade \
@@ -33,7 +34,13 @@ RUN apk add --upgrade \
         musl-dev \
         python3-dev \
         py3-pip \
-        git
+        git \
+        wget \
+        coreutils \
+        dpkg-dev dpkg \
+        linux-headers \
+        make \
+        openssl-dev
 
 # Create python venv to not clash with OS python packages
 RUN python3 -m venv /backend/
@@ -56,14 +63,68 @@ RUN . /backend/bin/activate && \
 # cleanup python dependencies that are not needed anymore
 RUN . /backend/bin/activate && \
     grep -v -x -f /installed_poetry_requirements.txt /installed_pip_requirements.txt > /build_requirements.txt && \
-    pip uninstall -y -r /build_requirements.txt 
+    pip uninstall -y -r /build_requirements.txt
 
 COPY ./backend /backend
 
 # Setup init script and config files
 COPY ./docker/init_scripts/* /
 COPY ./docker/nginx/default.conf /etc/nginx/nginx.conf
 
+# Install redis
+ENV REDIS_VERSION 7.2.4
+ENV REDIS_DOWNLOAD_URL http://download.redis.io/releases/redis-7.2.4.tar.gz
+ENV REDIS_DOWNLOAD_SHA 8d104c26a154b29fd67d6568b4f375212212ad41e0c2caa3d66480e78dbd3b59
+
+RUN wget -O redis.tar.gz "$REDIS_DOWNLOAD_URL"; \
+	echo "$REDIS_DOWNLOAD_SHA *redis.tar.gz" | sha256sum -c -; \
+	mkdir -p /usr/src/redis; \
+	tar -xzf redis.tar.gz -C /usr/src/redis --strip-components=1; \
+	rm redis.tar.gz; \
+    # disable Redis protected mode as it is unnecessary in context of Docker
+    # (ports are not automatically exposed when running inside Docker, but rather explicitly by specifying -p / -P)
+	grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' /usr/src/redis/src/config.c; \
+	sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' /usr/src/redis/src/config.c; \
+	grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' /usr/src/redis/src/config.c; \
+    # for future reference, we modify this directly in the source instead of just supplying a default configuration flag because apparently "if you specify any argument to redis-server, [it assumes] you are going to specify everything"
+    # (more exactly, this makes sure the default behavior of "save on SIGTERM" stays functional by default)
+	# (also, we do cross-builds, so we need to embed the appropriate "--build=xxx" values to that "./configure" invocation)
+    gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \
+	extraJemallocConfigureFlags="--build=$gnuArch"; \
+	dpkgArch="$(dpkg --print-architecture)"; \
+    case "${dpkgArch##*-}" in (amd64 | i386 | x32) extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-page=12";; (*) extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-page=16";; esac; \
+	extraJemallocConfigureFlags="$extraJemallocConfigureFlags --with-lg-hugepage=21"; \
+	grep -F 'cd jemalloc && ./configure ' /usr/src/redis/deps/Makefile; \
+	sed -ri 's!cd jemalloc && ./configure !&'"$extraJemallocConfigureFlags"' !' /usr/src/redis/deps/Makefile; \
+	grep -F "cd jemalloc && ./configure $extraJemallocConfigureFlags " /usr/src/redis/deps/Makefile; \
+	export BUILD_TLS=yes; \
+	make -C /usr/src/redis -j "$(nproc)" all; \
+	make -C /usr/src/redis install; \
+	serverMd5="$(md5sum /usr/local/bin/redis-server | cut -d' ' -f1)"; export serverMd5; \
+	find /usr/local/bin/redis* -maxdepth 0 \
+		-type f -not -name redis-server \
+		-exec sh -eux -c ' \
+			md5="$(md5sum "$1" | cut -d" " -f1)"; \
+			test "$md5" = "$serverMd5"; \
+		' -- '{}' ';' \
+		-exec ln -svfT 'redis-server' '{}' ';' \
+	; \
+	rm -r /usr/src/redis; \
+	runDeps="$( \
+		scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \
+			| tr ',' '\n' \
+			| sort -u \
+			| awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
+	)"; \
+	apk add --no-network --virtual .redis-rundeps $runDeps; \
+	apk del --no-network .build-deps; \
+	redis-cli --version; \
+	redis-server --version; \
+	echo '{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT","name":"redis-server-sbom","packages":[{"name":"redis-server","versionInfo":"7.2.4","SPDXID":"SPDXRef-Package--redis-server","externalRefs":[{"referenceCategory":"PACKAGE-MANAGER","referenceType":"purl","referenceLocator":"pkg:generic/[email protected]?os_name=alpine&os_version=3.19"}],"licenseDeclared":"BSD-3-Clause"}]}' > /usr/local/redis.spdx.json
+
+RUN mkdir /redis-data && chown 1000:1000 /redis-data
+VOLUME /redis-data
+
 # cleanup additional build dependencies
 RUN apk del \
         gcc \
@@ -72,7 +133,13 @@ RUN apk del \
         musl-dev \
         python3-dev \
         py3-pip \
-        git
+        git \
+        wget \
+        coreutils \
+        dpkg-dev dpkg \
+        linux-headers \
+        make \
+        openssl-dev
 
 # cleanup leftover files that are not needed at runtime
 RUN rm -r \
@@ -92,5 +159,6 @@ RUN mkdir /romm && chown 1000:1000 /romm -R
 
 # Expose ports and start
 EXPOSE 8080
+EXPOSE 6379/tcp
 WORKDIR /romm
 CMD ["/init"]

docker/init_scripts/init

@@ -56,6 +56,19 @@ start_bin_nginx () {
     fi
 }
 
+start_bin_redis-server () {
+    info_log "starting redis-server"
+
+    # Check if /usr/local/etc/redis/redis.conf exists and use it if so
+    if [ -f /usr/local/etc/redis/redis.conf ]; then
+        redis-server /usr/local/etc/redis/redis.conf &
+    else
+        redis-server &
+    fi
+    REDIS_PID=$!
+    echo $REDIS_PID > /tmp/redis-server.pid
+}
+
 # function that runs our independent python scripts and creates corresponding PID files,
 start_python () {
     SCRIPT="${1}"
@@ -93,6 +106,9 @@ cd /backend || { error_log "/backend directory doesn't seem to exist"; }
 
 # function definition done, lets start our main loop
 while true; do
+    # Start redis server if we dont have a corresponding PID file
+    watchdog_process_pid bin redis-server
+
     # Run needed database migrations on startup,
     # but only if it was not successful since the last full docker container start
     if [[ ${ALEMBIC_SUCCESS:="false"} == "false" ]]; then

examples/docker-compose.example.yml

@@ -10,8 +10,7 @@ services:
     container_name: romm
     restart: unless-stopped
     environment:
-      - DB_HOST=mariadb
-      - DB_PORT=3306
+      - DB_HOST=romm-db
       - DB_NAME=romm # Should match MYSQL_DATABASE in mariadb
       - DB_USER=romm-user # Should match MYSQL_USER in mariadb
       - DB_PASSWD= # Should match MYSQL_PASSWORD in mariadb
@@ -20,23 +19,20 @@ services:
       - ROMM_AUTH_SECRET_KEY= # Generate a key with `openssl rand -hex 32`
       - ROMM_AUTH_USERNAME=admin
       - ROMM_AUTH_PASSWORD= # default: admin
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
     volumes:
-      - romm_resources:/romm/resources" # Resources fetched from IGDB (covers, screenshots, etc.)
-      - "/path/to/library:/romm/library" # Your game library
-      - "/path/to/assets:/romm/assets" # Uploaded saves, states, etc.
-      - "/path/to/config:/romm/config" # [Optional] Path where config.yml is stored
-      - "/path/to/logs:/romm/logs" # [Optional] Path where logs are stored
+      - romm_resources:/romm/resources # Resources fetched from IGDB (covers, screenshots, etc.)
+      - /path/to/library:/romm/library # Your game library
+      - /path/to/assets:/romm/assets # Uploaded saves, states, etc.
+      - /path/to/config:/romm/config # [Optional] Path where config.yml is stored
+      - /path/to/logs:/romm/logs # [Optional] Path where logs are stored
     ports:
       - 80:8080
     depends_on:
-      - mariadb
-      - redis
+      - romm-db
 
-  mariadb:
+  romm-db:
     image: mariadb:latest
-    container_name: mariadb
+    container_name: romm-db
     restart: unless-stopped
     environment:
       - MYSQL_ROOT_PASSWORD= # Use a unique, secure password
@@ -45,12 +41,3 @@ services:
       - MYSQL_PASSWORD=
     volumes:
       - mysql_data:/var/lib/mysql
-    ports:
-      - 3306:3306
-
-  redis:
-    image: redis:alpine
-    container_name: redis
-    restart: unless-stopped
-    ports:
-      - 6379:6379

unraid_template/romm.xml

@@ -39,9 +39,6 @@
    <Config Name="DB_USER" Target="DB_USER" Default="romm" Mode="" Description="Database user" Type="Variable" Display="advanced" Required="true" Mask="false"/>
    <Config Name="DB_NAME" Target="DB_NAME" Default="romm" Mode="" Description="Database name" Type="Variable" Display="advanced" Required="true" Mask="false"/>
    <Config Name="DB_PASSWD" Target="DB_PASSWD" Default="" Mode="" Description="Database password for DB_USER" Type="Variable" Display="advanced" Required="true" Mask="true"/>
-   <Config Name="REDIS_HOST" Target="REDIS_HOST" Default="127.0.0.1" Mode="" Description="Redis host" Type="Variable" Display="advanced" Required="true" Mask="false"/>
-   <Config Name="REDIS_PORT" Target="REDIS_PORT" Default="6379" Mode="" Description="Redis port" Type="Variable" Display="advanced" Required="true" Mask="false"/>
-   <Config Name="REDIS_PASSWORD" Target="REDIS_PASSWORD" Default="" Mode="" Description="Redis password" Type="Variable" Display="advanced" Required="false" Mask="false"/>
    <Config Name="ROMM_AUTH_USERNAME" Target="ROMM_AUTH_USERNAME" Default="admin" Mode="" Description="Default admin username" Type="Variable" Display="advanced" Required="false" Mask="false"/>
    <Config Name="ROMM_AUTH_PASSWORD" Target="ROMM_AUTH_PASSWORD" Default="" Mode="" Description="Default admin password" Type="Variable" Display="advanced" Required="false" Mask="true"/>
    <Config Name="ROMM_AUTH_SECRET_KEY" Target="ROMM_AUTH_SECRET_KEY" Default="" Mode="" Description="Generate a key with `openssl rand -hex 32`" Type="Variable" Display="advanced" Required="false" Mask="true"/>