Merge branch 'dev'. Bring gRPC, Organizer, folders to 'master'.

.devcontainer/Dockerfile

@@ -1,10 +1,8 @@
-# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=16-bullseye
-FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:0-${VARIANT}
+ARG VARIANT=20-bookworm
+FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:${VARIANT}
 
-# [Optional] Uncomment this section to install additional OS packages.
-# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-#     && apt-get -y install --no-install-recommends <your-package-list-here>
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends protobuf-compiler libprotobuf-dev
 
 # [Optional] Uncomment if you want to install an additional version of node using nvm
 # ARG EXTRA_NODE_VERSION=10

.devcontainer/base.Dockerfile

@@ -1,6 +1,5 @@
-# [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster
-ARG VARIANT=16-bullseye
-FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:0-${VARIANT}
+ARG VARIANT=20-bookworm
+FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:${VARIANT}
 
 # Install tslint, typescript. eslint is installed by javascript image
 ARG NODE_MODULES="tslint-to-eslint-config typescript"

.devcontainer/devcontainer.json

@@ -7,8 +7,8 @@
 		// Update 'VARIANT' to pick a Node version: 18, 16, 14.
 		// Append -bullseye or -buster to pin to an OS version.
 		// Use -bullseye variants on local on arm64/Apple Silicon.
-		"args": { 
-			"VARIANT": "16-bullseye"
+		"args": {
+			"VARIANT": "20-bookworm"
 		}
 	},
 

.dockerignore

@@ -1,7 +1,10 @@
-**/test/VOLUME
-**/server/DEV_DATADIR*
-**/server/target
-**/server/dist_deb
+server/dist_deb
+client/dist_deb
+organizer/**/dist_deb
+
+test/VOLUME
+server/DEV_DATADIR*
+**/target
 
 **/__pycache__
 **/.venv

.github/workflows/build-deb.yml

@@ -0,0 +1,39 @@
+name: Release version with Debian packages
+
+on:
+  push:
+    tags:
+      - 'v*'  # Trigger the workflow on push tags like v1.0, v1.1, etc.
+
+jobs:
+  build-debian-package:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Build .deb files
+      run: make debian-docker
+
+    - name: Upload Debian Package
+      uses: actions/upload-artifact@v2
+      with:
+        name: debian-packages
+        path: ./dist_deb/*.deb
+
+  create-release:
+    needs: build-debian-package
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: debian-packages
+          path: dist_deb
+
+      - name: Create Release and Upload Assets
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh auth login --with-token <<< "${GITHUB_TOKEN}"
+          gh release create "${{ github.ref_name }}" dist_deb/*.deb --draft --title "Release ${{ github.ref_name }}" --notes "Release ${{ github.ref_name }}"

.gitignore

@@ -1 +1,2 @@
 dist_deb/
+.DS_Store

.vscode/extensions.json

@@ -1,4 +1,4 @@
 {
     "recommendations": [ "dbaeumer.vscode-eslint", "svelte.svelte-vscode", "dbaeumer.vscode-eslint", "ardenivanov.svelte-intellisense",
-        "bradlc.vscode-tailwindcss", "rust-lang.rust-analyzer", "csstools.postcss", "ms-vscode-remote.remote-containers" ]
+        "bradlc.vscode-tailwindcss", "csstools.postcss", "ms-vscode-remote.remote-containers" ]
 }

Dockerfile.client

@@ -0,0 +1,40 @@
+FROM node:18.16.0-bullseye-slim
+
+# Install system packages
+RUN apt-get -qy update >/dev/null
+RUN apt-get -qy install make protobuf-compiler >/dev/null
+
+ARG TARGET=dev
+RUN if [ "$TARGET" = "deb" ]; then apt-get -qy install make debhelper devscripts; fi >/dev/null
+
+# Create regular user
+
+ARG USER=docker
+ARG UID=1000
+ARG GID=1000
+RUN echo "#### USER=${USER}, UID=${UID}, GID=${GID}"
+RUN test -n "${USER}" && test -n "${UID}" && test -n "${GID} "
+
+RUN groupadd -f ${USER} --gid=${GID}
+RUN useradd -m ${USER} --uid=${UID} --gid=${GID} || true
+RUN mkdir -p /build/client
+RUN mkdir -p /build/protobuf/libs/typescript
+RUN chown -R ${UID}:${GID} /build
+
+WORKDIR /build/client
+USER ${UID}:${GID}
+
+COPY --chown=${UID}:${GID} client/package*.json ./
+RUN npm install
+
+COPY --chown=${UID}:${GID} client /build/client
+COPY --chown=${UID}:${GID} protobuf/proto /build/protobuf/proto
+COPY --chown=${UID}:${GID} protobuf/libs/typescript /build/protobuf/libs/typescript
+
+WORKDIR /build/protobuf/libs/typescript
+RUN make
+WORKDIR /build/client
+RUN make build-local
+
+EXPOSE 5173
+CMD ["make", "dev-local"]

Dockerfile → Dockerfile.demo

@@ -1,23 +1,12 @@
 ARG auth_variation=none
 
-FROM debian:bullseye-slim AS base
+FROM debian:bookworm-slim AS base
 
 # Install system packages
 RUN apt-get -qy update
 
-RUN apt-get -qy install python3 >/dev/null
-RUN apt-get -qy install ffmpeg >/dev/null
-RUN apt-get -qy install mediainfo >/dev/null
-RUN apt-get -qy install nginx >/dev/null
-RUN apt-get -qy install acl sudo >/dev/null
-
-# Version of sqlite3 that support ALTER TABLE DROP column
-RUN echo 'deb http://ftp.debian.org/debian bookworm main' >> /etc/apt/sources.list.d/bookworm.list
-RUN apt-get -qy update >/dev/null
-RUN apt-get -qy install -t bookworm sqlite3 >/dev/null
-RUN rm /etc/apt/sources.list.d/bookworm.list
-RUN apt-get -qy update >/dev/null
-
+RUN apt-get -qy install acl sudo python3.11 >/dev/null
+RUN apt-get -qy install sqlite3 ffmpeg mediainfo nginx >/dev/null
 
 # Add regular user (to match local user ID)
 ARG UID=1000
@@ -30,8 +19,7 @@ RUN mkdir -p /mnt/clapshot-data
 RUN chown -R ${UID} /mnt/clapshot-data
 
 # Install Clapshot server & client
-COPY dist_deb/clapshot-client_*.deb /root/
-COPY dist_deb/clapshot-server_*.deb /root/
+COPY dist_deb/*.deb /root/
 RUN echo "Current architecture: $(dpkg --print-architecture)"
 RUN dpkg --path-include '/usr/share/doc/*' --refuse-downgrade -i /root/*_$(dpkg --print-architecture).deb /root/*_all.deb
 
@@ -48,7 +36,7 @@ COPY test/docker-entry_no-auth.sh /root/docker-entry.sh
 # ------------- basic auth (with PHP htadmin for management) -------------
 
 FROM base AS base-git-php
-RUN apt-get -qy install git php7.4-fpm >/dev/null
+RUN apt-get -qy install git php8.2-fpm >/dev/null
 RUN mkdir -p /run/php
 
 

Dockerfile.org-python

@@ -0,0 +1,60 @@
+FROM debian:bookworm-slim AS python-bookworm-slim
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        build-essential \
+        debhelper \
+        dh-virtualenv \
+	dh-python \
+        python3 \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        libprotobuf-dev \
+        protobuf-compiler \
+        git \
+        ; \
+    rm -rf /var/lib/apt/lists/*;
+
+# ----------------------------------
+
+FROM python-bookworm-slim AS build
+
+# Install system packages
+RUN apt-get -qy update >/dev/null
+RUN apt-get -qy install make >/dev/null
+RUN apt-get -qy install sqlite3 >/dev/null
+
+# Build deps
+# RUN apt-get -qy install libssl-dev >/dev/null
+# RUN apt-get -qy install libsqlite3-dev >/dev/null
+RUN apt-get -qy install protobuf-compiler >/dev/null
+
+# Deb build deps
+RUN apt-get -qy install dh-virtualenv >/dev/null
+
+# Create regular user
+ARG USER=docker
+ARG UID=1000
+ARG GID=1000
+RUN echo "#### USER=${USER}, UID=${UID}, GID=${GID}"
+RUN test -n "${USER}" && test -n "${UID}" && test -n "${GID}"
+
+RUN groupadd -f ${USER} --gid=${GID}
+RUN useradd -m ${USER} --uid=${UID} --gid=${GID} || true
+RUN mkdir -p /build
+RUN chown -R ${UID}:${GID} /build
+
+USER ${UID}:${GID}
+RUN mkdir -p /build/organizer/basic_folders
+RUN mkdir -p /build/protobuf/libs/python
+
+# Copy sources
+COPY --chown=${UID}:${GID} protobuf/proto /build/protobuf/proto
+COPY --chown=${UID}:${GID} protobuf/libs/python /build/protobuf/libs/python
+COPY --chown=${UID}:${GID} organizer/basic_folders /build/organizer/basic_folders
+
+WORKDIR /build/organizer/basic_folders
+RUN ln -s ../ dist_deb

Dockerfile.server

@@ -0,0 +1,80 @@
+FROM rust:1.78-slim-bookworm AS rust-bookworm-slim
+
+# ----------------------------------
+
+FROM rust-bookworm-slim AS chef
+
+# Create regular user
+ARG USER=docker
+ARG UID=1000
+ARG GID=1000
+RUN echo "#### USER=${USER}, UID=${UID}, GID=${GID}"
+RUN test -n "${USER}" && test -n "${UID}" && test -n "${GID}"
+
+RUN groupadd -f ${USER} --gid=${GID}
+RUN useradd -m ${USER} --uid=${UID} --gid=${GID} || true
+
+# Install cargo chef (Rust build cache for Docker)
+USER ${UID}:${GID}
+RUN cargo install cargo-chef
+RUN cargo install cargo-deb
+USER root
+
+# Install system packages
+RUN apt-get -qy update
+RUN apt-get -qy install acl sudo >/dev/null
+
+# Install system packages
+RUN apt-get -qy update >/dev/null
+RUN apt-get -qy install make >/dev/null
+RUN apt-get -qy install ffmpeg >/dev/null
+RUN apt-get -qy install mediainfo >/dev/null
+RUN apt-get -qy install mscgen >/dev/null
+RUN apt-get -qy install sqlite3 >/dev/null
+
+# Rust build deps
+RUN apt-get -qy install libssl-dev >/dev/null
+RUN apt-get -qy install libsqlite3-dev >/dev/null
+RUN apt-get -qy install protobuf-compiler >/dev/null
+
+# Switch to regular user
+RUN mkdir -p /app
+RUN chown -R ${UID}:${GID} /app
+USER ${UID}:${GID}
+
+# ----------------------------------
+
+FROM chef AS planner
+
+RUN mkdir -p /app/server
+WORKDIR /app/server
+COPY --chown=${UID}:${GID} server/Cargo.toml .
+RUN cargo chef prepare --recipe-path recipe.json
+
+RUN mkdir -p /app/protobuf/libs/rust
+WORKDIR /app/protobuf/libs/rust
+COPY --chown=${UID}:${GID} ../protobuf/libs/rust/Cargo.toml .
+RUN cargo chef prepare --recipe-path recipe.json
+
+# ----------------------------------
+
+FROM chef AS builder
+
+# Prebuild dependencies (build cache)
+WORKDIR /app/protobuf/libs/rust
+COPY --from=planner /app/protobuf/libs/rust/recipe.json ./
+RUN cargo chef cook --recipe-path recipe.json
+
+WORKDIR /app/server
+COPY --from=planner /app/server/recipe.json ./
+RUN cargo chef cook --recipe-path recipe.json
+
+# Copy dep sources
+COPY --chown=${UID}:${GID} protobuf/proto /app/protobuf/proto
+COPY --chown=${UID}:${GID} protobuf/libs/rust /app/protobuf/libs/rust
+COPY --chown=${UID}:${GID} server /app/server
+
+# -- config entrypoint --
+WORKDIR /app/server
+EXPOSE 8095
+CMD ["make", "run-local"]

Makefile

@@ -3,6 +3,14 @@
 UID=$(shell id -u)
 GID=$(shell id -g)
 
+ifeq ($(TARGET_ARCH),)
+  ARCH=$(shell uname -m)
+  PLATFORM_STR =
+else
+  ARCH = $(TARGET_ARCH)
+  PLATFORM_STR = --platform linux/$(TARGET_ARCH)
+endif
+
 
 default:
 	@echo "Make target 'debian-docker' explicitly."
@@ -14,40 +22,48 @@ clean-debian:
 debian-docker:
 	(cd client; make debian-docker)
 	(cd server; make debian-docker)
-	@if [ "$(shell uname -m)" != "x86_64" ]; then \
+	(cd organizer; make debian-docker)
+	@if [ "${ARCH}" != "x86_64" ]; then \
 		echo "We're running on non-x86_64 architecture. Building x86_64 deb, too."; \
 		(cd server; export TARGET_ARCH=amd64; make debian-docker); \
+		(cd organizer; export TARGET_ARCH=amd64; make debian-docker); \
 	fi
 	mkdir -p dist_deb
 	cp client/dist_deb/* dist_deb/
 	cp server/dist_deb/* dist_deb/
+	cp organizer/basic_folders/dist_deb/* dist_deb/
+	rm dist_deb/*dbgsym*
 	ls -l dist_deb/
 
 clean:	clean-debian
 	(cd client; make clean)
 	(cd server; make clean)
+	(cd organizer; make clean)
+	(cd protobuf; make clean)
 
 docker:
 	(cd client; make docker)
 	(cd server; make docker)
+	(cd organizer; make docker)
 
 test:
 	(cd client; make test-docker)
 	(cd server; make test-docker)
 
 #run-docker: clean-debian debian-docker
 run-docker: debian-docker
-	DOCKER_BUILDKIT=1 docker build -t clapshot-comb --build-arg UID=${UID} --build-arg GID=${GID} .
+	DOCKER_BUILDKIT=1 docker build -t clapshot-comb --build-arg UID=${UID} --build-arg GID=${GID} -f Dockerfile.demo .
 	# Add a simple test video to incoming already
 	mkdir -p test/VOLUME/data/incoming
 	cp server/src/tests/assets/60fps-example.mp4 test/VOLUME/data/incoming/
 	docker run --rm -it -p 0.0.0.0:8080:80 --mount type=bind,source="$$(pwd)"/test/VOLUME,target=/mnt/clapshot-data  clapshot-comb
 
-build-docker-demo: debian-docker
+build-docker-demo: #debian-docker
 	@which jq || (echo "ERROR: Please install jq first." && exit 1)
 	$(eval PVER=$(shell jq -r '.version' client/package.json))
-	docker build -t clapshot:${PVER}-demo --build-arg UID=1002 --build-arg GID=1002 .
-	docker build -t clapshot:${PVER}-demo-htadmin --build-arg UID=1002 --build-arg GID=1002 . --build-arg auth_variation=htadmin
+	docker build -t clapshot:${PVER}-demo --build-arg UID=1002 --build-arg GID=1002 -f Dockerfile.demo .
+	docker build -t clapshot:${PVER}-demo-htadmin --build-arg UID=1002 --build-arg GID=1002 -f Dockerfile.demo . --build-arg auth_variation=htadmin
+
 	docker tag clapshot:${PVER}-demo clapshot:latest-demo
 	docker tag clapshot:${PVER}-demo-htadmin clapshot:latest-demo-htadmin
 	docker tag clapshot:${PVER}-demo elonen/clapshot:${PVER}-demo