v2023.11.0-alpha.1

photostructure · Nov 7, 2023 · f98e5a7 · f98e5a7
1 parent f7e15d7
commit f98e5a7
Show file tree

Hide file tree

Showing 46 changed files with 273 additions and 156 deletions.
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -2,7 +2,7 @@ name: Build Docker Images
 
 on:
   push:
-    # We only need to build on tags. If we include branches, we build twice.
+    # We only want to build on tags--if we include branches, we build twice.
     # branches:
     #   - main
     #   - alpha
@@ -14,69 +14,71 @@ jobs:
   build:
     name: Build Docker Images
     runs-on: ubuntu-latest
-
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-        with:
-          platforms: arm64
-
-      - name: Set up Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: actions/checkout@v4
 
+      # https://docs.docker.com/build/ci/github-actions/manage-tags-labels/
       # https://github.com/docker/metadata-action#tags-input
       # https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
-          images: photostructure/server,ghcr.io/photostructure/photostructure-for-servers
+          images: |
+            photostructure/server
+            ghcr.io/photostructure/server
           flavor: |
             latest=false
           tags: |
             # always include the full semver (v1.2.3-alpha.1)
             type=semver,pattern={{version}}
-            
+
             # tag as a prealpha build if it is such
             type=raw,value=prealpha,enable=${{ contains(github.ref_name, 'prealpha') }}
-            
+
             # unless this is a prealpha build, update the alpha tag
             type=raw,value=alpha,enable=${{ !contains(github.ref_name, 'prealpha') }}
-            
-            # always update the beta tag unless it's an alpha build, so people on :beta get the beta and stable builds:
+
+            # always update the beta tag unless it's an alpha or prealpha build, so people on :beta get the beta and stable builds:
             type=raw,value=beta,enable=${{ !contains(github.ref_name, 'alpha') }}
-            
-            # if it's not alpha or beta, update :stable, :latest, and version tags.
+
+            # if it's not *alpha or beta, update :stable, :latest, and version tags.
             type=raw,value=stable,enable=${{ !contains(github.ref_name, 'alpha') && !contains(github.ref_name, 'beta') }}
             type=raw,value=latest,enable=${{ !contains(github.ref_name, 'alpha') && !contains(github.ref_name, 'beta') }}
             type=semver,pattern=v{{major}},enable=${{ !contains(github.ref_name, 'alpha') && !contains(github.ref_name, 'beta') }}
             type=semver,pattern=v{{major}}.{{minor}},enable=${{ !contains(github.ref_name, 'alpha') && !contains(github.ref_name, 'beta') }}
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Log in to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: docker.io
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_PASS }}
 
       - name: Log in to GitHub Container Registry
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
-          username: ${{ github.repository_owner }}
+          username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push
         uses: docker/build-push-action@v4
         with:
           context: .
-          file: ./Dockerfile
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,65 @@ This is a detailed list of changes in each version.
 
 <!-- fix "tag context" for "next previous" context. I'd always done a search, clicked a thumb, and then clicked esc to go back to the search results. But...  if you click a thumb from a search,  and then click "next" or "previous", it ignores that you can from a search, and does the chronological next asset, which is very confusing/irritating. -->
 
+## v2023.11.0-alpha.1
+
+**To be released**
+
+### 💔/🐛 Exclusion globs have been simplified
+
+- Exclusion glob patterns are now applied both in and out of libraries: the
+  `excludeGlobsInLibrary` setting (introduced last build) was confusing, and
+  has been deleted.
+
+- The `disableIgnorableFilters` setting was renamed to
+  `omitDefaultExcludeGlobs`, but does the same thing: if set to `true`, users
+  start with an empty set of exclusion globs, and use `excludeGlobsAdd` to
+  build up whatever set of patterns they want.
+
+- See [the discord chat for details](https://discord.com/channels/818905168107012097/1170442569671512245)
+
+### Additional improvements
+
+- 🐛 Video transcodes in prior builds had a 2 minute timeout (due to a
+  misapplication of a default argument), which would slow down larger video
+  imports because transcode operations would timeout incorrectly and retry
+  (causing the system to bog down with the _same_ ffmpeg operation multiple
+  times)
+
+- ✨ `ffmpegHwaccel` can now be safely kept at `auto`--PhotoStructure will
+  attempt the transcode with `-hwaccel=auto`, but if that fails (due to
+  missing hardware support for the necessary codecs, for example), we'll
+  automatically re-try that transcode operation without a `-hwaccel` argument.
+
+- 🐛 `Asset.durationMs` is now properly copied from AssetFile variations up to
+  Asset. Prior builds could have missing duration timestamps in tag galleries.
+
+  Note that this is backfilled partially by a database migration that will be
+  applied automatically, as well as a step that invalidates all asset files
+  (and assets) that are videos and missing `.durationMs`.
+
+- 🐛 Fixed `./start.sh` warnings and errors related to python `disttools` and
+  `setuptools`
+
+- 🐛 Fixed [version health check
+  bug](https://discord.com/channels/818905168107012097/818907922767544340/1169801101806145566)
+  where running a newer version than what is advertised as being available was
+  marked as being "out of date". Also added channel advice if a more stable
+  channel is providing a newer release.
+
+- 📦 `./photostructure info --version-check` is now a whole thing, and invalidates prior cache.
+
+- 📦 Open Graph headers were simplified: we now only send one video or image
+  entry, using the closest available prerender to `openGraphTargetWidth`.
+  Previews seem to work properly now at least on Apple iMessages.
+
+- 📦 New `checkBasenameMatches` setting (defaults to true) adds Yet Another
+  asset file's existing-asset adoption search strategy. This is a minor
+  deduplication improvement, and doesn't seem to adversely impact import
+  speed.
+
+- 📦 Docker images should now include a proper set of labels
+
 ## v2023.11.0-prealpha.19
 
 **Released 2023 November 2**
@@ -67,9 +126,10 @@ Several new settings were added to let you suit this feature to your taste:
    require external network requests, those will be disabled by default with
    this meta setting as well.
 
-## Video improvements
+### Video improvements
 
-- 💔 **VLC support has been dropped.** Please use FFmpeg instead. [Setup instructions were updated for every platform.](/video)
+- 💔 **VLC support has been dropped.** Please use FFmpeg instead. [Setup
+  instructions were updated for every platform.](/video)
 
 - 🐛 Depending on how video files were encoded, metadata (like the video
   duration) could have been missing in prior builds. PhotoStructure now

diff --git a/Dockerfile b/Dockerfile
@@ -71,3 +71,5 @@ HEALTHCHECK CMD wget --quiet --output-document - http://localhost:1787/ping
 # docker-entrypoint.sh handles dropping privileges down to the "node" user in order
 # to support custom PUID/PGID
 ENTRYPOINT [ "/usr/bin/tini", "--", "/opt/photostructure/docker-entrypoint.sh" ]
+
+# Note: we don't set ENV here anymore. See docker-entrypoint.sh for details.
diff --git a/VERSION.json b/VERSION.json
@@ -1,13 +1,13 @@
 {
-  "version": "2023.11.0-prealpha.19",
+  "version": "2023.11.0-alpha.1",
   "versionMajor": 2023,
   "versionMinor": 11,
   "versionPatch": 0,
   "versionPrerelease": [
-    "prealpha",
-    19
+    "alpha",
+    1
   ],
-  "release": "2023.11.0-prealpha.19+20231102163722",
-  "gitSha": "c53b4343a3ba13824535898a378fd5cd98ea5134",
-  "gitDate": "2023-11-02T23:37:22.000Z"
+  "release": "2023.11.0-alpha.1+20231107131203",
+  "gitSha": "9b2c00b1e0ce2ea9685b10dde99ce4a0c430b719",
+  "gitDate": "2023-11-07T21:12:03.000Z"
 }
diff --git a/bin/info.js b/bin/info.js
diff --git a/bin/info.js.map b/bin/info.js.map
diff --git a/bin/list.js b/bin/list.js
diff --git a/bin/list.js.map b/bin/list.js.map
diff --git a/bin/logcat.js b/bin/logcat.js
diff --git a/bin/logcat.js.map b/bin/logcat.js.map
diff --git a/bin/logtail.js b/bin/logtail.js
diff --git a/bin/logtail.js.map b/bin/logtail.js.map
diff --git a/bin/main.js b/bin/main.js
diff --git a/bin/main.js.map b/bin/main.js.map
diff --git a/bin/photostructure.js b/bin/photostructure.js
diff --git a/bin/photostructure.js.map b/bin/photostructure.js.map
diff --git a/bin/sync.js b/bin/sync.js
diff --git a/bin/sync.js.map b/bin/sync.js.map
diff --git a/bin/web.js b/bin/web.js
diff --git a/bin/web.js.map b/bin/web.js.map
diff --git a/bin/worker.js b/bin/worker.js
diff --git a/bin/worker.js.map b/bin/worker.js.map
diff --git a/data/migrations/models/20231105_copy_asset_duration.sql b/data/migrations/models/20231105_copy_asset_duration.sql
@@ -0,0 +1,3 @@
+-- This file will run db/Migrations.copy_asset_duration
+-- Video Asset.durationMs fields may not have been set correctly by earlier v2023 builds.
+-- Note that this migration will also invalidate any prior Asset or AssetFile that is a video and a NULL durationMs.
diff --git a/defaults.env b/defaults.env
@@ -1,5 +1,5 @@
 #
-# Welcome to PhotoStructure! These are the settings for version 2023.9.0.
+# Welcome to PhotoStructure! These are the settings for version 2023.11.0.
 #
 # Please see https://photostructure.com/environment-variables for more
 # information about using environment variables with PhotoStructure.
@@ -1910,6 +1910,20 @@
 # PS_ASSET_AGGREGATION="intersection"
 
 
+# -------------------------------------------------
+# PS_CHECK_BASENAME_MATCHES or checkBasenameMatches
+# -------------------------------------------------
+#
+# Most cameras take JPG/RAW pairs with matching basenames (so "P01234" for
+# "P01234.JPG"). When PhotoStructure is trying to find matching assets for a
+# given file, may it check assets that have matching basename file variants?
+#
+# Note that image hash and other asset file comparisons will still be
+# performed before determining a given asset is a match.
+#
+# PS_CHECK_BASENAME_MATCHES="true"
+
+
 # --------------------------------------------------------
 # PS_DATES_BEFORE_ARE_ESTIMATED or datesBeforeAreEstimated
 # --------------------------------------------------------
@@ -2231,20 +2245,6 @@
 # PS_DISABLE_ALL_FILTERS="false"
 
 
-# -------------------------------------------------------
-# PS_DISABLE_IGNORABLE_FILTERS or disableIgnorableFilters
-# -------------------------------------------------------
-#
-# PhotoStructure has a (ton) of patterns used to avoid scanning system, cache,
-# and application directory hierarchies. Setting this to "true" will disable
-# all directory and file exclusion patterns.
-#
-# See the 'excludeGlobs*' settings for more precise editing of exclusion
-# patterns.
-#
-# PS_DISABLE_IGNORABLE_FILTERS="false"
-
-
 # ---------------------------------------
 # PS_EXCLUDE_GLOBS_ADD or excludeGlobsAdd
 # ---------------------------------------
@@ -2260,21 +2260,6 @@
 # PS_EXCLUDE_GLOBS_ADD="[]"
 
 
-# ----------------------------------------------------
-# PS_EXCLUDE_GLOBS_IN_LIBRARY or excludeGlobsInLibrary
-# ----------------------------------------------------
-#
-# Should "globs" match for files inside your library or originals directories?
-#
-# The defaults to false to make sure we don't skip directories that happen to
-# match one of the (many) exclusion patterns.
-#
-# Set this to true to apply file and directory exclusion globs even inside
-# your library directory hierarchy.
-#
-# PS_EXCLUDE_GLOBS_IN_LIBRARY="false"
-
-
 # -----------------------------------------
 # PS_EXCLUDE_GLOBS_OMIT or excludeGlobsOmit
 # -----------------------------------------
@@ -2283,13 +2268,31 @@
 # scanning system and application directories and otherwise irrelevant
 # directories and files.
 #
-# The patterns provided here are removed from that default set.
+# If you don't like one or two of them, add them here, and they will be
+# removed from the excluded globs list.
+#
+# If you don't want ANY of the default exclusion globs, see
+# "excludeGlobsOmitDefaults".
 #
 # See https://forum.photostructure.com/t/1458 for details.
 #
 # PS_EXCLUDE_GLOBS_OMIT="[]"
 
 
+# ----------------------------------------------------------
+# PS_EXCLUDE_GLOBS_OMIT_DEFAULTS or excludeGlobsOmitDefaults
+# ----------------------------------------------------------
+#
+# Setting this to true will omit ALL default exclusion globs, and is
+# equivalent to providing all default exclusion globs to the
+# "excludeGlobsOmit" setting. You're then free to add whatever exclusion globs
+# you want, using the "excludeGlobsAdd" setting.
+#
+# Aliases: "PS_OMIT_DEFAULT_EXCLUDE_GLOBS or PS_DISABLE_IGNORABLE_FILTERS"
+#
+# PS_EXCLUDE_GLOBS_OMIT_DEFAULTS="false"
+
+
 # -------------------------------------------------
 # PS_GLOBS_CASE_INSENSITIVE or globsCaseInsensitive
 # -------------------------------------------------
@@ -4603,9 +4606,19 @@
 # ----------------------------------
 #
 # FFmpeg supports both software and hardware encoders. Valid values include
-# "auto" which should work for everyone, "cuda" for NVIDIA GPUs, or use
-# "disable", "no", "false", or "" to disable. Run "ffmpeg -hwaccels" to see
-# supported acceleration methods. See
+# "auto" (which should "just work" for everyone), "h264_nvenc" for GPUs,
+# "cuda" for NVIDIA GPUs, "dxva2" or "d3d11va" for Windows, or use "disable"
+# to omit this argument.
+#
+# Run "ffmpeg -hwaccels" to see supported acceleration methods.
+#
+# Note that some hardware acceleration is only relevant for a handful of video
+# codecs, so what might help for one transcode may not for another.
+#
+# Note that PhotoStructure will automatically retry "disable" if there are
+# hwaccel errors.
+#
+# See
 # https://forum.photostructure.com/t/hardware-accelerated-encoding-transcoding/166>,
 # https://trac.ffmpeg.org/wiki/HWAccelIntro , and
 # <https://github.com/NVIDIA/nvidia-docker for more details.
@@ -4870,6 +4883,19 @@
 # PS_MIN_STREAM_CORR_PCT="50"
 
 
+# --------------------------------------------------
+# PS_OPEN_GRAPH_TARGET_WIDTH or openGraphTargetWidth
+# --------------------------------------------------
+#
+# When generating OpenGraph header metadata, include the preview image closest
+# to this number of pixels wide.
+#
+# See https://forum.photostructure.com/t/image-thumbnail-in-link-preview/1950
+# for details.
+#
+# PS_OPEN_GRAPH_TARGET_WIDTH="800"
+
+
 # ------------------------------------------
 # PS_PLACEHOLDER_THUMBS or placeholderThumbs
 # ------------------------------------------

diff --git a/package.json b/package.json
@@ -2,7 +2,7 @@
   "name": "photostructure",
   "productName": "PhotoStructure",
   "author": "PhotoStructure, Inc. <[email protected]>",
-  "version": "2023.11.0-prealpha.19",
+  "version": "2023.11.0-alpha.1",
   "description": "PhotoStructure for Servers",
   "repository": {
     "type": "git",
@@ -20,7 +20,7 @@
     "@iarna/toml": "2.2.5",
     "@parcel/watcher": "2.3.0",
     "@photostructure/tz-lookup": "8.0.0",
-    "@sentry/node": "7.76.0",
+    "@sentry/node": "7.77.0",
     "argon2": "0.31.1",
     "axios": "^1.6.0",
     "batch-cluster": "12.1.0",
@@ -41,11 +41,11 @@
     "moo": "0.5.2",
     "nearley": "2.20.1",
     "papaparse": "5.4.1",
-    "picomatch": "2.3.1",
+    "picomatch": "3.0.1",
     "platform-folders": "0.6.0",
     "plist": "3.1.0",
     "pug": "3.0.2",
-    "punycode": "2.3.0",
+    "punycode": "2.3.1",
     "semver": "7.5.4",
     "sharp": "0.32.6",
     "source-map-support": "0.5.21",

diff --git a/photostructure b/photostructure
diff --git a/public/3rd-party-licenses.txt.gz b/public/3rd-party-licenses.txt.gz
diff --git a/public/app/about.css.gz b/public/app/about.css.gz
diff --git a/public/app/about.css.map.gz b/public/app/about.css.map.gz
diff --git a/public/app/about.js.gz b/public/app/about.js.gz
diff --git a/public/app/app.css.gz b/public/app/app.css.gz
diff --git a/public/app/app.css.map.gz b/public/app/app.css.map.gz
diff --git a/public/app/app.js.gz b/public/app/app.js.gz
diff --git a/public/app/health.css.gz b/public/app/health.css.gz
diff --git a/public/app/health.css.map.gz b/public/app/health.css.map.gz
diff --git a/public/app/health.js.gz b/public/app/health.js.gz
diff --git a/public/app/install.js.gz b/public/app/install.js.gz
diff --git a/public/app/settings.css.gz b/public/app/settings.css.gz
diff --git a/public/app/settings.css.map.gz b/public/app/settings.css.map.gz
diff --git a/public/app/settings.js.gz b/public/app/settings.js.gz
diff --git a/public/app/splash.css.gz b/public/app/splash.css.gz
diff --git a/public/app/splash.css.map.gz b/public/app/splash.css.map.gz
diff --git a/public/app/splash.js.gz b/public/app/splash.js.gz