DON'T MERGE - i852 demo deploy test branch

.env

@@ -23,6 +23,7 @@ PASSENGER_APP_ENV=development
 RAILS_LOG_TO_STDOUT=true
 REDIS_HOST=redis
 SECRET_KEY_BASE=asdf
+SIDEKIQ_INTENSIVE_THREAD_COUNT=1
 SOLR_ADMIN_PASSWORD=SolrRocks
 SOLR_ADMIN_USER=solr
 SOLR_COLLECTION_NAME=hydra-development

app/jobs/create_derivatives_job_decorator.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# OVERRIDE Hyrax v3.6.0
+# @see CreateLargeDerivativesJob
+module CreateDerivativesJobDecorator
+  # OVERRIDE: Divert audio and video derivative
+  # creation to CreateLargeDerivativesJob.
+  def perform(file_set, file_id, filepath = nil)
+    return super if is_a?(CreateLargeDerivativesJob)
+    return super unless file_set.video? || file_set.audio?
+
+    CreateLargeDerivativesJob.perform_later(*arguments)
+    true
+  end
+end
+
+CreateDerivativesJob.prepend(CreateDerivativesJobDecorator)

app/jobs/create_large_derivatives_job.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# CreateLargeDerivativesJob is intended to be used for resource-intensive derivative
+# generation (e.g. video processing). It is functionally similar to CreateDerivativesJob,
+# except that it queues jobs in the :resource_intensive queue.
+#
+# The worker responsible for processing jobs in the :resource_intensive queue should be
+# configured to have more resources dedicated to it, especially CPU. Otherwise, the
+# `ffmpeg` commands that this job class eventually triggers could be throttled.
+#
+# @see CreateDerivativesJobDecorator
+# @see Hydra::Derivatives::Processors::Ffmpeg
+# @see https://github.com/scientist-softserv/palni-palci/issues/852
+class CreateLargeDerivativesJob < CreateDerivativesJob
+  queue_as :resource_intensive
+end

app/presenters/concerns/hyrax/iiif_av/displays_content_decorator.rb

@@ -42,18 +42,11 @@ def video_display_content(_url, label = '')
           height = solr_document.height&.try(:to_i) || 240
           duration = conformed_duration_in_seconds
           IIIFManifest::V3::DisplayContent.new(
-            # Hyrax::IiifAv::Engine.routes.url_helpers.iiif_av_content_url(
-            #   solr_document.id,
-            #   label: label,
-            #   host: request.base_url
-            # ),
-            # TODO: This is a hack to pull the download url from hyrax as the video resource.
-            #       Ultimately we want to fix the processing times of the video derivatives so it doesn't take
-            #       hours to days to complete.  The draw back of doing it this way is that we're using the original
-            #       video file which is fine if it's already processed, but if it's a raw, then it is not ideal for
-            #       streaming purposes.  The good thing is that PALs seem to be processing the video derivatives out
-            #       of band first before ingesting so we shouldn't run into this issue.
-            Hyrax::Engine.routes.url_helpers.download_url(solr_document.id, host: request.base_url, protocol: 'https'),
+            Hyrax::IiifAv::Engine.routes.url_helpers.iiif_av_content_url(
+              solr_document.id,
+              label: label,
+              host: request.base_url
+            ),
             label: label,
             width: width,
             height: height,

bin/helm_deploy

@@ -23,7 +23,7 @@ WORKER_IMAGE="${WORKER_IMAGE:-ghcr.io/samvera/hyku/worker}"
 DEPLOY_TAG="${DEPLOY_TAG:-latest}"
 WORKER_TAG="${WORKER_TAG:-$DEPLOY_TAG}"
 
-helm pull oci://ghcr.io/samvera/charts/hyrax --version 2.0.0 --untar --untardir charts
+helm pull oci://ghcr.io/samvera/charts/hyrax --version 3.5.1 --untar --untardir charts
 
 helm repo update
 

bin/worker

@@ -9,4 +9,8 @@ else
   puts 'DATABASE_URL not set, no pool change needed'
 end
 
-exec "echo $DATABASE_URL && bundle exec sidekiq"
+if ENV['SIDEKIQ_CONFIG']
+  exec "echo $DATABASE_URL && bundle exec sidekiq -C #{ENV['SIDEKIQ_CONFIG']}"
+else
+  exec "echo $DATABASE_URL && bundle exec sidekiq"
+end

config/initializers/file_set_derivatives_overrides.rb

@@ -48,17 +48,12 @@ def create_video_derivatives(filename)
     original_size = "#{width}x#{height}"
     size = width.nil? || height.nil? ? DEFAULT_VIDEO_SIZE : original_size
     Hydra::Derivatives::Processors::Video::Processor.config.size_attributes = size
-    # HACK: Commented out the non-thumbnail derivative generation as they are clogging the ecosystem.
-    # See https://github.com/scientist-softserv/palni-palci/issues/924
-    # rubocop:disable Style/TrailingCommaInHashLiteral
     Hydra::Derivatives::VideoDerivatives.create(filename,
                                                 outputs: [{ label: :thumbnail, format: 'jpg',
                                                             url: derivative_url('thumbnail') },
-                                                          # { label: 'webm', format: 'webm',
-                                                          #   url: derivative_url('webm') },
-                                                          # { label: 'mp4', format: 'mp4',
-                                                          #   url: derivative_url('mp4') }
-                                                         ])
-    # rubocop:enable Style/TrailingCommaInHashLiteral
+                                                          { label: 'webm', format: 'webm',
+                                                            url: derivative_url('webm') },
+                                                          { label: 'mp4', format: 'mp4',
+                                                            url: derivative_url('mp4') }])
   end
-end
+end

config/sidekiq_resource_intensive.yml

@@ -0,0 +1,7 @@
+---
+:concurrency: <%= ENV['SIDEKIQ_INTENSIVE_THREAD_COUNT'] %>
+:queues:
+  - default
+  - import
+  - export
+  - resource_intensive

docker-compose.yml

@@ -20,6 +20,35 @@ x-app: &app
   networks:
     internal:
 
+x-app-worker: &app-worker
+  <<: *app
+  build:
+    context: .
+    target: hyku-worker
+    args:
+      - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync exiftool
+      - HYKU_BULKRAX_ENABLED=true
+    cache_from:
+      - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest}
+      - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
+  image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
+  command: sh -l -c 'bundle && bundle exec sidekiq'
+  depends_on:
+    check_volumes:
+      condition: service_completed_successfully
+    initialize_app:
+      condition: service_completed_successfully
+    db:
+      condition: service_started
+    solr:
+      condition: service_started
+    fcrepo:
+      condition: service_started
+    redis:
+      condition: service_started
+    zoo:
+      condition: service_started
+
 volumes:
   assets:
   cache:
@@ -145,6 +174,8 @@ services:
         condition: service_started
       worker:
         condition: service_started
+      worker_resource_intensive:
+        condition: service_started
       initialize_app:
         condition: service_completed_successfully
     # ports:
@@ -153,33 +184,11 @@ services:
       - 3000
 
   worker:
-    <<: *app
-    build:
-      context: .
-      target: hyku-worker
-      args:
-        - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync exiftool
-        - HYKU_BULKRAX_ENABLED=true
-      cache_from:
-        - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest}
-        - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
-    image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
-    command: sh -l -c 'bundle && bundle exec sidekiq'
-    depends_on:
-      check_volumes:
-        condition: service_completed_successfully
-      initialize_app:
-        condition: service_completed_successfully
-      db:
-        condition: service_started
-      solr:
-        condition: service_started
-      fcrepo:
-        condition: service_started
-      redis:
-        condition: service_started
-      zoo:
-        condition: service_started
+    <<: *app-worker
+
+  worker_resource_intensive:
+    <<: *app-worker
+    command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq_resource_intensive.yml'
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows