deferred v18.x release - to apply in v19.0?

changelog

@@ -3,13 +3,19 @@ turnkey-canvas-18.1 (1) turnkey; urgency=low
   * Install latest Canvas LTS stable (prod branch), Canvas RCE API and required
     dependencies. Canvas installed from upstream git repo.
 
+  * Pre-install QTI import tool - as requested/suggested on TKL forums.
+
   * Update Ruby (3.1.6).
 
   * Update bundler to 2.5.10 - as per "Production Start" doc.
 
   * Disable Apache mod_evasie for Canvas - part of #1965.
 
-  * Run switchman_inst_jobs:install:migrations - closes #1965.
+  * Bump delayed worker RAM allowance. Real fix for  - closes #1978 & #1979.
+    Also properly resolves #1965.
+
+  * Add LTI JWK keys to default conf (regenerated at firstboot) - should close
+    #1977 - although has not been confirmed, needs user feedback.
 
   * Update GEM_PATH in Apache conf - didn't seem to be causing issues, but
     better for it to be correct path.
@@ -26,7 +32,7 @@ turnkey-canvas-18.1 (1) turnkey; urgency=low
 
   * Reduce log noise by creating ntpsec log dir - closes #1952.
 
- -- Jeremy Davis <[email protected]>  Sat, 06 Jul 2024 11:31:36 +0000
+ -- Jeremy Davis <[email protected]>  Wed, 28 Aug 2024 00:36:52 +0000
 
 turnkey-canvas-18.0 (1) turnkey; urgency=low
 

conf.d/52canvas-configs

@@ -1,9 +1,32 @@
 #!/bin/bash -ex
 
+# load common env vars
 source /usr/local/src/canvas.conf
 
+CONF_DIR="$WEBROOT/config"
+USER=www-data
+
+add_comment() {
+    cat <<EOF
+# This config file is generated by TurnKey Linux at build time.
+# it includes core "production" functionality configuration.
+#
+# See "$conf_file.example" for all available options.
+#
+# If adding/copying keys (i.e. lines including ':') ALL leading spaces
+# from the example config options MUST be included.
+#
+# After editing config, restart apache2 and canvas_init services:
+#
+#   systemctl restart apache2 canvas_init
+
+EOF
+}
+
 # setup database config
-cat >$WEBROOT/config/database.yml<<EOF
+conf_file=database.yml
+cat > "$CONF_DIR/$conf_file" <<EOF
+$(add_comment)
 production:
   adapter: postgresql
   encoding: utf8
@@ -14,39 +37,46 @@ production:
   timeout: 5000
 EOF
 
-# setup email config
-cat >$WEBROOT/config/outgoing_mail.yml<<EOF
+conf_file=outgoing_mail.yml
+cat > "$CONF_DIR/$conf_file" <<EOF
+$(add_comment)
 production:
-  tls: false
   delivery_method: "sendmail"
-  enable_starttls_auto: false
+  #tls: false
+  #enable_starttls_auto: false
   address: "localhost"
   port: "25"
   domain: "$DOMAIN"
   outgoing_address: "$ADMIN_MAIL"
   default_name: "TurnKey Canvas"
 EOF
 
-# setup domain config
-cat >$WEBROOT/config/domain.yml<<EOF
+conf_file=domain.yml
+cat > "$CONF_DIR/$conf_file" <<EOF
+$(add_comment)
 production:
   domain: "$DOMAIN"
   ssl: true
 EOF
 
-# setup cache store config
-cat >$WEBROOT/config/cache_store.yml<<EOF
+conf_file=cache_store.yml
+cat > "$CONF_DIR/$conf_file" <<EOF
+$(add_comment)
 production:
   cache_store: redis_cache_store
 EOF
 
-cat >$WEBROOT/config/redis.yml<<EOF
+conf_file=redis.yml
+cat > "$CONF_DIR/$conf_file" <<EOF
+$(add_comment)
 production:
   url:
   - redis://localhost
 EOF
 
-cat >$WEBROOT/config/delayed_jobs.yml<<EOF
+conf_file=delayed_jobs.yml
+cat > "$CONF_DIR/$conf_file" <<EOF
+$(add_comment)
 production:
   workers:
   - queue: canvas_queue
@@ -57,15 +87,17 @@ production:
 
   max_run_time: 28800
   worker_max_job_count: 20
-  worker_max_memory_usage: 536870912
+  worker_max_memory_usage: 1073741824
 
 default:
   workers:
   - queue: canvas_queue
 EOF
 
 # Canvas Rich Content Editor API
-cat >$WEBROOT/config/vault_contents.yml<<EOF
+conf_file=vault_contents.yml
+cat > "$CONF_DIR/$conf_file" <<EOF
+$(add_comment)
 production:
   'app-canvas/data/secrets':
     data:
@@ -74,19 +106,32 @@ production:
         signing_secret: "turnkey1"
 EOF
 
-cat >$WEBROOT/config/security.yml<<EOF
+conf_file=security.yml
+cat > "$CONF_DIR/$conf_file" <<EOF
+$(add_comment)
 production: &default
   # replace this with a random string of at least 20 characters
   encryption_key: 1234512345123451234512345123451234512345123451234512345
-  lti_iss: 'https://canvas.instructure.com'
+  lti_iss: "https://$DOMAIN"
 EOF
 
-cat >$WEBROOT/config/dynamic_settings.yml<<EOF
+conf_file=dynamic_settings.yml
+# use example JWK values from example conf - regen at firstboot
+cat > "$CONF_DIR/$conf_file" <<EOF
+$(add_comment)
 production:
   config:
     canvas:
       rich-content-service:
         app-host: "canvas-rce-api-host"
+  store:
+    canvas:
+      lti-keys:
+        # these values are (re)generated on firstboot by /usr/lib/inithooks/bin/regen_jwks
+        # see comments in dynamic_settings.yml.example for more info
+        jwk-past.json: "{\"kty\":\"RSA\",\"e\":\"AQAB\",\"n\":\"uX1MpfEMQCBUMcj0sBYI-iFaG5Nodp3C6OlN8uY60fa5zSBd83-iIL3n_qzZ8VCluuTLfB7rrV_tiX727XIEqQ\",\"kid\":\"2018-05-18T22:33:20Z\",\"d\":\"pYwR64x-LYFtA13iHIIeEvfPTws50ZutyGfpHN-kIZz3k-xVpun2Hgu0hVKZMxcZJ9DkG8UZPqD-zTDbCmCyLQ\",\"p\":\"6OQ2bi_oY5fE9KfQOcxkmNhxDnIKObKb6TVYqOOz2JM\",\"q\":\"y-UBef95njOrqMAxJH1QPds3ltYWr8QgGgccmcATH1M\",\"dp\":\"Ol_xkL7rZgNFt_lURRiJYpJmDDPjgkDVuafIeFTS4Ic\",\"dq\":\"RtzDY5wXr5TzrwWEztLCpYzfyAuF_PZj1cfs976apsM\",\"qi\":\"XA5wnwIrwe5MwXpaBijZsGhKJoypZProt47aVCtWtPE\"}"
+        jwk-present.json: "{\"kty\":\"RSA\",\"e\":\"AQAB\",\"n\":\"uX1MpfEMQCBUMcj0sBYI-iFaG5Nodp3C6OlN8uY60fa5zSBd83-iIL3n_qzZ8VCluuTLfB7rrV_tiX727XIEqQ\",\"kid\":\"2018-06-18T22:33:20Z\",\"d\":\"pYwR64x-LYFtA13iHIIeEvfPTws50ZutyGfpHN-kIZz3k-xVpun2Hgu0hVKZMxcZJ9DkG8UZPqD-zTDbCmCyLQ\",\"p\":\"6OQ2bi_oY5fE9KfQOcxkmNhxDnIKObKb6TVYqOOz2JM\",\"q\":\"y-UBef95njOrqMAxJH1QPds3ltYWr8QgGgccmcATH1M\",\"dp\":\"Ol_xkL7rZgNFt_lURRiJYpJmDDPjgkDVuafIeFTS4Ic\",\"dq\":\"RtzDY5wXr5TzrwWEztLCpYzfyAuF_PZj1cfs976apsM\",\"qi\":\"XA5wnwIrwe5MwXpaBijZsGhKJoypZProt47aVCtWtPE\"}"
+        jwk-future.json: "{\"kty\":\"RSA\",\"e\":\"AQAB\",\"n\":\"uX1MpfEMQCBUMcj0sBYI-iFaG5Nodp3C6OlN8uY60fa5zSBd83-iIL3n_qzZ8VCluuTLfB7rrV_tiX727XIEqQ\",\"kid\":\"2018-07-18T22:33:20Z\",\"d\":\"pYwR64x-LYFtA13iHIIeEvfPTws50ZutyGfpHN-kIZz3k-xVpun2Hgu0hVKZMxcZJ9DkG8UZPqD-zTDbCmCyLQ\",\"p\":\"6OQ2bi_oY5fE9KfQOcxkmNhxDnIKObKb6TVYqOOz2JM\",\"q\":\"y-UBef95njOrqMAxJH1QPds3ltYWr8QgGgccmcATH1M\",\"dp\":\"Ol_xkL7rZgNFt_lURRiJYpJmDDPjgkDVuafIeFTS4Ic\",\"dq\":\"RtzDY5wXr5TzrwWEztLCpYzfyAuF_PZj1cfs976apsM\",\"qi\":\"XA5wnwIrwe5MwXpaBijZsGhKJoypZProt47aVCtWtPE\"}"
 EOF
 
 # copy example configurations
@@ -95,31 +140,27 @@ for c in $CONFIGS; do
     cp $WEBROOT/config/$c.yml.example $WEBROOT/config/$c.yml
 done
 
-WEBROOT=/var/www/canvas
-USER=www-data
-
-# configure permissions
-mkdir -p $WEBROOT/log
-mkdir -p $WEBROOT/tmp/pids
-mkdir -p $WEBROOT/tmp/files
-mkdir -p $WEBROOT/tmp/attachment_fu
-mkdir -p $WEBROOT/public/assets
-mkdir -p $WEBROOT/public/stylesheets/compiled
-mkdir -p $WEBROOT/app/stylesheets/brandable_css_brands
+# create dirs & configure permissions
+dirs=(log tmp/pids tmp/files tmp/attachment_fu public/assets
+      public/stylesheets/compiled app/stylesheets/brandable_css_brands)
+for dir in "${dirs[@]}"; do
+    mkdir -p "$WEBROOT/$dir"
+done
 
 touch $WEBROOT/Gemfile.lock
 touch $WEBROOT/app/stylesheets/_brandable_variables_defaults_autogenerated.scss
 
-chown -R root:www-data $WEBROOT
-chown -R $USER:$USER $WEBROOT/log
-chown -R $USER:$USER $WEBROOT/tmp
+chown -R root:www-data "$WEBROOT"
+for dir in log tmp config; do
+    chown -R "$USER":"$USER" "$WEBROOT/$dir"
+done
+
+files=(public/assets Gemfile.lock config.ru config/environment.rb
+       app/stylesheets/_brandable_variables_defaults_autogenerated.scss)
+for file in "${files[@]}"; do
+    chown "$USER":"$USER" "$WEBROOT/$file"
+done
+
 find $WEBROOT/tmp -type d -exec chmod 755 {} \;
 find $WEBROOT/tmp -type f -exec chmod 644 {} \;
-chown -R $USER:$USER $WEBROOT/config
-chown $USER:$USER $WEBROOT/public/assets \
-                  $WEBROOT/app/stylesheets/_brandable_variables_defaults_autogenerated.scss \
-                  $WEBROOT/app/stylesheets/brandable_css_brands \
-                  $WEBROOT/Gemfile.lock \
-                  $WEBROOT/config.ru \
-                  $WEBROOT/config/environment.rb
 chmod 640 $WEBROOT/config/*.yml

conf.d/55canvas-install

@@ -1,17 +1,18 @@
 #!/bin/bash -ex
 
-# disable Aapche mod_evasive for now - see
+# disable Apache mod_evasive for now - see
 # https://github.com/turnkeylinux/tracker/issues/1965
 a2dismod evasive
 
 source /usr/local/src/canvas.conf
-
 YARN_CACHE=/var/cache/yarn
+export RAILS_ENV=production
+export BUNDLE_PATH="vendor/bundle"
 
 # preset answers for automated install
 export CANVAS_LMS_ADMIN_EMAIL=$ADMIN_MAIL
 export CANVAS_LMS_ADMIN_PASSWORD=$ADMIN_PASS
-export CANVAS_LMS_ACCOUNT_NAME="TurnKey Canvas"
+export CANVAS_LMS_ACCOUNT_NAME="TurnKey super admin"
 export CANVAS_LMS_STATS_COLLECTION="opt-out"
 
 # set lang to quieten down install
@@ -20,19 +21,16 @@ export LC_ALL=en_US.UTF-8
 
 # install canvas, populate database and compile assets
 service redis-server start
-[ "$FAB_HTTP_PROXY" ] && export http_proxy=$FAB_HTTP_PROXY
+[[ "$FAB_HTTP_PROXY" ]] && export http_proxy=$FAB_HTTP_PROXY
+[[ "$FAB_HTTPS_PROXY" ]] && export https_proxy=$FAB_HTTPS_PROXY
 
 cd $WEBROOT
 
 # canvas needs specific version of bundler - see
 # https://github.com/instructure/canvas-lms/wiki/Production-Start#bundler-and-canvas-dependencies
 gem install bundler --version 2.5.10
 
-# set ruby/rails vars
-export RAILS_ENV=production
-export BUNDLE_PATH="vendor/bundle"
-
-bundle config set --local path vendor/bundle
+bundle config set --local path $BUNDLE_PATH
 bundle config set without 'mysql sqlite'
 bundle install
 
@@ -46,41 +44,20 @@ yarn install --network-timeout 1000000 --scripts-prepend-node-path --cache-folde
 
 # COMPILE_ASSETS_BRANC_CONFIGS variable:
 # https://github.com/instructure/canvas-lms/issues/2023
-
-# fix for webpack failure
-# https://stackoverflow.com/questions/69692842/error-message-error0308010cdigital-envelope-routinesunsupported
-sed -i 's/5120/5120 --openssl-legacy-provider/g' package.json
-
 COMPILE_ASSETS_BRAND_CONFIGS=0 bundle exec rake canvas:compile_assets
 chown -R www-data:www-data $WEBROOT/public/dist/brandable_css
 
-# apply another patch; this time to the init script
-_patch=/usr/local/src/canvas_init.patch
-git apply $_patch
-rm -rf $_patch
-
-# https://github.com/instructure/canvas-lms/issues/2034
-# mv db/migrate/20210812210129_add_singleton_column.rb db/migrate/20111111214311_add_singleton_column.rb
-# https://github.com/instructure/canvas-lms/issues/2035
-# mv db/migrate/20210823222355_change_immersive_reader_allowed_on_to_on.rb .
-# weird zeitwerk path issue fix
-sed -i 's|\(lib/ssl_common\)|./\1|' app/models/report_snapshot.rb
 bundle exec rake db:initial_setup
-# mv 20210823222355_change_immersive_reader_allowed_on_to_on.rb  db/migrate/
 bundle exec rake db:migrate
 
-# resolve (errant) 'failed to allocate memory' issue - closes
-# https://github.com/turnkeylinux/tracker/issues/1965
-bundle exec rake switchman_inst_jobs:install:migrations
-
 mkdir -p log tmp/pids public/assets 
 
 # stop services
 service postgresql stop
 service redis-server stop
 
-unset http_proxy
+unset http_proxy https_proxy
 
-# clean out installation log
+# clean out installation log & cache
 rm -f $WEBROOT/log/*
 rm -rf $YARN_CACHE

conf.d/57canvas-qti-import

@@ -0,0 +1,10 @@
+#!/bin/bash -ex
+
+# Install QTI import tool - as requested/noted on TKL forums:
+# https://www.turnkeylinux.org/forum/support/wed-20231122-1754/canvas-fails-import-qti-quiz-files
+
+source /usr/local/src/canvas.conf
+
+QTI=QTIMigrationTool
+git clone --depth 1 https://github.com/instructure/$QTI.git $WEBROOT/vendor/$QTI
+chmod +x $WEBROOT/vendor/$QTI

conf.d/70initscript

@@ -6,8 +6,8 @@ source /usr/local/src/canvas.conf
 ln -s $WEBROOT/script/canvas_init /etc/init.d/canvas_init
 
 # changes applied with patch in overlays/usr/local/src/canvas_init.patch
-# during conf.d/55canvas-install
-# sed -i '/exec su/ s#$# -s /bin/bash#' $WEBROOT/script/canvas_init
-# sed -i "s|exec script\/delayed_job \$@|bundle exec script\/delayed_job \$\@|" $WEBROOT/script/canvas_init
+cd $WEBROOT
+_patch=$SRC/canvas_init.patch
+git apply $_patch
 
 update-rc.d canvas_init defaults

conf.d/90finish

@@ -16,19 +16,21 @@ APACHE_CONF=/etc/apache2/sites-available/canvas.conf
 sed -i "/BUNDLE_PATH/ s|__BUNDLE_PATH__|$BUNDLE_PATH|" "$APACHE_CONF"
 sed -i "/GEM_PATH/ s|__GEM_PATH__|$GEM_PATH|" "$APACHE_CONF"
 
+PASSGR_CONF=/etc/apache2/mods-available/passenger.conf
+# 6 min timeout - ensures Canvas loads at build time testing
+echo "PassengerStartTimeout 360" >> $PASSGR_CONF
+# part of fix for https://github.com/phusion/passenger/issues/2397
+# other part in /etc/tmpfiles.d/passenger.conf (overlay)
+echo "PassengerInstanceRegistryDir /run/passenger-instreg" >> $PASSGR_CONF
+
 # create convenience symlink for logs
 rm -rf /var/log/canvas
 ln -s $WEBROOT/log /var/log/canvas
 
-chown -R www-data:www-data /var/www/canvas/{config,log}
-
-# fix for github.com/phusion/passenger/wiki/Debugging-application-startup-problems
-echo "PassengerStartTimeout 300" >> /etc/apache2/mods-available/passenger.conf
-
-# part of fix for https://github.com/phusion/passenger/issues/2397
-echo "PassengerInstanceRegistryDir /run/passenger-instreg" >> /etc/apache2/mods-available/passenger.conf
-# see also /etc/tmpfiles.d/passenger.conf in overlay
-
+chown -R www-data:www-data $WEBROOT/{config,log}
 chmod 400 $WEBROOT/config/cache_store.yml
 
+py3clean /
+yarn cache clean
+
 rm -f /usr/local/src/canvas.conf