From 38b49729639d83a5061e4215321b68408051ed8f Mon Sep 17 00:00:00 2001 From: Thomas von Deyen Date: Tue, 3 Dec 2024 22:03:59 +0100 Subject: [PATCH] Add upgrader for active storage --- app/models/alchemy/attachment.rb | 13 +++ app/models/alchemy/picture.rb | 19 +++ ...11080918_rename_alchemy_attachment_file.rb | 13 +++ ...80918_rename_alchemy_picture_image_file.rb | 16 +++ .../factories/attachment_factory.rb | 1 - lib/alchemy/upgrader/eight_zero.rb | 108 ++++++++++++++++++ .../tasks/active_storage_migration.rb | 100 ++++++++++++++++ .../alchemy/install/install_generator.rb | 13 +++ lib/tasks/alchemy/upgrade.rake | 32 +++++- spec/dummy/config/initializers/dragonfly.rb | 2 + spec/dummy/config/storage.yml | 4 + ..._rename_alchemy_attachment_file.alchemy.rb | 14 +++ ...name_alchemy_picture_image_file.alchemy.rb | 17 +++ spec/dummy/db/schema.rb | 24 ++-- spec/models/alchemy/picture_spec.rb | 4 +- spec/views/admin/pictures/show_spec.rb | 6 +- .../alchemy/admin/ingredients/edit_spec.rb | 6 +- .../alchemy/ingredients/audio_view_spec.rb | 2 +- .../alchemy/ingredients/video_view_spec.rb | 2 +- 19 files changed, 368 insertions(+), 28 deletions(-) create mode 100644 db/migrate/20240611080918_rename_alchemy_attachment_file.rb create mode 100644 db/migrate/20240611080918_rename_alchemy_picture_image_file.rb create mode 100644 lib/alchemy/upgrader/eight_zero.rb create mode 100644 lib/alchemy/upgrader/tasks/active_storage_migration.rb create mode 100644 spec/dummy/db/migrate/20240611152553_rename_alchemy_attachment_file.alchemy.rb create mode 100644 spec/dummy/db/migrate/20240611152554_rename_alchemy_picture_image_file.alchemy.rb diff --git a/app/models/alchemy/attachment.rb b/app/models/alchemy/attachment.rb index 5623cbf32c..196f0be7ef 100644 --- a/app/models/alchemy/attachment.rb +++ b/app/models/alchemy/attachment.rb @@ -24,6 +24,19 @@ class Attachment < BaseRecord include Alchemy::Taggable include Alchemy::TouchElements + attr_readonly( + :legacy_image_file_name, + :legacy_image_file_size, + :legacy_image_file_uid + ) + + deprecate( + :legacy_image_file_name, + :legacy_image_file_size, + :legacy_image_file_uid, + deprecator: Alchemy::Deprecation + ) + # Use ActiveStorage file attachments has_one_attached :file, service: :alchemy_cms diff --git a/app/models/alchemy/picture.rb b/app/models/alchemy/picture.rb index 269101ea02..6d7ade35c6 100644 --- a/app/models/alchemy/picture.rb +++ b/app/models/alchemy/picture.rb @@ -80,6 +80,25 @@ def self.preprocessor_class=(klass) @_preprocessor_class = klass end + attr_readonly( + :legacy_image_file_format, + :legacy_image_file_height, + :legacy_image_file_name, + :legacy_image_file_size, + :legacy_image_file_uid, + :legacy_image_file_width + ) + + deprecate( + :legacy_image_file_format, + :legacy_image_file_height, + :legacy_image_file_name, + :legacy_image_file_size, + :legacy_image_file_uid, + :legacy_image_file_width, + deprecator: Alchemy::Deprecation + ) + # Use ActiveStorage image processing has_one_attached :image_file, service: :alchemy_cms do |attachable| # Only works in Rails 7.1 diff --git a/db/migrate/20240611080918_rename_alchemy_attachment_file.rb b/db/migrate/20240611080918_rename_alchemy_attachment_file.rb new file mode 100644 index 0000000000..caf952940c --- /dev/null +++ b/db/migrate/20240611080918_rename_alchemy_attachment_file.rb @@ -0,0 +1,13 @@ +class RenameAlchemyAttachmentFile < ActiveRecord::Migration[7.0] + COLUMNS = %i[ + file_name + file_size + file_uid + ] + + def change + COLUMNS.each do |column| + rename_column :alchemy_attachments, column, :"legacy_#{column}" + end + end +end diff --git a/db/migrate/20240611080918_rename_alchemy_picture_image_file.rb b/db/migrate/20240611080918_rename_alchemy_picture_image_file.rb new file mode 100644 index 0000000000..00b21693ee --- /dev/null +++ b/db/migrate/20240611080918_rename_alchemy_picture_image_file.rb @@ -0,0 +1,16 @@ +class RenameAlchemyPictureImageFile < ActiveRecord::Migration[7.0] + COLUMNS = %i[ + image_file_format + image_file_height + image_file_name + image_file_size + image_file_uid + image_file_width + ] + + def change + COLUMNS.each do |column| + rename_column :alchemy_pictures, column, :"legacy_#{column}" + end + end +end diff --git a/lib/alchemy/test_support/factories/attachment_factory.rb b/lib/alchemy/test_support/factories/attachment_factory.rb index 94f5949195..d988e7b9fb 100644 --- a/lib/alchemy/test_support/factories/attachment_factory.rb +++ b/lib/alchemy/test_support/factories/attachment_factory.rb @@ -20,6 +20,5 @@ end name { "image" } - file_name { "image.png" } end end diff --git a/lib/alchemy/upgrader/eight_zero.rb b/lib/alchemy/upgrader/eight_zero.rb new file mode 100644 index 0000000000..44681547c8 --- /dev/null +++ b/lib/alchemy/upgrader/eight_zero.rb @@ -0,0 +1,108 @@ +require "alchemy/shell" +require "alchemy/upgrader/tasks/active_storage_migration" +require "benchmark" +require "dragonfly" +require "dragonfly_svg" +require "fileutils" +require "thor" + +module Alchemy + class Upgrader::EightZero < Upgrader + include Thor::Base + include Thor::Actions + + class << self + def install_active_storage + Rake::Task["active_storage:install"].invoke + Rake::Task["db:migrate"].invoke + + text = <<-YAML.strip_heredoc + + alchemy_cms: + service: Disk + root: <%= Rails.root.join("storage") %> + YAML + + storage_yml = Rails.application.root.join("config/storage.yml") + if File.exist?(storage_yml) + task.insert_into_file(storage_yml, text) + else + task.create_file(storage_yml, text) + end + end + + def prepare_dragonfly_config + task.prepend_to_file "config/initializers/dragonfly.rb", <<~RUBY + require "dragonfly" + require "dragonfly_svg" + RUBY + end + + def migrate_pictures_to_active_storage + Dragonfly.logger = Rails.logger + + Alchemy::Picture.class_eval do + extend Dragonfly::Model + dragonfly_accessor :legacy_image_file, app: :alchemy_pictures + end + + pictures_without_as_attachment = Alchemy::Picture.where.missing(:image_file_attachment) + count = pictures_without_as_attachment.count + if count > 0 + log "Migrating #{count} Dragonfly image file(s) to ActiveStorage." + realtime = Benchmark.realtime do + pictures_without_as_attachment.find_each do |picture| + Alchemy::Upgrader::Tasks::ActiveStorageMigration.migrate_picture(picture) + print "." + end + end + puts "\nDone in #{realtime.round(2)}s!" + else + log "No Dragonfly image files for migration found.", :skip + end + end + + def migrate_attachments_to_active_storage + Dragonfly.logger = Rails.logger + + Alchemy::Attachment.class_eval do + extend Dragonfly::Model + dragonfly_accessor :legacy_file, app: :alchemy_attachments + end + + attachments_without_as_attachment = Alchemy::Attachment.where.missing(:file_attachment) + count = attachments_without_as_attachment.count + if count > 0 + log "Migrating #{count} Dragonfly attachment file(s) to ActiveStorage." + realtime = Benchmark.realtime do + attachments_without_as_attachment.find_each do |attachment| + Alchemy::Upgrader::Tasks::ActiveStorageMigration.migrate_attachment(attachment) + print "." + end + end + puts "\nDone in #{realtime.round(2)}s!" + else + log "No Dragonfly attachment files for migration found.", :skip + end + end + + def remove_dragonfly_todo + todo <<-TXT.strip_heredoc + Please check if all pictures and attachments are migrated to ActiveStorage. + + If so, you can remove the Dragonfly gem, its configuration and storage by running: + + rm config/initializers/dragonfly.rb + rm -rf uploads + + TXT + end + + private + + def task + @_task || new + end + end + end +end diff --git a/lib/alchemy/upgrader/tasks/active_storage_migration.rb b/lib/alchemy/upgrader/tasks/active_storage_migration.rb new file mode 100644 index 0000000000..a8c529224b --- /dev/null +++ b/lib/alchemy/upgrader/tasks/active_storage_migration.rb @@ -0,0 +1,100 @@ +require "active_storage/service" +require "active_storage/service/disk_service" + +module Alchemy + class Upgrader + module Tasks + class ActiveStorageMigration + DEFAULT_CONTENT_TYPE = "application/octet-stream" + DISK_SERVICE = ActiveStorage::Service::DiskService + SERVICE_NAME = :alchemy_cms + + METADATA = { + identified: true, # Skip identifying file type + analyzed: true, # Skip analyze job + composed: true # Skip checksum check + } + + class << self + def migrate_picture(picture) + Alchemy::Deprecation.silence do + uid = picture.legacy_image_file_uid + key = key_for_uid(uid) + content_type = Mime::Type.lookup_by_extension(picture.legacy_image_file_format) || DEFAULT_CONTENT_TYPE + Alchemy::Picture.transaction do + blob = ActiveStorage::Blob.create!( + key: key, + filename: picture.legacy_image_file_name, + byte_size: picture.legacy_image_file_size, + content_type: content_type, + # Prevents (down)loading the original file + metadata: METADATA.merge( + width: picture.legacy_image_file_width, + height: picture.legacy_image_file_height + ), + service_name: SERVICE_NAME + ) + picture.create_image_file_attachment!( + name: :image_file, + record: picture, + blob: blob + ) + end + move_file(Rails.root.join("uploads/pictures", uid), key) + end + end + + def migrate_attachment(attachment) + Alchemy::Deprecation.silence do + uid = attachment.legacy_file_uid + key = key_for_uid(uid) + Alchemy::Attachment.transaction do + blob = ActiveStorage::Blob.create!( + key: key, + filename: attachment.legacy_file_name, + byte_size: attachment.legacy_file_size, + content_type: attachment.file_mime_type.presence || DEFAULT_CONTENT_TYPE, + metadata: METADATA, + service_name: SERVICE_NAME + ) + attachment.create_file_attachment!( + record: attachment, + name: :file, + blob: blob + ) + end + move_file(Rails.root.join("uploads/attachments", uid), key) + end + end + + private + + # ActiveStorage::Service::DiskService stores files in a folder structure + # based on the first two characters of the file uid. + def key_for_uid(uid) + case service + when DISK_SERVICE + uid.split("/").last + else + uid + end + end + + def move_file(uid, key) + case service + when DISK_SERVICE + if File.exist?(uid) + service.send(:make_path_for, key) + FileUtils.cp uid, service.send(:path_for, key) + end + end + end + + def service + ActiveStorage::Blob.services.fetch(SERVICE_NAME) + end + end + end + end + end +end diff --git a/lib/generators/alchemy/install/install_generator.rb b/lib/generators/alchemy/install/install_generator.rb index 4af2237ade..4bfc118a00 100644 --- a/lib/generators/alchemy/install/install_generator.rb +++ b/lib/generators/alchemy/install/install_generator.rb @@ -71,6 +71,19 @@ def install_assets end end + def install_active_storage + rake "active_storage:install:migrations" + end + + def set_active_storage_service + insert_into_file app_config_path.join("storage.yml"), <<-YAML.strip_heredoc + + alchemy_cms: + service: Disk + root: <%= Rails.root.join("storage") %> + YAML + end + def copy_demo_views return if options[:skip_demo_files] diff --git a/lib/tasks/alchemy/upgrade.rake b/lib/tasks/alchemy/upgrade.rake index de7fa571fc..fe80d63f24 100644 --- a/lib/tasks/alchemy/upgrade.rake +++ b/lib/tasks/alchemy/upgrade.rake @@ -6,7 +6,8 @@ require "alchemy/version" namespace :alchemy do desc "Upgrades your app to AlchemyCMS v#{Alchemy::VERSION}." task upgrade: [ - "alchemy:upgrade:prepare" + "alchemy:upgrade:prepare", + "alchemy:upgrade:8.0:run" ] do Alchemy::Upgrader.display_todos end @@ -28,5 +29,34 @@ namespace :alchemy do task config: [:environment] do Alchemy::Upgrader.copy_new_config_file end + + namespace "8.0" do + task "run" => [ + "alchemy:upgrade:8.0:install_active_storage", + "alchemy:upgrade:8.0:prepare_dragonfly_config", + "alchemy:upgrade:8.0:migrate_pictures_to_active_storage", + "alchemy:upgrade:8.0:migrate_attachments_to_active_storage" + ] + + desc "Install active_storage" + task :install_active_storage do + Alchemy::Upgrader::EightZero.install_active_storage + end + + desc "Prepare Dragonfly config" + task :prepare_dragonfly_config do + Alchemy::Upgrader::EightZero.prepare_dragonfly_config + end + + desc "Migrate pictures to active_storage" + task :migrate_pictures_to_active_storage do + Alchemy::Upgrader::EightZero.migrate_pictures_to_active_storage + end + + desc "Migrate attachments to active_storage" + task :migrate_attachments_to_active_storage do + Alchemy::Upgrader::EightZero.migrate_attachments_to_active_storage + end + end end end diff --git a/spec/dummy/config/initializers/dragonfly.rb b/spec/dummy/config/initializers/dragonfly.rb index ea3b9b9854..74f11c4a19 100644 --- a/spec/dummy/config/initializers/dragonfly.rb +++ b/spec/dummy/config/initializers/dragonfly.rb @@ -1,3 +1,5 @@ +require "dragonfly" +require "dragonfly_svg" # frozen_string_literal: true # AlchemyCMS Dragonfly configuration. diff --git a/spec/dummy/config/storage.yml b/spec/dummy/config/storage.yml index d32f76e8fb..f1a392497c 100644 --- a/spec/dummy/config/storage.yml +++ b/spec/dummy/config/storage.yml @@ -32,3 +32,7 @@ local: # service: Mirror # primary: local # mirrors: [ amazon, google, microsoft ] + +alchemy_cms: + service: Disk + root: <%= Rails.root.join("storage") %> diff --git a/spec/dummy/db/migrate/20240611152553_rename_alchemy_attachment_file.alchemy.rb b/spec/dummy/db/migrate/20240611152553_rename_alchemy_attachment_file.alchemy.rb new file mode 100644 index 0000000000..3c56d08687 --- /dev/null +++ b/spec/dummy/db/migrate/20240611152553_rename_alchemy_attachment_file.alchemy.rb @@ -0,0 +1,14 @@ +# This migration comes from alchemy (originally 20240611080918) +class RenameAlchemyAttachmentFile < ActiveRecord::Migration[7.0] + COLUMNS = %i[ + file_name + file_size + file_uid + ] + + def change + COLUMNS.each do |column| + rename_column :alchemy_attachments, column, :"legacy_#{column}" + end + end +end diff --git a/spec/dummy/db/migrate/20240611152554_rename_alchemy_picture_image_file.alchemy.rb b/spec/dummy/db/migrate/20240611152554_rename_alchemy_picture_image_file.alchemy.rb new file mode 100644 index 0000000000..3a623642ae --- /dev/null +++ b/spec/dummy/db/migrate/20240611152554_rename_alchemy_picture_image_file.alchemy.rb @@ -0,0 +1,17 @@ +# This migration comes from alchemy (originally 20240611080918) +class RenameAlchemyPictureImageFile < ActiveRecord::Migration[7.0] + COLUMNS = %i[ + image_file_format + image_file_height + image_file_name + image_file_size + image_file_uid + image_file_width + ] + + def change + COLUMNS.each do |column| + rename_column :alchemy_pictures, column, :"legacy_#{column}" + end + end +end diff --git a/spec/dummy/db/schema.rb b/spec/dummy/db/schema.rb index 2728ed3600..dc0f2a2af8 100644 --- a/spec/dummy/db/schema.rb +++ b/spec/dummy/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_04_11_155901) do +ActiveRecord::Schema[7.2].define(version: 2024_06_11_152554) do create_table "active_storage_attachments", force: :cascade do |t| t.string "name", null: false t.string "record_type", null: false @@ -41,16 +41,16 @@ create_table "alchemy_attachments", force: :cascade do |t| t.string "name" - t.string "file_name" + t.string "legacy_file_name" t.string "file_mime_type" - t.integer "file_size" + t.integer "legacy_file_size" t.integer "creator_id" t.integer "updater_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.string "file_uid" + t.string "legacy_file_uid" t.index ["creator_id"], name: "index_alchemy_attachments_on_creator_id" - t.index ["file_uid"], name: "index_alchemy_attachments_on_file_uid" + t.index ["legacy_file_uid"], name: "index_alchemy_attachments_on_legacy_file_uid" t.index ["updater_id"], name: "index_alchemy_attachments_on_updater_id" end @@ -234,19 +234,19 @@ create_table "alchemy_pictures", force: :cascade do |t| t.string "name" - t.string "image_file_name" - t.integer "image_file_width" - t.integer "image_file_height" + t.string "legacy_image_file_name" + t.integer "legacy_image_file_width" + t.integer "legacy_image_file_height" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.integer "creator_id" t.integer "updater_id" t.string "upload_hash" - t.string "image_file_uid" - t.integer "image_file_size" - t.string "image_file_format" + t.string "legacy_image_file_uid" + t.integer "legacy_image_file_size" + t.string "legacy_image_file_format" t.index ["creator_id"], name: "index_alchemy_pictures_on_creator_id" - t.index ["image_file_name"], name: "index_alchemy_pictures_on_image_file_name" + t.index ["legacy_image_file_name"], name: "index_alchemy_pictures_on_legacy_image_file_name" t.index ["name"], name: "index_alchemy_pictures_on_name" t.index ["updater_id"], name: "index_alchemy_pictures_on_updater_id" end diff --git a/spec/models/alchemy/picture_spec.rb b/spec/models/alchemy/picture_spec.rb index ff1fd98b31..c3d68dd6a6 100644 --- a/spec/models/alchemy/picture_spec.rb +++ b/spec/models/alchemy/picture_spec.rb @@ -88,7 +88,7 @@ module Alchemy describe ".alchemy_resource_filters" do context "with image file formats" do - let!(:picture) { create(:alchemy_picture, image_file_format: "png") } + let!(:picture) { create(:alchemy_picture, image_file: image_file) } it "returns a list of filters with image file formats" do expect(Alchemy::Picture.alchemy_resource_filters).to eq([ @@ -403,7 +403,7 @@ module Alchemy describe "#convertible?" do let(:picture) do - Picture.new(image_file_format: "image/png") + build(:alchemy_picture, image_file: image_file) end subject { picture.convertible? } diff --git a/spec/views/admin/pictures/show_spec.rb b/spec/views/admin/pictures/show_spec.rb index 9b5af7b2d0..1ff3dd103b 100644 --- a/spec/views/admin/pictures/show_spec.rb +++ b/spec/views/admin/pictures/show_spec.rb @@ -11,11 +11,7 @@ end let(:picture) do - create(:alchemy_picture, { - image_file: image, - name: "animated", - image_file_name: "animated.gif" - }) + create(:alchemy_picture, image_file: image) end let(:language) { create(:alchemy_language) } diff --git a/spec/views/alchemy/admin/ingredients/edit_spec.rb b/spec/views/alchemy/admin/ingredients/edit_spec.rb index 57bf0e0ecb..2adbd6b43b 100644 --- a/spec/views/alchemy/admin/ingredients/edit_spec.rb +++ b/spec/views/alchemy/admin/ingredients/edit_spec.rb @@ -17,11 +17,7 @@ end let(:picture) do - create(:alchemy_picture, { - image_file: image, - name: "img", - image_file_name: "img.png" - }) + create(:alchemy_picture, image_file: image) end let(:ingredient) { Alchemy::Ingredients::Picture.new(id: 1, picture: picture) } diff --git a/spec/views/alchemy/ingredients/audio_view_spec.rb b/spec/views/alchemy/ingredients/audio_view_spec.rb index 7cf1c51448..5d37c29269 100644 --- a/spec/views/alchemy/ingredients/audio_view_spec.rb +++ b/spec/views/alchemy/ingredients/audio_view_spec.rb @@ -8,7 +8,7 @@ end let(:attachment) do - build_stubbed(:alchemy_attachment, file: file, name: "a podcast", file_name: "image with spaces.png") + build_stubbed(:alchemy_attachment, file: file) end let(:ingredient) do diff --git a/spec/views/alchemy/ingredients/video_view_spec.rb b/spec/views/alchemy/ingredients/video_view_spec.rb index 7441ce8b5e..66dcb2e338 100644 --- a/spec/views/alchemy/ingredients/video_view_spec.rb +++ b/spec/views/alchemy/ingredients/video_view_spec.rb @@ -8,7 +8,7 @@ end let(:attachment) do - build_stubbed(:alchemy_attachment, file: file, name: "a movie", file_name: "image with spaces.png") + build_stubbed(:alchemy_attachment, file: file) end let(:ingredient) do