From 3be1c08286be1cf9a7e7d368cc1492b4b4640237 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Sun, 8 Dec 2024 16:12:40 -0500 Subject: [PATCH 01/12] Override enclosure url and/or prefix --- app/models/enclosure_url_builder.rb | 13 +++++++++++-- ...42_add_enclosure_override_url_to_episodes.rb | 6 ++++++ db/schema.rb | 4 +++- test/models/enclosure_url_builder_test.rb | 17 ++++++++++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 db/migrate/20241208194342_add_enclosure_override_url_to_episodes.rb diff --git a/app/models/enclosure_url_builder.rb b/app/models/enclosure_url_builder.rb index b794456ad..22c5b5988 100644 --- a/app/models/enclosure_url_builder.rb +++ b/app/models/enclosure_url_builder.rb @@ -21,8 +21,17 @@ def podcast_episode_url(podcast, episode, feed = nil) feed ||= podcast.default_feed prefix = feed.try(:enclosure_prefix) - url = base_enclosure_url(podcast, episode, feed) - enclosure_prefix_url(url, prefix) + base_url = if !episode.enclosure_override_url.blank? + episode.enclosure_override_url + else + base_enclosure_url(podcast, episode, feed) + end + + if episode.enclosure_override_prefix + base_url + else + enclosure_prefix_url(base_url, prefix) + end end def base_enclosure_url(podcast, episode, feed) diff --git a/db/migrate/20241208194342_add_enclosure_override_url_to_episodes.rb b/db/migrate/20241208194342_add_enclosure_override_url_to_episodes.rb new file mode 100644 index 000000000..f78fddfe1 --- /dev/null +++ b/db/migrate/20241208194342_add_enclosure_override_url_to_episodes.rb @@ -0,0 +1,6 @@ +class AddEnclosureOverrideUrlToEpisodes < ActiveRecord::Migration[7.2] + def change + add_column :episodes, :enclosure_override_url, :string + add_column :episodes, :enclosure_override_prefix, :boolean + end +end diff --git a/db/schema.rb b/db/schema.rb index c6e3e4bd9..523fd862e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_11_20_165556) do +ActiveRecord::Schema[7.2].define(version: 2024_12_08_194342) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" enable_extension "uuid-ossp" @@ -165,6 +165,8 @@ t.integer "medium" t.integer "lock_version", default: 0, null: false t.string "categories", array: true + t.string "enclosure_override_url" + t.boolean "enclosure_override_prefix" t.index ["categories"], name: "index_episodes_on_categories", using: :gin t.index ["guid"], name: "index_episodes_on_guid", unique: true t.index ["keyword_xid"], name: "index_episodes_on_keyword_xid", unique: true diff --git a/test/models/enclosure_url_builder_test.rb b/test/models/enclosure_url_builder_test.rb index a73b3d3dc..333b6a6f7 100644 --- a/test/models/enclosure_url_builder_test.rb +++ b/test/models/enclosure_url_builder_test.rb @@ -5,7 +5,7 @@ let(:template) { "https://#{ENV["DOVETAIL_HOST"]}/{slug}/{guid}/{original_filename}" } let(:prefix) { "http://www.podtrac.com/pts/redirect.mp3/media.blubrry.com/jojego/" } let(:podcast) { create(:podcast, enclosure_prefix: prefix, enclosure_template: template) } - let(:episode) { create(:episode_with_media, podcast: podcast, prx_uri: "/api/v1/stories/87683") } + let(:episode) { create(:episode_with_media, podcast: podcast) } let(:feed) { create(:feed, podcast: podcast, slug: "no-ads-pls") } let(:builder) { EnclosureUrlBuilder.new } @@ -118,6 +118,21 @@ _(url).must_match(/http:\/\/www.podtrac.com\/pts\/redirect.mp3\/media.blubrry.com\/jojego\/dovetail.prxu.org\/#{podcast.id}\/ba047dce-9df5-4132-a04b-31d24c7c55a(\d+)\/audio\.mp3/) end + it "can make an enclosure url for an override url, but not override prefix" do + episode.enclosure_override_url = "https://a.io/a.mp3" + episode.enclosure_override_prefix = false + + url = builder.podcast_episode_url(podcast, episode) + _(url).must_equal("http://www.podtrac.com/pts/redirect.mp3/media.blubrry.com/jojego/a.io/a.mp3") + end + it "can make an enclosure url for an override url, also override prefix" do + episode.enclosure_override_url = "https://a.io/a.mp3" + episode.enclosure_override_prefix = true + + url = builder.podcast_episode_url(podcast, episode) + _(url).must_equal("https://a.io/a.mp3") + end + it "applies template to audio file link" do podcast.enclosure_prefix = nil podcast.enclosure_template = "http://foo.com/r{extension}/b/n/{host}{+path}" From d8d454cfdd2f158ec0521f680ed6fe8b133c1875 Mon Sep 17 00:00:00 2001 From: Andrew Kuklewicz Date: Tue, 10 Dec 2024 09:05:12 -0500 Subject: [PATCH 02/12] Refactor episode media, add override medium --- app/controllers/episode_media_controller.rb | 2 + app/helpers/episodes_helper.rb | 4 +- app/models/concerns/episode_media.rb | 132 +++++++++++++++++- app/models/episode.rb | 79 ----------- app/models/external_media_resource.rb | 34 +++++ app/models/tasks/analyze_media_task.rb | 43 ++++++ app/views/episode_media/_form.html.erb | 16 +++ app/views/episode_media/_form_main.html.erb | 3 +- app/views/episode_media/_form_status.html.erb | 2 +- .../episodes/_form_distribution.html.erb | 2 +- config/locales/en.yml | 1 + test/models/episode_test.rb | 3 +- test/models/external_media_resource_test.rb | 7 + 13 files changed, 235 insertions(+), 93 deletions(-) create mode 100644 app/models/external_media_resource.rb create mode 100644 app/models/tasks/analyze_media_task.rb create mode 100644 test/models/external_media_resource_test.rb diff --git a/app/controllers/episode_media_controller.rb b/app/controllers/episode_media_controller.rb index a8b595ede..c7aefad5f 100644 --- a/app/controllers/episode_media_controller.rb +++ b/app/controllers/episode_media_controller.rb @@ -67,6 +67,8 @@ def episode_params :lock_version, :medium, :ad_breaks, + :enclosure_override_url, + :enclosure_override_prefix, contents_attributes: %i[id position original_url file_size _destroy _retry], uncut_attributes: %i[id segmentation original_url file_size _destroy _retry] ) diff --git a/app/helpers/episodes_helper.rb b/app/helpers/episodes_helper.rb index 69ac00c2a..da2aa6e90 100644 --- a/app/helpers/episodes_helper.rb +++ b/app/helpers/episodes_helper.rb @@ -54,9 +54,9 @@ def episode_status_class(episode) def episode_media_status(episode) if episode_all_media(episode).any? { |m| upload_problem?(m) } "error" - elsif episode_all_media(episode).any? { |m| upload_processing?(m) } + elsif episode_all_media(episode).any? { |m| upload_processing?(m) } || episode.override_processing? "processing" - elsif episode.media_ready?(true) + elsif episode.media_ready?(true) || episode.override_ready? "complete" elsif episode.published_at.present? "incomplete-published" diff --git a/app/models/concerns/episode_media.rb b/app/models/concerns/episode_media.rb index f4cb94609..494b5bb79 100644 --- a/app/models/concerns/episode_media.rb +++ b/app/models/concerns/episode_media.rb @@ -4,14 +4,95 @@ module EpisodeMedia extend ActiveSupport::Concern included do + enum :medium, [:audio, :uncut, :video, :override], prefix: true + # NOTE: this just-in-time creates new media versions # TODO: convert to sql, so we don't have to load/check every episode? # TODO: stop loading non-latest media versions scope :feed_ready, -> { includes(media_versions: :media_resources).select { |e| e.feed_ready? } } + + has_many :media_versions, -> { order("created_at DESC") }, dependent: :destroy + has_many :contents, -> { order("position ASC, created_at DESC") }, autosave: true, dependent: :destroy, inverse_of: :episode + has_one :uncut, -> { order("created_at DESC") }, autosave: true, dependent: :destroy, inverse_of: :episode + has_one :external_media_resource, -> { order("created_at DESC") }, autosave: true, dependent: :destroy, inverse_of: :episode + + accepts_nested_attributes_for :contents, allow_destroy: true, reject_if: ->(c) { c[:id].blank? && c[:original_url].blank? } + accepts_nested_attributes_for :uncut, allow_destroy: true, reject_if: ->(u) { u[:id].blank? && u[:original_url].blank? } + + validate :validate_media_ready, if: :strict_validations + + after_save :destroy_out_of_range_contents, if: ->(e) { e.segment_count_previously_changed? } + after_save :analyze_external_media + end + + def validate_media_ready + return unless published_at.present? && media? + + # media must be complete on _initial_ publish + # otherwise - having files in any status is good enough + is_ready = + if published_at_was.blank? + media_ready?(true) || override_ready? + elsif medium_uncut? + uncut.present? && !uncut.marked_for_destruction? + elsif override? + external_media_ready? + else + media_ready?(false) + end + + unless is_ready + errors.add(:base, :media_not_ready, message: "media not ready") + end + end + + def medium=(new_medium) + super + + if medium_changed? && medium_was.present? + if medium_was == "uncut" && medium == "audio" + uncut&.mark_for_destruction + elsif medium_was == "audio" && medium == "uncut" + if (c = contents.first) + build_uncut.tap do |u| + u.file_size = contents.first.file_size + u.duration = contents.first.duration + + # use the feeder cdn url for older completed files + is_old = (Time.now - c.created_at) > 24.hours + u.original_url = (c.status_complete? && is_old) ? c.url : c.original_url + end + end + contents.each(&:mark_for_destruction) + else + contents.each(&:mark_for_destruction) + end + end + + self.segment_count = 1 if medium_video? || medium_override? + end + + def copy_media(force = false) + contents.each { |c| c.copy_media(force) } + images.each { |i| i.copy_media(force) } + transcript&.copy_media(force) + uncut&.copy_media(force) + end + + def build_contents + segment_range.map do |p| + contents.find { |c| c.position == p } || contents.build(position: p) + end + end + + def destroy_out_of_range_contents + if segment_count.present? && segment_count.positive? + contents.where.not(position: segment_range.to_a).destroy_all + end end def feed_ready? - !media? || complete_media? + !media? || complete_media? || override_ready? end def cut_media_version! @@ -97,7 +178,9 @@ def media_content_type(feed = nil) feed_content_type = feed.try(:mime_type) # if audio AND feed has a mime type, dovetail will transcode to that - if (media_content_type || "").starts_with?("audio") + if override? + external_media_resource&.mime_type || "audio/mpeg" + elsif (media_content_type || "").starts_with?("audio") feed_content_type || media_content_type elsif media_content_type media_content_type @@ -111,11 +194,19 @@ def video_content_type?(*args) end def media_duration - media.inject(0.0) { |s, c| s + c.duration.to_f } + podcast.try(:duration_padding).to_f + if override? + external_media_resource&.duration + else + media.inject(0.0) { |s, c| s + c.duration.to_f } + podcast.try(:duration_padding).to_f + end end def media_file_size - media.inject(0) { |s, c| s + c.file_size.to_i } + if override? + external_media_resource&.file_size + else + media.inject(0) { |s, c| s + c.file_size.to_i } + end end def media_ready?(must_be_complete = true) @@ -131,14 +222,18 @@ def media_ready?(must_be_complete = true) end def media_status - states = media.map(&:status).uniq + states = if override? + [external_media_resource&.status] + else + media.map(&:status).uniq + end if !(%w[started created processing retrying] & states).empty? "processing" elsif states.any? { |s| s == "error" } "error" elsif states.any? { |s| s == "invalid" } "invalid" - elsif media_ready? + elsif media_ready? || override_ready? "complete" end end @@ -146,4 +241,29 @@ def media_status def media_url media.first.try(:href) end + + def override_ready? + override? && external_media_ready? + end + + def override_processing? + override? && !external_media_ready? + end + + def override? + medium_override? || !enclosure_override_url.blank? + end + + def external_media_ready? + external_media_resource&.status_complete? + end + + def analyze_external_media + if enclosure_override_url.blank? + external_media_resource&.destroy + elsif enclosure_override_url != external_media_resource&.original_url + external_media_resource&.destroy + create_external_media_resource(original_url: enclosure_override_url) + end + end end diff --git a/app/models/episode.rb b/app/models/episode.rb index 36e9826e1..69ffa909a 100644 --- a/app/models/episode.rb +++ b/app/models/episode.rb @@ -23,22 +23,15 @@ class Episode < ApplicationRecord acts_as_paranoid - serialize :overrides, coder: HashSerializer - belongs_to :podcast, -> { with_deleted }, touch: true has_many :episode_imports - has_many :contents, -> { order("position ASC, created_at DESC") }, autosave: true, dependent: :destroy, inverse_of: :episode - has_many :media_versions, -> { order("created_at DESC") }, dependent: :destroy has_many :images, -> { order("created_at DESC") }, class_name: "EpisodeImage", autosave: true, dependent: :destroy, inverse_of: :episode has_one :ready_image, -> { complete_or_replaced.order("created_at DESC") }, class_name: "EpisodeImage" - has_one :uncut, -> { order("created_at DESC") }, autosave: true, dependent: :destroy, inverse_of: :episode has_one :transcript, -> { order("created_at DESC") }, dependent: :destroy, inverse_of: :episode - accepts_nested_attributes_for :contents, allow_destroy: true, reject_if: ->(c) { c[:id].blank? && c[:original_url].blank? } accepts_nested_attributes_for :images, allow_destroy: true, reject_if: ->(i) { i[:id].blank? && i[:original_url].blank? } - accepts_nested_attributes_for :uncut, allow_destroy: true, reject_if: ->(u) { u[:id].blank? && u[:original_url].blank? } accepts_nested_attributes_for :transcript, allow_destroy: true, reject_if: ->(t) { t[:id].blank? && t[:original_url].blank? } validates :podcast_id, :guid, presence: true @@ -53,12 +46,10 @@ class Episode < ApplicationRecord validates :explicit, inclusion: {in: %w[true false]}, allow_nil: true validates :segment_count, presence: true, if: :strict_validations validates :segment_count, numericality: {only_integer: true, greater_than: 0, less_than_or_equal_to: MAX_SEGMENT_COUNT}, allow_nil: true - validate :validate_media_ready, if: :strict_validations before_validation :set_defaults, :set_external_keyword, :sanitize_text after_save :publish_updated, if: ->(e) { e.published_at_previously_changed? } - after_save :destroy_out_of_range_contents, if: ->(e) { e.segment_count_previously_changed? } scope :published, -> { where("episodes.published_at IS NOT NULL AND episodes.published_at <= now()") } scope :published_by, ->(offset) { where("episodes.published_at IS NOT NULL AND episodes.published_at <= ?", Time.now - offset) } @@ -71,8 +62,6 @@ class Episode < ApplicationRecord scope :dropdate_asc, -> { reorder(Arel.sql("#{DROP_DATE} ASC NULLS FIRST")) } scope :dropdate_desc, -> { reorder(Arel.sql("#{DROP_DATE} DESC NULLS LAST")) } - enum :medium, [:audio, :uncut, :video], prefix: true - alias_attribute :number, :episode_number alias_attribute :season, :season_number @@ -198,36 +187,6 @@ def url_was super || embed_player_landing_url(podcast, self) end - def medium=(new_medium) - super - - if medium_changed? && medium_was.present? - if medium_was == "uncut" && medium == "audio" - uncut&.mark_for_destruction - elsif medium_was == "audio" && medium == "uncut" - if (c = contents.first) - build_uncut.tap do |u| - u.file_size = contents.first.file_size - u.duration = contents.first.duration - - # use the feeder cdn url for older completed files - is_old = (Time.now - c.created_at) > 24.hours - u.original_url = (c.status_complete? && is_old) ? c.url : c.original_url - end - end - contents.each(&:mark_for_destruction) - else - contents.each(&:mark_for_destruction) - end - end - - self.segment_count = 1 if medium_video? - end - - def overrides - self[:overrides] ||= HashWithIndifferentAccess.new - end - def categories self[:categories] || [] end @@ -236,13 +195,6 @@ def categories=(cats) self[:categories] = sanitize_categories(cats, false).presence end - def copy_media(force = false) - contents.each { |c| c.copy_media(force) } - images.each { |i| i.copy_media(force) } - transcript&.copy_media(force) - uncut&.copy_media(force) - end - def publish! Rails.logger.tagged("Episode#publish!") do apple_mark_for_reupload! @@ -306,18 +258,6 @@ def segment_range 1..segment_count.to_i end - def build_contents - segment_range.map do |p| - contents.find { |c| c.position == p } || contents.build(position: p) - end - end - - def destroy_out_of_range_contents - if segment_count.present? && segment_count.positive? - contents.where.not(position: segment_range.to_a).destroy_all - end - end - def published_or_released_date if published_at.present? published_at @@ -326,25 +266,6 @@ def published_or_released_date end end - def validate_media_ready - return unless published_at.present? && media? - - # media must be complete on _initial_ publish - # otherwise - having files in any status is good enough - is_ready = - if published_at_was.blank? - media_ready?(true) - elsif medium_uncut? - uncut.present? && !uncut.marked_for_destruction? - else - media_ready?(false) - end - - unless is_ready - errors.add(:base, :media_not_ready, message: "media not ready") - end - end - def ready_transcript transcript if transcript&.status_complete? end diff --git a/app/models/external_media_resource.rb b/app/models/external_media_resource.rb new file mode 100644 index 000000000..f3a887b0b --- /dev/null +++ b/app/models/external_media_resource.rb @@ -0,0 +1,34 @@ +class ExternalMediaResource < MediaResource + validates :duration, numericality: {greater_than: 0}, if: :status_complete? + after_create_commit :analyze_media + + def guid + self[:guid] + end + + def url + original_url + end + + def media_url + original_url + end + + def copy_media(force = false) + end + + def analyze_media(force = false) + if force || !(status_complete? || task) + Tasks::AnalyzeMediaTask.create! do |task| + task.owner = self + end.start! + end + end + + def retry! + if retryable? + status_retrying! + analyze_media(true) + end + end +end diff --git a/app/models/tasks/analyze_media_task.rb b/app/models/tasks/analyze_media_task.rb new file mode 100644 index 000000000..f14d946f5 --- /dev/null +++ b/app/models/tasks/analyze_media_task.rb @@ -0,0 +1,43 @@ +class Tasks::AnalyzeMediaTask < ::Task + before_save :update_media_resource, if: ->(t) { t.status_changed? && t.media_resource } + + def media_resource + owner + end + + def source_url + media_resource.original_url + end + + def porter_tasks + [{Type: "Inspect"}] + end + + def update_media_resource + media_resource.status = status + + if complete? + info = porter_callback_inspect + media_resource.mime_type = porter_callback_mime + media_resource.medium = (media_resource.mime_type || "").split("/").first + media_resource.file_size = porter_callback_size + + if info[:Audio] + media_resource.sample_rate = info[:Audio][:Frequency].to_i + media_resource.channels = info[:Audio][:Channels].to_i + media_resource.duration = info[:Audio][:Duration].to_f / 1000 + media_resource.bit_rate = info[:Audio][:Bitrate].to_i / 1000 + end + + # only return for actual videos - not detected images in id3 tags + if info[:Video] && media_resource.mime_type&.starts_with?("video") + media_resource.duration = info[:Video][:Duration].to_f / 1000 + media_resource.width = info[:Video][:Width].to_i + media_resource.height = info[:Video][:Height].to_i + media_resource.frame_rate = info[:Video][:Framerate].to_f.round + end + end + + media_resource.save! + end +end diff --git a/app/views/episode_media/_form.html.erb b/app/views/episode_media/_form.html.erb index 9ad71d22e..316192e07 100644 --- a/app/views/episode_media/_form.html.erb +++ b/app/views/episode_media/_form.html.erb @@ -24,9 +24,25 @@
<% if episode.medium_uncut? %> <%= render "form_uncut", episode: episode, form: form %> + <% elsif episode.medium_override? %> + <%# no uploads for overrides %> <% else %> <%= render "form_contents", episode: episode, form: form %> <% end %> +
+
+ <%= form.text_field :enclosure_override_url %> + <%= form.label :enclosure_override_url, required: episode.medium_override? %> + <%= field_help_text t(".help.enclosure_override_url") %> +
+
+ <%= form.check_box :enclosure_override_prefix %> +
+ <%= form.label :enclosure_override_prefix %> + <%= help_text t(".help.enclosure_override_prefix") %> +
+
+
diff --git a/app/views/episode_media/_form_main.html.erb b/app/views/episode_media/_form_main.html.erb index 87a463249..e48d85bc2 100644 --- a/app/views/episode_media/_form_main.html.erb +++ b/app/views/episode_media/_form_main.html.erb @@ -8,8 +8,7 @@ - -<% if episode.medium_video? %> +<% if episode.medium_video? || episode.medium_override? %> <%= form.hidden_field :segment_count, value: 1 %> <% else %>
diff --git a/app/views/episode_media/_form_status.html.erb b/app/views/episode_media/_form_status.html.erb index 692900ad3..48265bac7 100644 --- a/app/views/episode_media/_form_status.html.erb +++ b/app/views/episode_media/_form_status.html.erb @@ -6,7 +6,7 @@