Merge pull request #7 from projecthydra-labs/elastic_transcoder

Add AWS Elastic Transcoder support

.gitignore

@@ -14,3 +14,4 @@
 mkmf.log
 coverage
 .ruby-version
+.byebug_history

.rubocop.yml

@@ -19,25 +19,18 @@ Metrics/AbcSize:
 
 Metrics/MethodLength:
   Exclude:
-    - 'lib/active_encode/engine_adapters/matterhorn_adapter.rb'
-    - 'lib/active_encode/engine_adapters/zencoder_adapter.rb'
-    - 'lib/active_encode/engine_adapters/shingoncoder_adapter.rb'
+    - 'lib/active_encode/engine_adapters/*'
 
 Metrics/ClassLength:
   Exclude:
-    - 'lib/active_encode/engine_adapters/matterhorn_adapter.rb'
-    - 'lib/active_encode/engine_adapters/zencoder_adapter.rb'
-    - 'lib/active_encode/engine_adapters/shingoncoder_adapter.rb'
+    - 'lib/active_encode/engine_adapters/*'
 
 Metrics/CyclomaticComplexity:
   Exclude:
-    - 'lib/active_encode/engine_adapters/matterhorn_adapter.rb'
-    - 'lib/active_encode/engine_adapters/zencoder_adapter.rb'
-    - 'lib/active_encode/engine_adapters/shingoncoder_adapter.rb'
+    - 'lib/active_encode/engine_adapters/*'
 
 Style/FileName:
   Exclude:
-    - 'lib/active-encode.rb' # Remove this when https://github.com/projecthydra-labs/active-encode/issues/1 is fixed
 
 Style/IndentationConsistency:
   EnforcedStyle: rails

Gemfile

@@ -9,3 +9,4 @@ gem 'rubocop-rspec', require: false
 gem 'rubyhorn', git: "https://github.com/avalonmediasystem/rubyhorn.git"
 gem 'zencoder'
 gem 'shingoncoder'
+gem 'aws-sdk'

README.md

@@ -29,14 +29,14 @@ ActiveEncode::Base.create(File.open('spec/fixtures/Bars_512kb.mp4'))
 Create returns an encoding job that has been submitted to the encoding engine for processing.  At this point it will have an id, a state, the input, and any additional information the encoding engine returns.
 
 ```ruby
-#<ActiveEncode::Base:0x00000003f3cd90 @input="http://localhost:8080/files/mediapackage/edcac316-1f98-44b1-88ca-0ce6f80aebc0/ff43c56f-7b8f-4d9c-a846-6e51de2e8cb4/Bars_512kb.mp4", @options={:preset=>"avalon", :stream_base=>"file:///home/cjcolvar/Code/avalon/avalon/red5/webapps/avalon/streams"}, @id="12154", @state=:running, @current_operations=[], @percent_complete=0.0, @output=[], @errors=[], @tech_metadata={}> 
+#<ActiveEncode::Base:0x00000003f3cd90 @input="http://localhost:8080/files/mediapackage/edcac316-1f98-44b1-88ca-0ce6f80aebc0/ff43c56f-7b8f-4d9c-a846-6e51de2e8cb4/Bars_512kb.mp4", @options={:preset=>"avalon", :stream_base=>"file:///home/cjcolvar/Code/avalon/avalon/red5/webapps/avalon/streams"}, @id="12154", @state=:running, @current_operations=[], @percent_complete=0.0, @output=[], @errors=[], @tech_metadata={}>
 ```
 ```ruby
 encode.id  # "12103"
 encode.state  # :running
 ```
 
-This encode can be looked back up later using #find.  Alternatively, use #reload to refresh an instance with the latest information from the 
+This encode can be looked back up later using #find.  Alternatively, use #reload to refresh an instance with the latest information from the
 
 ```ruby
 encode = ActiveEncode::Base.find("12103")
@@ -85,17 +85,15 @@ end
 
 Engine adapters are shims between ActiveEncode and the back end encoding service.  Each service has its own API and idiosyncracies so consult the table below to see what features are supported by each adapter.  Add an additional engines by creating an engine adapter class that implements :create, :find, :list, :cancel, :purge, and :remove_output.
 
-| Feature | Matterhorn Adapter | Zencoder Adapter (prototype) | Shingoncoder | Inline Adapter (In progress) | Test Adapter |
-| --- | --- | --- | --- | --- |
-| Create | X | X | X | X | X |
-| Find | X | X | X | X	| X	|
-| List	| |	| | | |
-| Cancel | X | X | | | X |
-| Purge	| X	| |  | | X	|
-| Remove output	| X	| | | | |
-| Preset | X | | | | |
-| Multiple outputs | X (via preset)	| | |	| |
-
+| Adapter/Feature          | Create | Find | List | Cancel | Purge | Remove Output | Preset | Multiple Outputs |
+|--------------------------|--------|------|------|--------|-------|---------------|--------|------------------|
+| Matterhorn               |    X   |  X   |      |   X    |   X   |       X       |   X    |         X        |
+| Zencoder                 |    X   |  X   |      |   X    |       |               |        |                  |
+| Shingoncoder (prototype) |    X   |  X   |      |        |       |               |        |                  |
+| Shingoncoder             |    X   |  X   |      |        |       |               |        |                  |
+| AWS Elastic Transcoder   |    X   |  X   |      |   X    |       |               |        |                  |
+| Inline                   |    X   |  X   |      |        |       |               |        |                  |
+| Test                     |    X   |  X   |      |   X    |       |       X       |        |                  |
 
 ## Contributing
 

lib/active_encode/engine_adapters.rb

@@ -13,6 +13,7 @@ module EngineAdapters
     autoload :InlineAdapter
     autoload :ZencoderAdapter
     autoload :ShingoncoderAdapter
+    autoload :ElasticTranscoderAdapter
     autoload :TestAdapter
 
     ADAPTER = 'Adapter'.freeze

lib/active_encode/engine_adapters/elastic_transcoder_adapter.rb

@@ -0,0 +1,152 @@
+module ActiveEncode
+  module EngineAdapters
+    class ElasticTranscoderAdapter
+      # TODO: add a stub for an input helper (supplied by an initializer) that transforms encode.input into a zencoder accepted url
+      def create(encode)
+        job = client.create_job(
+          input: { key: encode.input },
+          pipeline_id: encode.options[:pipeline_id],
+          output_key_prefix: encode.options[:output_key_prefix],
+          outputs: encode.options[:outputs],
+          user_metadata: encode.options[:user_metadata]
+        ).job
+
+        build_encode(get_job_details(job.id), encode.class)
+      end
+
+      def find(id, opts = {})
+        build_encode(get_job_details(id), opts[:cast])
+      end
+
+      # TODO: implement list_jobs_by_pipeline and list_jobs_by_status
+      def list(*_filters)
+        raise NotImplementedError
+      end
+
+      # Can only cancel jobs with status = "Submitted"
+      def cancel(encode)
+        response = client.cancel_job(id: encode.id)
+        build_encode(get_job_details(encode.id), encode.class) if response.successful?
+      end
+
+      def purge(_encode)
+        raise NotImplementedError
+      end
+
+      def remove_output(_encode, _output_id)
+        raise NotImplementedError
+      end
+
+      private
+
+        # Needs region and credentials setup per http://docs.aws.amazon.com/sdkforruby/api/Aws/ElasticTranscoder/Client.html
+        def client
+          @client ||= Aws::ElasticTranscoder::Client.new
+        end
+
+        def get_job_details(job_id)
+          client.read_job(id: job_id).job
+        end
+
+        def build_encode(job, cast)
+          return nil if job.nil?
+          encode = cast.new(convert_input(job), convert_options(job))
+          encode.id = job.id
+          encode.state = convert_state(job)
+          encode.current_operations = convert_current_operations(job)
+          encode.percent_complete = convert_percent_complete(job)
+          encode.created_at = convert_time(job.timing["submit_time_millis"])
+          encode.updated_at = convert_time(job.timing["start_time_millis"])
+          encode.finished_at = convert_time(job.timing["finish_time_millis"])
+          encode.output = convert_output(job)
+          encode.errors = convert_errors(job)
+          encode.tech_metadata = convert_tech_metadata(job.input.detected_properties)
+          encode
+        end
+
+        def convert_time(time_millis)
+          return nil if time_millis.nil?
+          Time.at(time_millis / 1000).iso8601
+        end
+
+        def convert_state(job)
+          case job.status
+          when "Submitted", "Progressing" # Should there be a queued state?
+            :running
+          when "Canceled"
+            :cancelled
+          when "Error"
+            :failed
+          when "Complete"
+            :completed
+          end
+        end
+
+        def convert_current_operations(_job)
+          current_ops = []
+          current_ops
+        end
+
+        def convert_percent_complete(job)
+          case job.status
+          when "Submitted"
+            10
+          when "Progressing"
+            50
+          when "Complete"
+            100
+          else
+            0
+          end
+        end
+
+        def convert_input(job)
+          job.input
+        end
+
+        def convert_options(_job_details)
+          {}
+        end
+
+        def convert_output(job)
+          output = []
+          job.outputs.each do |o|
+            # It is assumed that the first part of the output key can be used to label the  output
+            # e.g. "quality-medium/somepath/filename.flv"
+            label = o.key.split("/", 2).first
+            url = job.output_key_prefix + o.key
+            extras = { id: o.id, url: url, label: label }
+            extras[:hls_url] = url + ".m3u8" if url.include?("/hls/") # TODO: find a better way to signal hls
+            output << convert_tech_metadata(o).merge(extras)
+          end
+          output
+        end
+
+        def convert_errors(job)
+          job.outputs.select { |o| o.status == "Error" }.collect(&:status_detail).compact
+        end
+
+        def convert_tech_metadata(props)
+          return {} if props.nil? || props.empty?
+          metadata_fields = {
+            file_size: { key: :file_size, method: :itself },
+            duration_millis: { key: :duration, method: :to_s },
+            frame_rate: { key: :video_framerate, method: :itself },
+            segment_duration: { key: :segment_duration, method: :itself },
+            width: { key: :width, method: :itself },
+            height: { key: :height, method: :itself }
+          }
+
+          metadata = {}
+          props.each_pair do |key, value|
+            next if value.nil?
+            conversion = metadata_fields[key.to_sym]
+            next if conversion.nil?
+            metadata[conversion[:key]] = value.send(conversion[:method])
+          end
+
+          metadata
+        end
+    end
+  end
+end

lib/active_encode/version.rb

@@ -1,3 +1,3 @@
 module ActiveEncode
-  VERSION = '0.0.3'.freeze
+  VERSION = '0.1.0'.freeze
 end

spec/fixtures/elastic_transcoder/input_completed.json

@@ -0,0 +1 @@
+{"key":"somefile.mp4","detected_properties":{"width":1280,"height":720,"frame_rate":"25","file_size":21069678,"duration_millis":117312}}

spec/fixtures/elastic_transcoder/input_generic.json

@@ -0,0 +1 @@
+{"key":"somefile.mp4"}

spec/fixtures/elastic_transcoder/input_progressing.json

@@ -0,0 +1 @@
+{"key":"somefile.mp4","detected_properties":{"width":1280,"height":720,"frame_rate":"25","file_size":21069678,"duration_millis":117312}}

spec/fixtures/elastic_transcoder/job_canceled.json

@@ -0,0 +1 @@
+{"id":"1471963629141-kmcocm","arn":"arn:aws:elastictranscoder:us-west-2:039358184980:job/1471963629141-kmcocm","pipeline_id":"1470797373927-1ukxrn","output_key_prefix":"elastic-transcoder-samples/output/hls/","playlists":[],"status":"Canceled","timing":{"submit_time_millis":1471963629189,"finish_time_millis":1471975185271}}

spec/fixtures/elastic_transcoder/job_completed.json

@@ -0,0 +1 @@
+{"id":"1471963629141-kmcocm","arn":"arn:aws:elastictranscoder:us-west-2:039358184980:job/1471963629141-kmcocm","pipeline_id":"1470797373927-1ukxrn","output_key_prefix":"elastic-transcoder-samples/output/hls/","playlists":[],"status":"Complete","timing":{"submit_time_millis":1471963629189,"start_time_millis":1471975169865,"finish_time_millis":1471975185271}}

spec/fixtures/elastic_transcoder/job_created.json

@@ -0,0 +1 @@
+{"id":"1471963629141-kmcocm","arn":"arn:aws:elastictranscoder:us-west-2:039358184980:job/1471963629141-kmcocm","pipeline_id":"1470797373927-1ukxrn","input":{"key":"somefile.mp4"},"output":{"id":"1","key":"hlsAudio/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a","preset_id":"1351620000001-200071","segment_duration":"2.0","status":"Submitted","watermarks":[]},"output_key_prefix":"elastic-transcoder-samples/output/hls/","playlists":[],"status":"Submitted","timing":{"submit_time_millis":1471963629189}}

spec/fixtures/elastic_transcoder/job_failed.json

@@ -0,0 +1 @@
+{"id":"1471963629141-kmcocm","arn":"arn:aws:elastictranscoder:us-west-2:039358184980:job/1471963629141-kmcocm","pipeline_id":"1470797373927-1ukxrn","output_key_prefix":"elastic-transcoder-samples/output/hls/","playlists":[],"status":"Error","timing":{"submit_time_millis":1471963629189,"start_time_millis":1471975169865,"finish_time_millis":1471975185271}}

spec/fixtures/elastic_transcoder/job_progressing.json

@@ -0,0 +1 @@
+{"id":"1471963629141-kmcocm","arn":"arn:aws:elastictranscoder:us-west-2:039358184980:job/1471963629141-kmcocm","pipeline_id":"1470797373927-1ukxrn","output_key_prefix":"elastic-transcoder-samples/output/hls/","playlists":[],"status":"Progressing","timing":{"submit_time_millis":1471963629189,"start_time_millis":1471975169865}}

spec/fixtures/elastic_transcoder/output_canceled.json

@@ -0,0 +1 @@
+{"id":"2","key":"hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a","preset_id":"1351620000001-200050","segment_duration":"2.0","status":"Canceled","watermarks":[]}

spec/fixtures/elastic_transcoder/output_completed.json

@@ -0,0 +1 @@
+{"id":"2","key":"hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a","preset_id":"1351620000001-200050","segment_duration":"2.0","status":"Complete","status_detail":"The input file for this job contains 6 audio channels and the preset is configured for 2 audio channels. Amazon Elastic Transcoder audio-channel mapping may not result in the desired audio. Some individual segment files for this output have a higher bit rate than the average bit rate of the transcoded media. Playlists including this output will record a higher bit rate than the rate specified by the preset.","duration":118,"width":400,"height":224,"frame_rate":"25","file_size":6901104,"duration_millis":117353,"watermarks":[]}

spec/fixtures/elastic_transcoder/output_failed.json

@@ -0,0 +1 @@
+{"id":"2","key":"hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a","preset_id":"1351620000001-200050","segment_duration":"2.0","status":"Error","status_detail":"3002 0419623b-342f-4bde-9e3a-439477a95f2c: The specified object could not be saved in the specified bucket because an object by that name already exists: bucket=elasticbeanstalk-us-west-2-039358184980, key=elastic-transcoder-samples/output/hls/hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a00000.ts.","watermarks":[]}

spec/fixtures/elastic_transcoder/output_progressing.json

@@ -0,0 +1 @@
+{"id":"2","key":"hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a","preset_id":"1351620000001-200050","segment_duration":"2.0","status":"Progressing","watermarks":[]}

spec/fixtures/elastic_transcoder/output_submitted.json

@@ -0,0 +1 @@
+{"id":"2","key":"hls0400k/e8fe80f5b7063b12d567b90c0bdf6322116bba11ac458fe9d62921644159fe4a","preset_id":"1351620000001-200050","segment_duration":"2.0","status":"Submitted","watermarks":[]}