Scale canvas video strategy

demo/src/main/java/com/otaliastudios/transcoder/demo/TranscoderActivity.java

@@ -20,6 +20,10 @@
 import com.otaliastudios.transcoder.engine.TrackStatus;
 import com.otaliastudios.transcoder.engine.TrackType;
 import com.otaliastudios.transcoder.internal.Logger;
+import com.otaliastudios.transcoder.scale.DownVideoScaler;
+import com.otaliastudios.transcoder.scale.StretchVideoScaler;
+import com.otaliastudios.transcoder.scale.UpVideoScaler;
+import com.otaliastudios.transcoder.scale.VideoScaler;
 import com.otaliastudios.transcoder.sink.DataSink;
 import com.otaliastudios.transcoder.sink.DefaultDataSink;
 import com.otaliastudios.transcoder.source.DataSource;
@@ -59,6 +63,7 @@ public class TranscoderActivity extends AppCompatActivity implements
     private RadioGroup mVideoFramesGroup;
     private RadioGroup mVideoResolutionGroup;
     private RadioGroup mVideoAspectGroup;
+    private RadioGroup mVideoScalingGroup;
     private RadioGroup mVideoRotationGroup;
     private RadioGroup mSpeedGroup;
     private RadioGroup mAudioReplaceGroup;
@@ -134,6 +139,7 @@ protected void onCreate(Bundle savedInstanceState) {
         mVideoFramesGroup = findViewById(R.id.frames);
         mVideoResolutionGroup = findViewById(R.id.resolution);
         mVideoAspectGroup = findViewById(R.id.aspect);
+        mVideoScalingGroup = findViewById(R.id.scale);
         mVideoRotationGroup = findViewById(R.id.rotation);
         mSpeedGroup = findViewById(R.id.speed);
         mAudioSampleRateGroup = findViewById(R.id.sampleRate);
@@ -143,6 +149,7 @@ protected void onCreate(Bundle savedInstanceState) {
         mVideoFramesGroup.setOnCheckedChangeListener(mRadioGroupListener);
         mVideoResolutionGroup.setOnCheckedChangeListener(mRadioGroupListener);
         mVideoAspectGroup.setOnCheckedChangeListener(mRadioGroupListener);
+        mVideoScalingGroup.setOnCheckedChangeListener(mRadioGroupListener);
         mAudioSampleRateGroup.setOnCheckedChangeListener(mRadioGroupListener);
         mTrimStartView.addTextChangedListener(mTextListener);
         mTrimEndView.addTextChangedListener(mTextListener);
@@ -296,11 +303,18 @@ private void transcode() {
             default: speed = 1F;
         }
 
+        VideoScaler videoScaler;
+        switch (mVideoScalingGroup.getCheckedRadioButtonId()) {
+            case R.id.scale_down: videoScaler = new DownVideoScaler(); break;
+            case R.id.scale_stretch: videoScaler = new StretchVideoScaler(); break;
+            default: videoScaler = new UpVideoScaler(); break;
+        }
+
         // Launch the transcoding operation.
         mTranscodeStartTime = SystemClock.uptimeMillis();
         setIsTranscoding(true);
         DataSink sink = new DefaultDataSink(mTranscodeOutputFile.getAbsolutePath());
-        TranscoderOptions.Builder builder = Transcoder.into(sink);
+        TranscoderOptions.Builder builder = Transcoder.into(sink).setVideoScaler(videoScaler);
         if (mAudioReplacementUri == null) {
             if (mTranscodeInputUri1 != null) {
                 DataSource source = new UriDataSource(this, mTranscodeInputUri1);

demo/src/main/res/layout/activity_transcoder.xml

@@ -205,6 +205,42 @@
                     android:layout_height="wrap_content" />
             </RadioGroup>
 
+            <!-- VIDEO SCALE METHOD -->
+            <TextView
+                android:padding="16dp"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:text="Video Scaling Method" />
+            <RadioGroup
+                android:id="@+id/scale"
+                android:checkedButton="@id/scale_up"
+                android:orientation="horizontal"
+                android:gravity="center"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content">
+                <com.google.android.material.radiobutton.MaterialRadioButton
+                    android:id="@+id/scale_up"
+                    android:text="Up"
+                    android:paddingLeft="8dp"
+                    android:paddingRight="8dp"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content" />
+                <com.google.android.material.radiobutton.MaterialRadioButton
+                    android:id="@+id/scale_down"
+                    android:text="down"
+                    android:paddingLeft="8dp"
+                    android:paddingRight="8dp"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content" />
+                <com.google.android.material.radiobutton.MaterialRadioButton
+                    android:id="@+id/scale_stretch"
+                    android:text="stretch"
+                    android:paddingLeft="8dp"
+                    android:paddingRight="8dp"
+                    android:layout_width="wrap_content"
+                    android:layout_height="wrap_content" />
+            </RadioGroup>
+
             <!-- VIDEO ROTATION -->
             <TextView
                 android:padding="16dp"

lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java

@@ -8,6 +8,8 @@
 import com.otaliastudios.transcoder.engine.TrackType;
 import com.otaliastudios.transcoder.resample.AudioResampler;
 import com.otaliastudios.transcoder.resample.DefaultAudioResampler;
+import com.otaliastudios.transcoder.scale.UpVideoScaler;
+import com.otaliastudios.transcoder.scale.VideoScaler;
 import com.otaliastudios.transcoder.sink.DataSink;
 import com.otaliastudios.transcoder.sink.DefaultDataSink;
 import com.otaliastudios.transcoder.source.DataSource;
@@ -51,6 +53,7 @@ private TranscoderOptions() {}
     private TimeInterpolator timeInterpolator;
     private AudioStretcher audioStretcher;
     private AudioResampler audioResampler;
+    private VideoScaler videoScaler;
 
     TranscoderListener listener;
     Handler listenerHandler;
@@ -104,6 +107,11 @@ public AudioResampler getAudioResampler() {
         return audioResampler;
     }
 
+    @NonNull
+    public VideoScaler getVideoScaler() {
+        return videoScaler;
+    }
+
     public static class Builder {
         private DataSink dataSink;
         private final List<DataSource> audioDataSources = new ArrayList<>();
@@ -117,6 +125,7 @@ public static class Builder {
         private TimeInterpolator timeInterpolator;
         private AudioStretcher audioStretcher;
         private AudioResampler audioResampler;
+        private VideoScaler videoScaler;
 
         Builder(@NonNull String outPath) {
             this.dataSink = new DefaultDataSink(outPath);
@@ -199,13 +208,13 @@ public Builder setAudioTrackStrategy(@Nullable TrackStrategy trackStrategy) {
          * Sets the video output strategy. If absent, this defaults to the 16:9
          * strategy returned by {@link DefaultVideoStrategies#for720x1280()}.
          *
-         * @param trackStrategy the desired strategy
+         * @param videoTrackStrategy the desired strategy
          * @return this for chaining
          */
         @NonNull
         @SuppressWarnings("unused")
-        public Builder setVideoTrackStrategy(@Nullable TrackStrategy trackStrategy) {
-            this.videoTrackStrategy = trackStrategy;
+        public Builder setVideoTrackStrategy(@Nullable TrackStrategy videoTrackStrategy) {
+            this.videoTrackStrategy = videoTrackStrategy;
             return this;
         }
 
@@ -318,6 +327,18 @@ public Builder setAudioResampler(@NonNull AudioResampler audioResampler) {
             return this;
         }
 
+        /**
+         * Set an {@link VideoScaler} to change the resolution of the video frames
+         * so that they fit the new resolution
+         *
+         */
+        @NonNull
+        @SuppressWarnings("unused")
+        public Builder setVideoScaler(@NonNull VideoScaler videoScaler) {
+            this.videoScaler = videoScaler;
+            return this;
+        }
+
         /**
          * Generates muted audio data sources if needed
          * @return The list of audio data sources including the muted sources
@@ -389,6 +410,9 @@ public TranscoderOptions build() {
             if (audioResampler == null) {
                 audioResampler = new DefaultAudioResampler();
             }
+            if (videoScaler == null) {
+                videoScaler = new UpVideoScaler();
+            }
             TranscoderOptions options = new TranscoderOptions();
             options.listener = listener;
             options.audioDataSources = buildAudioDataSources();
@@ -402,6 +426,7 @@ public TranscoderOptions build() {
             options.timeInterpolator = timeInterpolator;
             options.audioStretcher = audioStretcher;
             options.audioResampler = audioResampler;
+            options.videoScaler = videoScaler;
             return options;
         }
 

lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java

@@ -171,6 +171,7 @@ private void openCurrentStep(@NonNull TrackType type, @NonNull TranscoderOptions
                     case VIDEO:
                         transcoder = new VideoTrackTranscoder(dataSource, mDataSink,
                                 interpolator,
+                                options.getVideoScaler(),
                                 options.getVideoRotation());
                         break;
                     case AUDIO:

lib/src/main/java/com/otaliastudios/transcoder/scale/DownVideoScaler.java

@@ -0,0 +1,20 @@
+package com.otaliastudios.transcoder.scale;
+
+import androidx.annotation.NonNull;
+
+import com.otaliastudios.transcoder.transcode.internal.VideoDecoderOutput;
+
+/**
+ * An {@link VideoScaler} that scale the video down so that no side exceed the new resolution
+ * Sides that are too small will have black borders
+ */
+public class DownVideoScaler implements VideoScaler {
+    @Override
+    public void scaleOutput(@NonNull VideoDecoderOutput videoDecoderOutput, float scaleX, float scaleY, boolean flipped) {
+        if (flipped) { // The drawable is not affected by the flip so we need to reverse it
+            videoDecoderOutput.setDrawableScale(1.0F / scaleX, 1.0F / scaleY);
+        } else {
+            videoDecoderOutput.setDrawableScale(1.0F / scaleY, 1.0F / scaleX);
+        }
+    }
+}

lib/src/main/java/com/otaliastudios/transcoder/scale/StretchVideoScaler.java

@@ -0,0 +1,16 @@
+package com.otaliastudios.transcoder.scale;
+
+import androidx.annotation.NonNull;
+
+import com.otaliastudios.transcoder.transcode.internal.VideoDecoderOutput;
+
+/**
+ * An {@link VideoScaler} that strech the video so that they match the video resolution
+ * at the cost of deforming the images
+ */
+public class StretchVideoScaler implements VideoScaler {
+    @Override
+    public void scaleOutput(@NonNull VideoDecoderOutput videoDecoderOutput, float scaleX, float scaleY, boolean flipped) {
+        //No scaling will automatically stretch the frames to fill all the drawable space
+    }
+}

lib/src/main/java/com/otaliastudios/transcoder/scale/UpVideoScaler.java

@@ -0,0 +1,16 @@
+package com.otaliastudios.transcoder.scale;
+
+import androidx.annotation.NonNull;
+
+import com.otaliastudios.transcoder.transcode.internal.VideoDecoderOutput;
+
+/**
+ * An {@link VideoScaler} that scale the video up so that it touches all sides
+ * of the new resolution and exceeding parts will be truncated
+ */
+public class UpVideoScaler implements VideoScaler {
+    @Override
+    public void scaleOutput(@NonNull VideoDecoderOutput videoDecoderOutput, float scaleX, float scaleY, boolean flipped) {
+        videoDecoderOutput.setScale(scaleX, scaleY);
+    }
+}

lib/src/main/java/com/otaliastudios/transcoder/scale/VideoScaler.java

@@ -0,0 +1,22 @@
+package com.otaliastudios.transcoder.scale;
+
+import androidx.annotation.NonNull;
+
+import com.otaliastudios.transcoder.transcode.internal.VideoDecoderOutput;
+
+/**
+ * Scale the frames when they are not of the expected resolution
+ */
+public interface VideoScaler {
+    /**
+     * Apply the scaling to the video decoder output
+     *
+     * It can be done using VideoDecoderOutput.setScale or VideoDecoderOutput.setDrawableScale
+     *
+     * @param videoDecoderOutput the video decoder output
+     * @param scaleX the input width/height
+     * @param scaleY the output width/height
+     * @param flipped whether or not the frame was rotated by 90 degrees
+     */
+    void scaleOutput(@NonNull VideoDecoderOutput videoDecoderOutput, float scaleX, float scaleY, boolean flipped);
+}

lib/src/main/java/com/otaliastudios/transcoder/strategy/DefaultVideoStrategy.java

@@ -16,6 +16,7 @@
 import com.otaliastudios.transcoder.strategy.size.Resizer;
 import com.otaliastudios.transcoder.internal.Logger;
 import com.otaliastudios.transcoder.internal.MediaFormatConstants;
+import com.otaliastudios.transcoder.transcode.internal.VideoDecoderOutput;
 
 import androidx.annotation.NonNull;
 

lib/src/main/java/com/otaliastudios/transcoder/strategy/TrackStrategy.java

@@ -2,12 +2,11 @@
 
 import android.media.MediaFormat;
 
+import androidx.annotation.NonNull;
+
 import com.otaliastudios.transcoder.engine.TrackStatus;
 import com.otaliastudios.transcoder.strategy.size.Resizer;
 
-import androidx.annotation.NonNull;
-import androidx.annotation.Nullable;
-
 import java.util.List;
 
 /**

lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java

@@ -16,20 +16,20 @@
 package com.otaliastudios.transcoder.transcode;
 
 import android.media.MediaCodec;
-import android.media.MediaExtractor;
 import android.media.MediaFormat;
 
 import androidx.annotation.NonNull;
 
 import com.otaliastudios.transcoder.engine.TrackType;
+import com.otaliastudios.transcoder.internal.Logger;
 import com.otaliastudios.transcoder.internal.MediaCodecBuffers;
+import com.otaliastudios.transcoder.internal.MediaFormatConstants;
+import com.otaliastudios.transcoder.scale.VideoScaler;
 import com.otaliastudios.transcoder.sink.DataSink;
 import com.otaliastudios.transcoder.source.DataSource;
 import com.otaliastudios.transcoder.time.TimeInterpolator;
 import com.otaliastudios.transcoder.transcode.internal.VideoDecoderOutput;
 import com.otaliastudios.transcoder.transcode.internal.VideoEncoderInput;
-import com.otaliastudios.transcoder.internal.Logger;
-import com.otaliastudios.transcoder.internal.MediaFormatConstants;
 import com.otaliastudios.transcoder.transcode.internal.VideoFrameDropper;
 
 import java.nio.ByteBuffer;
@@ -46,16 +46,19 @@ public class VideoTrackTranscoder extends BaseTrackTranscoder {
     private MediaCodec mEncoder; // Keep this since we want to signal EOS on it.
     private VideoFrameDropper mFrameDropper;
     private final TimeInterpolator mTimeInterpolator;
+    private final VideoScaler mVideoScaler;
     private final int mSourceRotation;
     private final int mExtraRotation;
 
     public VideoTrackTranscoder(
             @NonNull DataSource dataSource,
             @NonNull DataSink dataSink,
             @NonNull TimeInterpolator timeInterpolator,
+            @NonNull VideoScaler videoScaler,
             int rotation) {
         super(dataSource, dataSink, TrackType.VIDEO);
         mTimeInterpolator = timeInterpolator;
+        mVideoScaler = videoScaler;
         mSourceRotation = dataSource.getOrientation();
         mExtraRotation = rotation;
     }
@@ -130,7 +133,7 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF
         } else if (inputRatio < outputRatio) { // Input taller. We have a scaleY.
             scaleY = outputRatio / inputRatio;
         }
-        mDecoderOutputSurface.setScale(scaleX, scaleY);
+        mVideoScaler.scaleOutput(mDecoderOutputSurface, scaleX, scaleY, flip);
     }
 
     @Override

lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java

@@ -87,6 +87,21 @@ public void setScale(float scaleX, float scaleY) {
         mScaleY = scaleY;
     }
 
+    /**
+     * Scale the canvas along the two axes.
+     * @param scaleX x scale
+     * @param scaleY y scale
+     */
+    @SuppressWarnings("unused")
+    public void setDrawableScale(float scaleX, float scaleY) {
+        mDrawable.setRect(
+                -1.0F  * scaleX,
+                -1.0F * scaleY,
+                1.0F * scaleX,
+                1.0F * scaleY
+        );
+    }
+
     /**
      * Sets the desired frame rotation with respect
      * to its natural orientation.