Added ability to override implementation of VideoEncoderInput and Vid…

…eoDecoderOutput for custom video postproccessing
deepmedia · x0rb0t · Jul 25, 2020 · Jul 25, 2020 · Jul 25, 2020 · Sep 29, 2020
commit 0515ae82fd023a962ea6dd71975b5de5290056d2
diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java
@@ -7,6 +7,8 @@
 import android.os.Looper;
 
 import com.otaliastudios.transcoder.engine.TrackType;
+import com.otaliastudios.transcoder.io_factory.DecoderIOFactory;
+import com.otaliastudios.transcoder.io_factory.DefaultDecoderIOFactory;
 import com.otaliastudios.transcoder.resample.AudioResampler;
 import com.otaliastudios.transcoder.resample.DefaultAudioResampler;
 import com.otaliastudios.transcoder.sink.DataSink;
@@ -44,6 +46,7 @@ public class TranscoderOptions {
     private TranscoderOptions() {}
 
     private DataSink dataSink;
+    private DecoderIOFactory decoderIOFactory;
     private List<DataSource> videoDataSources;
     private List<DataSource> audioDataSources;
     private TrackStrategy audioTrackStrategy;
@@ -62,6 +65,11 @@ public DataSink getDataSink() {
         return dataSink;
     }
 
+    @NonNull
+    public DecoderIOFactory getDecoderIOFactory() {
+        return decoderIOFactory;
+    }
+
     @NonNull
     public List<DataSource> getAudioDataSources() {
         return audioDataSources;
@@ -108,6 +116,7 @@ public AudioResampler getAudioResampler() {
 
     public static class Builder {
         private DataSink dataSink;
+        private DecoderIOFactory decoderIOFactory;
         private final List<DataSource> audioDataSources = new ArrayList<>();
         private final List<DataSource> videoDataSources = new ArrayList<>();
         private TranscoderListener listener;
@@ -396,11 +405,16 @@ public TranscoderOptions build() {
             if (audioResampler == null) {
                 audioResampler = new DefaultAudioResampler();
             }
+            if (decoderIOFactory == null) {
+                decoderIOFactory = new DefaultDecoderIOFactory();
+            }
+
             TranscoderOptions options = new TranscoderOptions();
             options.listener = listener;
             options.audioDataSources = buildAudioDataSources();
             options.videoDataSources = videoDataSources;
             options.dataSink = dataSink;
+            options.decoderIOFactory = decoderIOFactory;
             options.listenerHandler = listenerHandler;
             options.audioTrackStrategy = audioTrackStrategy;
             options.videoTrackStrategy = videoTrackStrategy;

diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java
@@ -170,6 +170,7 @@ private void openCurrentStep(@NonNull TrackType type, @NonNull TranscoderOptions
                 switch (type) {
                     case VIDEO:
                         transcoder = new VideoTrackTranscoder(dataSource, mDataSink,
+                                options.getDecoderIOFactory(),
                                 interpolator,
                                 options.getVideoRotation());
                         break;

diff --git a/lib/src/main/java/com/otaliastudios/transcoder/io_factory/DecoderIOFactory.java b/lib/src/main/java/com/otaliastudios/transcoder/io_factory/DecoderIOFactory.java
@@ -0,0 +1,13 @@
+package com.otaliastudios.transcoder.io_factory;
+
+import android.view.Surface;
+
+import com.otaliastudios.transcoder.transcode.base.VideoDecoderOutputBase;
+import com.otaliastudios.transcoder.transcode.base.VideoEncoderInputBase;
+
+import org.jetbrains.annotations.NotNull;
+
+public interface DecoderIOFactory {
+    @NotNull VideoEncoderInputBase createVideoInput(@NotNull Surface surface);
+    @NotNull VideoDecoderOutputBase createVideoOutput();
+}
diff --git a/lib/src/main/java/com/otaliastudios/transcoder/io_factory/DefaultDecoderIOFactory.java b/lib/src/main/java/com/otaliastudios/transcoder/io_factory/DefaultDecoderIOFactory.java
@@ -0,0 +1,24 @@
+package com.otaliastudios.transcoder.io_factory;
+
+import android.view.Surface;
+
+import com.otaliastudios.transcoder.transcode.internal.VideoDecoderOutput;
+import com.otaliastudios.transcoder.transcode.internal.VideoEncoderInput;
+import com.otaliastudios.transcoder.transcode.base.VideoDecoderOutputBase;
+import com.otaliastudios.transcoder.transcode.base.VideoEncoderInputBase;
+
+import org.jetbrains.annotations.NotNull;
+
+public class DefaultDecoderIOFactory implements DecoderIOFactory {
+    @NotNull
+    @Override
+    public VideoEncoderInputBase createVideoInput(@NotNull Surface surface) {
+        return new VideoEncoderInput(surface);
+    }
+
+    @NotNull
+    @Override
+    public VideoDecoderOutputBase createVideoOutput() {
+        return new VideoDecoderOutput();
+    }
+}
diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/VideoTrackTranscoder.java
@@ -16,21 +16,23 @@
 package com.otaliastudios.transcoder.transcode;
 
 import android.media.MediaCodec;
-import android.media.MediaExtractor;
 import android.media.MediaFormat;
 
 import androidx.annotation.NonNull;
 
 import com.otaliastudios.transcoder.engine.TrackType;
 import com.otaliastudios.transcoder.internal.MediaCodecBuffers;
+import com.otaliastudios.transcoder.io_factory.DecoderIOFactory;
 import com.otaliastudios.transcoder.sink.DataSink;
 import com.otaliastudios.transcoder.source.DataSource;
 import com.otaliastudios.transcoder.time.TimeInterpolator;
-import com.otaliastudios.transcoder.transcode.internal.VideoDecoderOutput;
-import com.otaliastudios.transcoder.transcode.internal.VideoEncoderInput;
 import com.otaliastudios.transcoder.internal.Logger;
 import com.otaliastudios.transcoder.internal.MediaFormatConstants;
 import com.otaliastudios.transcoder.transcode.internal.VideoFrameDropper;
+import com.otaliastudios.transcoder.transcode.base.VideoDecoderOutputBase;
+import com.otaliastudios.transcoder.transcode.base.VideoEncoderInputBase;
+
+import org.jetbrains.annotations.NotNull;
 
 import java.nio.ByteBuffer;
 
@@ -41,20 +43,24 @@ public class VideoTrackTranscoder extends BaseTrackTranscoder {
     @SuppressWarnings("unused")
     private static final Logger LOG = new Logger(TAG);
 
-    private VideoDecoderOutput mDecoderOutputSurface;
-    private VideoEncoderInput mEncoderInputSurface;
+    private VideoDecoderOutputBase mDecoderOutputSurface;
+    private VideoEncoderInputBase mEncoderInputSurface;
     private MediaCodec mEncoder; // Keep this since we want to signal EOS on it.
     private VideoFrameDropper mFrameDropper;
+    private DecoderIOFactory mDecoderIOFactory;
     private final TimeInterpolator mTimeInterpolator;
     private final int mSourceRotation;
     private final int mExtraRotation;
 
+
     public VideoTrackTranscoder(
             @NonNull DataSource dataSource,
             @NonNull DataSink dataSink,
+            @NotNull DecoderIOFactory decoderIOFactory,
             @NonNull TimeInterpolator timeInterpolator,
             int rotation) {
         super(dataSource, dataSink, TrackType.VIDEO);
+        mDecoderIOFactory = decoderIOFactory;
         mTimeInterpolator = timeInterpolator;
         mSourceRotation = dataSource.getOrientation();
         mExtraRotation = rotation;
@@ -76,7 +82,7 @@ protected void onConfigureEncoder(@NonNull MediaFormat format, @NonNull MediaCod
 
     @Override
     protected void onStartEncoder(@NonNull MediaFormat format, @NonNull MediaCodec encoder) {
-        mEncoderInputSurface = new VideoEncoderInput(encoder.createInputSurface());
+        mEncoderInputSurface = mDecoderIOFactory.createVideoInput(encoder.createInputSurface());
         super.onStartEncoder(format, encoder);
     }
 
@@ -98,10 +104,7 @@ protected void onConfigureDecoder(@NonNull MediaFormat format, @NonNull MediaCod
         // refer: https://android.googlesource.com/platform/frameworks/av/+blame/lollipop-release/media/libstagefright/Utils.cpp
         format.setInteger(MediaFormatConstants.KEY_ROTATION_DEGREES, 0);
 
-        // The rotation we should apply is the intrinsic source rotation, plus any extra
-        // rotation that was set into the TranscoderOptions.
-        mDecoderOutputSurface = new VideoDecoderOutput();
-        mDecoderOutputSurface.setRotation((mSourceRotation + mExtraRotation) % 360);
+        mDecoderOutputSurface = mDecoderIOFactory.createVideoOutput();
         decoder.configure(format, mDecoderOutputSurface.getSurface(), null, 0);
     }
 
@@ -112,25 +115,7 @@ protected void onCodecsStarted(@NonNull MediaFormat inputFormat, @NonNull MediaF
                 inputFormat.getInteger(MediaFormat.KEY_FRAME_RATE),
                 outputFormat.getInteger(MediaFormat.KEY_FRAME_RATE));
         mEncoder = encoder;
-
-        // Cropping support.
-        // Ignoring any outputFormat KEY_ROTATION (which is applied at playback time), the rotation
-        // difference between input and output is mSourceRotation + mExtraRotation.
-        int rotation = (mSourceRotation + mExtraRotation) % 360;
-        boolean flip = (rotation % 180) != 0;
-        float inputWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH);
-        float inputHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT);
-        float inputRatio = inputWidth / inputHeight;
-        float outputWidth = flip ? outputFormat.getInteger(MediaFormat.KEY_HEIGHT) : outputFormat.getInteger(MediaFormat.KEY_WIDTH);
-        float outputHeight = flip ? outputFormat.getInteger(MediaFormat.KEY_WIDTH) : outputFormat.getInteger(MediaFormat.KEY_HEIGHT);
-        float outputRatio = outputWidth / outputHeight;
-        float scaleX = 1, scaleY = 1;
-        if (inputRatio > outputRatio) { // Input wider. We have a scaleX.
-            scaleX = inputRatio / outputRatio;
-        } else if (inputRatio < outputRatio) { // Input taller. We have a scaleY.
-            scaleY = outputRatio / inputRatio;
-        }
-        mDecoderOutputSurface.setScale(scaleX, scaleY);
+        mDecoderOutputSurface.configureWith(inputFormat, outputFormat, mSourceRotation, mExtraRotation);
     }
 
     @Override

diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/base/VideoDecoderOutputBase.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/base/VideoDecoderOutputBase.java
@@ -0,0 +1,37 @@
+package com.otaliastudios.transcoder.transcode.base;
+
+import android.media.MediaFormat;
+import android.view.Surface;
+
+import androidx.annotation.NonNull;
+
+public interface VideoDecoderOutputBase {
+
+
+    /**
+     * Configures rotation and scale for given formats.
+     * @param inputFormat input format
+     * @param outputFormat output format
+     * @param sourceRotation source rotation
+     * @param extraRotation extra rotation
+     */
+    void configureWith(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat, int sourceRotation, int extraRotation);
+
+
+    /**
+     * Returns a Surface to draw onto.
+     * @return the output surface
+     */
+    Surface getSurface();
+
+    /**
+     * Waits for a new frame drawn into our surface (see {@link #getSurface()}),
+     * then draws it using OpenGL.
+     */
+    void drawFrame();
+
+    /**
+     * Discard all resources held by this class, notably the EGL context.
+     */
+    void release();
+}
diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/base/VideoEncoderInputBase.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/base/VideoEncoderInputBase.java
@@ -0,0 +1,6 @@
+package com.otaliastudios.transcoder.transcode.base;
+
+public interface VideoEncoderInputBase {
+    void onFrame(long presentationTimeUs);
+    void release();
+}
diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoDecoderOutput.java
@@ -2,6 +2,7 @@
 
 
 import android.graphics.SurfaceTexture;
+import android.media.MediaFormat;
 import android.opengl.Matrix;
 import android.view.Surface;
 
@@ -12,6 +13,7 @@
 import com.otaliastudios.opengl.program.GlTextureProgram;
 import com.otaliastudios.opengl.texture.GlTexture;
 import com.otaliastudios.transcoder.internal.Logger;
+import com.otaliastudios.transcoder.transcode.base.VideoDecoderOutputBase;
 
 /**
  * The purpose of this class is to create a {@link Surface} associated to a certain GL texture.
@@ -26,7 +28,7 @@
  * NOTE: By default, the Surface will be using a BufferQueue in asynchronous mode, so we
  * can potentially drop frames.
  */
-public class VideoDecoderOutput {
+public class VideoDecoderOutput implements VideoDecoderOutputBase {
     private static final String TAG = VideoDecoderOutput.class.getSimpleName();
     private static final Logger LOG = new Logger(TAG);
 
@@ -77,29 +79,39 @@ public void onFrameAvailable(SurfaceTexture surfaceTexture) {
         mSurface = new Surface(mSurfaceTexture);
     }
 
-    /**
-     * Sets the frame scale along the two axes.
-     * @param scaleX x scale
-     * @param scaleY y scale
-     */
-    public void setScale(float scaleX, float scaleY) {
+
+    @Override
+    public void configureWith(@NonNull MediaFormat inputFormat, @NonNull MediaFormat outputFormat, int sourceRotation, int extraRotation) {
+        // The rotation we should apply is the intrinsic source rotation, plus any extra
+        // rotation that was set into the TranscoderOptions.
+
+        // Cropping support.
+        // Ignoring any outputFormat KEY_ROTATION (which is applied at playback time), the rotation
+        // difference between input and output is mSourceRotation + mExtraRotation.
+        int rotation = (sourceRotation + extraRotation) % 360;
+        boolean flip = (rotation % 180) != 0;
+        float inputWidth = inputFormat.getInteger(MediaFormat.KEY_WIDTH);
+        float inputHeight = inputFormat.getInteger(MediaFormat.KEY_HEIGHT);
+        float inputRatio = inputWidth / inputHeight;
+        float outputWidth = flip ? outputFormat.getInteger(MediaFormat.KEY_HEIGHT) : outputFormat.getInteger(MediaFormat.KEY_WIDTH);
+        float outputHeight = flip ? outputFormat.getInteger(MediaFormat.KEY_WIDTH) : outputFormat.getInteger(MediaFormat.KEY_HEIGHT);
+        float outputRatio = outputWidth / outputHeight;
+        float scaleX = 1, scaleY = 1;
+        if (inputRatio > outputRatio) { // Input wider. We have a scaleX.
+            scaleX = inputRatio / outputRatio;
+        } else if (inputRatio < outputRatio) { // Input taller. We have a scaleY.
+            scaleY = outputRatio / inputRatio;
+        }
         mScaleX = scaleX;
         mScaleY = scaleY;
-    }
-
-    /**
-     * Sets the desired frame rotation with respect
-     * to its natural orientation.
-     * @param rotation rotation
-     */
-    public void setRotation(int rotation) {
         mRotation = rotation;
     }
 
     /**
      * Returns a Surface to draw onto.
      * @return the output surface
      */
+    @Override
     @NonNull
     public Surface getSurface() {
         return mSurface;
@@ -108,6 +120,7 @@ public Surface getSurface() {
     /**
      * Discard all resources held by this class, notably the EGL context.
      */
+    @Override
     public void release() {
         mProgram.release();
         mSurface.release();
@@ -124,6 +137,7 @@ public void release() {
      * Waits for a new frame drawn into our surface (see {@link #getSurface()}),
      * then draws it using OpenGL.
      */
+    @Override
     public void drawFrame() {
         awaitNewFrame();
         drawNewFrame();

diff --git a/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoEncoderInput.java b/lib/src/main/java/com/otaliastudios/transcoder/transcode/internal/VideoEncoderInput.java
@@ -9,6 +9,7 @@
 
 import com.otaliastudios.opengl.core.EglCore;
 import com.otaliastudios.opengl.surface.EglWindowSurface;
+import com.otaliastudios.transcoder.transcode.base.VideoEncoderInputBase;
 
 /**
  * The purpose of this class is basically doing OpenGL initialization.
@@ -19,7 +20,7 @@
  * Calls to {@link #onFrame(long)} cause a frame of data to be sent to the surface, thus
  * to the {@link android.media.MediaCodec} input.
  */
-public class VideoEncoderInput {
+public class VideoEncoderInput implements VideoEncoderInputBase {
     @SuppressWarnings("unused")
     private static final String TAG = VideoEncoderInput.class.getSimpleName();
 
@@ -37,11 +38,13 @@ public VideoEncoderInput(@NonNull Surface surface) {
         mEglSurface.makeCurrent();
     }
 
+    @Override
     public void onFrame(long presentationTimeUs) {
         mEglSurface.setPresentationTime(presentationTimeUs * 1000L);
         mEglSurface.swapBuffers();
     }
 
+    @Override
     public void release() {
         // NOTE: Original code calls android.view.Surface.release()
         // after the egl core releasing. This should not be an issue.