Merge MatrixTransformationProcessor and ExternalTextureProcessor.

This saves an intermediate texture copy step for use-cases where matrix transformations are the first or only effects in the chain. PiperOrigin-RevId: 460239403
androidx · Jul 12, 2022 · 7dc54ef · 7dc54ef
1 parent 18f4068
commit 7dc54ef
Show file tree

Hide file tree

Showing 8 changed files with 212 additions and 255 deletions.
diff --git a/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl b/libraries/transformer/src/main/assets/shaders/vertex_shader_tex_transform_es2.glsl
diff --git a/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl b/libraries/transformer/src/main/assets/shaders/vertex_shader_transformation_es2.glsl
@@ -13,13 +13,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// ES 2 vertex shader that applies the 4 * 4 transformation matrix
-// uTransformationMatrix.
+// ES 2 vertex shader that applies the 4 * 4 transformation matrices
+// uTransformationMatrix and the uTexTransformationMatrix.
 
 attribute vec4 aFramePosition;
 uniform mat4 uTransformationMatrix;
+uniform mat4 uTexTransformationMatrix;
 varying vec2 vTexSamplingCoord;
 void main() {
   gl_Position = uTransformationMatrix * aFramePosition;
-  vTexSamplingCoord = vec2(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5);
+  vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
+  vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy;
 }
diff --git a/...ders/vertex_shader_tex_transform_es3.glsl → ...ers/vertex_shader_transformation_es3.glsl b/...ders/vertex_shader_tex_transform_es3.glsl → ...ers/vertex_shader_transformation_es3.glsl
@@ -1,5 +1,5 @@
 #version 300 es
-// Copyright 2022 The Android Open Source Project
+// Copyright 2021 The Android Open Source Project
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,15 +13,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// ES 3 vertex shader that applies an external surface texture's 4 * 4 texture
-// transformation matrix to convert the texture coordinates to the sampling
-// locations.
+// ES 3 vertex shader that applies the 4 * 4 transformation matrices
+// uTransformationMatrix and the uTexTransformationMatrix.
 
 in vec4 aFramePosition;
-uniform mat4 uTexTransform;
+uniform mat4 uTransformationMatrix;
+uniform mat4 uTexTransformationMatrix;
 out vec2 vTexSamplingCoord;
 void main() {
-  gl_Position = aFramePosition;
+  gl_Position = uTransformationMatrix * aFramePosition;
   vec4 texturePosition = vec4(aFramePosition.x * 0.5 + 0.5, aFramePosition.y * 0.5 + 0.5, 0.0, 1.0);
-  vTexSamplingCoord = (uTexTransform * texturePosition).xy;
+  vTexSamplingCoord = (uTexTransformationMatrix * texturePosition).xy;
 }
diff --git a/...aries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java b/...aries/transformer/src/main/java/androidx/media3/transformer/ExternalTextureProcessor.java
@@ -15,71 +15,13 @@
  */
 package androidx.media3.transformer;
 
-import static androidx.media3.common.util.Assertions.checkArgument;
-import static androidx.media3.common.util.Assertions.checkStateNotNull;
-
-import android.content.Context;
-import android.opengl.GLES20;
-import android.util.Size;
-import androidx.media3.common.util.GlProgram;
-import androidx.media3.common.util.GlUtil;
-import java.io.IOException;
-
-/** Copies frames from an external texture and applies color transformations for HDR if needed. */
-/* package */ class ExternalTextureProcessor extends SingleFrameGlTextureProcessor {
-
-  private static final String VERTEX_SHADER_TEX_TRANSFORM_PATH =
-      "shaders/vertex_shader_tex_transform_es2.glsl";
-  private static final String VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH =
-      "shaders/vertex_shader_tex_transform_es3.glsl";
-  private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
-      "shaders/fragment_shader_copy_external_es2.glsl";
-  private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
-      "shaders/fragment_shader_copy_external_yuv_es3.glsl";
-  // Color transform coefficients from
-  // https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/libstagefright/colorconversion/ColorConverter.cpp;l=668-670;drc=487adf977a50cac3929eba15fad0d0f461c7ff0f.
-  private static final float[] MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM = {
-    1.168f, 1.168f, 1.168f,
-    0.0f, -0.188f, 2.148f,
-    1.683f, -0.652f, 0.0f,
-  };
-
-  private final GlProgram glProgram;
-
-  /**
-   * Creates a new instance.
-   *
-   * @param useHdr Whether to process the input as an HDR signal.
-   * @throws FrameProcessingException If a problem occurs while reading shader files.
-   */
-  public ExternalTextureProcessor(Context context, boolean useHdr) throws FrameProcessingException {
-    String vertexShaderFilePath =
-        useHdr ? VERTEX_SHADER_TEX_TRANSFORM_ES3_PATH : VERTEX_SHADER_TEX_TRANSFORM_PATH;
-    String fragmentShaderFilePath =
-        useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
-    try {
-      glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
-    } catch (IOException | GlUtil.GlException e) {
-      throw new FrameProcessingException(e);
-    }
-    // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y.
-    glProgram.setBufferAttribute(
-        "aFramePosition",
-        GlUtil.getNormalizedCoordinateBounds(),
-        GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
-    if (useHdr) {
-      // In HDR editing mode the decoder output is sampled in YUV.
-      glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
-    }
-  }
-
-  @Override
-  public Size configure(int inputWidth, int inputHeight) {
-    checkArgument(inputWidth > 0, "inputWidth must be positive");
-    checkArgument(inputHeight > 0, "inputHeight must be positive");
-
-    return new Size(inputWidth, inputHeight);
-  }
+/**
+ * Interface for a {@link GlTextureProcessor} that samples from an external texture.
+ *
+ * <p>Use {@link #setTextureTransformMatrix(float[])} to provide the texture's transformation
+ * matrix.
+ */
+/* package */ interface ExternalTextureProcessor extends GlTextureProcessor {
 
   /**
    * Sets the texture transform matrix for converting an external surface texture's coordinates to
@@ -88,35 +30,5 @@ public Size configure(int inputWidth, int inputHeight) {
    * @param textureTransformMatrix The external surface texture's {@linkplain
    *     android.graphics.SurfaceTexture#getTransformMatrix(float[]) transform matrix}.
    */
-  public void setTextureTransformMatrix(float[] textureTransformMatrix) {
-    checkStateNotNull(glProgram);
-    glProgram.setFloatsUniform("uTexTransform", textureTransformMatrix);
-  }
-
-  @Override
-  public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException {
-    checkStateNotNull(glProgram);
-    try {
-      glProgram.use();
-      glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
-      glProgram.bindAttributesAndUniforms();
-      // The four-vertex triangle strip forms a quad.
-      GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
-      GlUtil.checkGlError();
-    } catch (GlUtil.GlException e) {
-      throw new FrameProcessingException(e, presentationTimeUs);
-    }
-  }
-
-  @Override
-  public void release() throws FrameProcessingException {
-    super.release();
-    if (glProgram != null) {
-      try {
-        glProgram.delete();
-      } catch (GlUtil.GlException e) {
-        throw new FrameProcessingException(e);
-      }
-    }
-  }
+  void setTextureTransformMatrix(float[] textureTransformMatrix);
 }
diff --git a/.../src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java b/.../src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java
@@ -24,6 +24,7 @@
 import android.opengl.EGLExt;
 import android.opengl.EGLSurface;
 import android.opengl.GLES20;
+import android.opengl.Matrix;
 import android.util.Size;
 import android.view.Surface;
 import android.view.SurfaceHolder;
@@ -50,7 +51,8 @@
  * <p>This wrapper is used for the final {@link GlTextureProcessor} instance in the chain of {@link
  * GlTextureProcessor} instances used by {@link FrameProcessor}.
  */
-/* package */ final class FinalMatrixTransformationProcessorWrapper implements GlTextureProcessor {
+/* package */ final class FinalMatrixTransformationProcessorWrapper
+    implements GlTextureProcessor, ExternalTextureProcessor {
 
   private static final String TAG = "FinalProcessorWrapper";
 
@@ -61,7 +63,9 @@
   private final long streamOffsetUs;
   private final DebugViewProvider debugViewProvider;
   private final FrameProcessor.Listener frameProcessorListener;
+  private final boolean sampleFromExternalTexture;
   private final boolean useHdr;
+  private final float[] textureTransformMatrix;
 
   private int inputWidth;
   private int inputHeight;
@@ -86,6 +90,7 @@ public FinalMatrixTransformationProcessorWrapper(
       long streamOffsetUs,
       FrameProcessor.Listener frameProcessorListener,
       DebugViewProvider debugViewProvider,
+      boolean sampleFromExternalTexture,
       boolean useHdr) {
     this.context = context;
     this.matrixTransformations = matrixTransformations;
@@ -94,7 +99,11 @@ public FinalMatrixTransformationProcessorWrapper(
     this.streamOffsetUs = streamOffsetUs;
     this.debugViewProvider = debugViewProvider;
     this.frameProcessorListener = frameProcessorListener;
+    this.sampleFromExternalTexture = sampleFromExternalTexture;
     this.useHdr = useHdr;
+
+    textureTransformMatrix = new float[16];
+    Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0);
   }
 
   /**
@@ -239,7 +248,9 @@ private MatrixTransformationProcessor createMatrixTransformationProcessorForOutp
             outputSurfaceInfo.width, outputSurfaceInfo.height, Presentation.LAYOUT_SCALE_TO_FIT));
 
     MatrixTransformationProcessor matrixTransformationProcessor =
-        new MatrixTransformationProcessor(context, matrixTransformationListBuilder.build());
+        new MatrixTransformationProcessor(
+            context, matrixTransformationListBuilder.build(), sampleFromExternalTexture, useHdr);
+    matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
     Size outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight);
     checkState(outputSize.getWidth() == outputSurfaceInfo.width);
     checkState(outputSize.getHeight() == outputSurfaceInfo.height);
@@ -265,6 +276,20 @@ public void release() throws FrameProcessingException {
     }
   }
 
+  @Override
+  public void setTextureTransformMatrix(float[] textureTransformMatrix) {
+    System.arraycopy(
+        /* src= */ textureTransformMatrix,
+        /* srcPos= */ 0,
+        /* dest= */ this.textureTransformMatrix,
+        /* destPost= */ 0,
+        /* length= */ textureTransformMatrix.length);
+
+    if (matrixTransformationProcessor != null) {
+      matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
+    }
+  }
+
   public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
     if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) {
       this.outputSurfaceInfo = outputSurfaceInfo;