Skip to content

Commit

Permalink
Check targetSdkVersion for frame dropping workaround.
Browse files Browse the repository at this point in the history 
Based on
https://developer.android.com/reference/android/media/MediaCodec#using-an-output-surface,
frame dropping behaviour depends on the target SDK version.
After this change transformer will only use
MediaFormat#KEY_ALLOW_FRAME_DROP if both the target and system SDK
version are at least 29 and default to its pre 29 behaviour where each
decoder output frame must be processed before a new one is rendered
to prevent frame dropping otherwise.

Also remove deprecated Transformer.Builder constructor without a
context and the context setter.

PiperOrigin-RevId: 453971097
  • Loading branch information
@hmsch @marcbaechinger
hmsch authored and marcbaechinger committed Jun 9, 2022
1 parent cc1f32d commit a105d03
Show file tree
Hide file tree
Showing 17 changed files with 104 additions and 80 deletions.
2 changes: 2 additions & 0 deletions RELEASENOTES.md
View file
Expand Up @@ -191,6 +191,8 @@
`DEFAULT_TRACK_SELECTOR_PARAMETERS_WITHOUT_CONTEXT` otherwise.
* Remove constructor `DefaultTrackSelector(ExoTrackSelection.Factory)`.
Use `DefaultTrackSelector(Context, ExoTrackSelection.Factory)` instead.
* Remove `Transformer.Builder.setContext`. The `Context` should be passed
to the `Transformer.Builder` constructor instead.

### 1.0.0-alpha03 (2022-03-14)

Expand Down
1 change: 1 addition & 0 deletions demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View file
Expand Up @@ -260,6 +260,7 @@ private Transformer createTransformer(@Nullable Bundle bundle, String filePath)
.setRemoveVideo(bundle.getBoolean(ConfigurationActivity.SHOULD_REMOVE_VIDEO))
.setEncoderFactory(
new DefaultEncoderFactory(
/* context= */ this,
EncoderSelector.DEFAULT,
/* enableFallback= */ bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK)));

Expand Down
4 changes: 2 additions & 2 deletions libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View file
Expand Up @@ -224,8 +224,8 @@ public static final class ForceEncodeEncoderFactory implements Codec.EncoderFact
private final Codec.EncoderFactory encoderFactory;

/** Creates an instance that wraps {@link DefaultEncoderFactory}. */
public ForceEncodeEncoderFactory() {
encoderFactory = Codec.EncoderFactory.DEFAULT;
public ForceEncodeEncoderFactory(Context context) {
encoderFactory = new DefaultEncoderFactory(context);
}

/**
Expand Down
6 changes: 4 additions & 2 deletions ...transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View file
Expand Up @@ -42,7 +42,8 @@ public void videoEditing_completesWithConsistentFrameCount() throws Exception {
.setTransformationRequest(
new TransformationRequest.Builder().setResolution(480).build())
.setEncoderFactory(
new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false))
new DefaultEncoderFactory(
context, EncoderSelector.DEFAULT, /* enableFallback= */ false))
.build();
// Result of the following command:
// ffprobe -count_frames -select_streams v:0 -show_entries stream=nb_read_frames sample.mp4
Expand All @@ -67,7 +68,8 @@ public void videoOnly_completesWithConsistentDuration() throws Exception {
.setTransformationRequest(
new TransformationRequest.Builder().setResolution(480).build())
.setEncoderFactory(
new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false))
new DefaultEncoderFactory(
context, EncoderSelector.DEFAULT, /* enableFallback= */ false))
.build();
long expectedDurationMs = 967;

Expand Down
6 changes: 3 additions & 3 deletions .../androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java
View file
Expand Up @@ -48,7 +48,7 @@ public void repeatedTranscode_givesConsistentLengthOutput() throws Exception {
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder().setRotationDegrees(45).build())
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory())
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(context))
.build())
.build();

Expand Down Expand Up @@ -78,7 +78,7 @@ public void repeatedTranscodeNoAudio_givesConsistentLengthOutput() throws Except
.setRemoveAudio(true)
.setTransformationRequest(
new TransformationRequest.Builder().setRotationDegrees(45).build())
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory())
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(context))
.build())
.build();

Expand Down Expand Up @@ -107,7 +107,7 @@ public void repeatedTranscodeNoVideo_givesConsistentLengthOutput() throws Except
new Transformer.Builder(context)
.setRemoveVideo(true)
.setTransformationRequest(new TransformationRequest.Builder().build())
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory())
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(context))
.build())
.build();

Expand Down
4 changes: 2 additions & 2 deletions ...transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java
View file
Expand Up @@ -52,7 +52,7 @@ public void transformWithDecodeEncode_ssimIsGreaterThan90Percent() throws Except
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder().setVideoMimeType(MimeTypes.VIDEO_H264).build())
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory())
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(context))
.setRemoveAudio(true)
.build();

Expand Down Expand Up @@ -119,7 +119,7 @@ public void transcodeAvcToAvc320x240_ssimIsGreaterThan90Percent() throws Excepti
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder().setVideoMimeType(MimeTypes.VIDEO_H264).build())
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory())
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(context))
.setRemoveAudio(true)
.build();

Expand Down
17 changes: 12 additions & 5 deletions ...s/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java
View file
Expand Up @@ -52,7 +52,9 @@ public void transform() throws Exception {
String testId = TAG + "_transform";
Context context = ApplicationProvider.getApplicationContext();
Transformer transformer =
new Transformer.Builder(context).setEncoderFactory(new ForceEncodeEncoderFactory()).build();
new Transformer.Builder(context)
.setEncoderFactory(new ForceEncodeEncoderFactory(context))
.build();
new TransformerAndroidTestRunner.Builder(context, transformer)
.setMaybeCalculateSsim(true)
.build()
Expand Down Expand Up @@ -80,6 +82,7 @@ public void transformToSpecificBitrate() throws Exception {
.setEncoderFactory(
new ForceEncodeEncoderFactory(
/* wrappedEncoderFactory= */ new DefaultEncoderFactory(
context,
EncoderSelector.DEFAULT,
new VideoEncoderSettings.Builder().setBitrate(5_000_000).build(),
/* enableFallback= */ true)))
Expand All @@ -104,7 +107,9 @@ public void transform4K60() throws Exception {
}

Transformer transformer =
new Transformer.Builder(context).setEncoderFactory(new ForceEncodeEncoderFactory()).build();
new Transformer.Builder(context)
.setEncoderFactory(new ForceEncodeEncoderFactory(context))
.build();
new TransformerAndroidTestRunner.Builder(context, transformer)
.setMaybeCalculateSsim(true)
.setTimeoutSeconds(180)
Expand All @@ -125,7 +130,9 @@ public void transform8K24() throws Exception {
return;
}
Transformer transformer =
new Transformer.Builder(context).setEncoderFactory(new ForceEncodeEncoderFactory()).build();
new Transformer.Builder(context)
.setEncoderFactory(new ForceEncodeEncoderFactory(context))
.build();
new TransformerAndroidTestRunner.Builder(context, transformer)
.setMaybeCalculateSsim(true)
.setTimeoutSeconds(180)
Expand All @@ -139,7 +146,7 @@ public void transformNoAudio() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
Transformer transformer =
new Transformer.Builder(context)
.setEncoderFactory(new ForceEncodeEncoderFactory())
.setEncoderFactory(new ForceEncodeEncoderFactory(context))
.setRemoveAudio(true)
.build();
new TransformerAndroidTestRunner.Builder(context, transformer)
Expand All @@ -154,7 +161,7 @@ public void transformNoVideo() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
Transformer transformer =
new Transformer.Builder(context)
.setEncoderFactory(new ForceEncodeEncoderFactory())
.setEncoderFactory(new ForceEncodeEncoderFactory(context))
.setRemoveVideo(true)
.build();
new TransformerAndroidTestRunner.Builder(context, transformer)
Expand Down
1 change: 1 addition & 0 deletions ...mer/src/androidTest/java/androidx/media3/transformer/mh/analysis/BitrateAnalysisTest.java
View file
Expand Up @@ -118,6 +118,7 @@ public void analyzeBitrate() throws Exception {
.setEncoderFactory(
new AndroidTestUtil.ForceEncodeEncoderFactory(
/* wrappedEncoderFactory= */ new DefaultEncoderFactory(
context,
EncoderSelector.DEFAULT,
new VideoEncoderSettings.Builder()
.setBitrate(bitrate)
Expand Down
1 change: 1 addition & 0 deletions ...roidTest/java/androidx/media3/transformer/mh/analysis/EncoderPerformanceAnalysisTest.java
View file
Expand Up @@ -128,6 +128,7 @@ public void analyzeEncoderPerformance() throws Exception {
.setEncoderFactory(
new AndroidTestUtil.ForceEncodeEncoderFactory(
/* wrappedEncoderFactory= */ new DefaultEncoderFactory(
context,
EncoderSelector.DEFAULT,
new VideoEncoderSettings.Builder()
.setEncoderPerformanceParameters(operatingRate, priority)
Expand Down
6 changes: 0 additions & 6 deletions libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java
View file
Expand Up @@ -41,9 +41,6 @@ public interface Codec {
/** A factory for {@linkplain Codec decoder} instances. */
interface DecoderFactory {

/** A default {@code DecoderFactory} implementation. */
DecoderFactory DEFAULT = new DefaultDecoderFactory();

/**
* Returns a {@link Codec} for audio decoding.
*
Expand Down Expand Up @@ -72,9 +69,6 @@ Codec createForVideoDecoding(
/** A factory for {@linkplain Codec encoder} instances. */
interface EncoderFactory {

/** A default {@code EncoderFactory} implementation. */
EncoderFactory DEFAULT = new DefaultEncoderFactory();

/**
* Returns a {@link Codec} for audio encoding.
*
Expand Down
26 changes: 19 additions & 7 deletions libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java
View file
Expand Up @@ -21,6 +21,7 @@
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.common.util.Util.SDK_INT;

import android.content.Context;
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
import android.media.MediaFormat;
Expand Down Expand Up @@ -54,6 +55,8 @@ public final class DefaultCodec implements Codec {
private final MediaCodec mediaCodec;
@Nullable private final Surface inputSurface;

private final boolean decoderNeedsFrameDroppingWorkaround;

private @MonotonicNonNull Format outputFormat;
@Nullable private ByteBuffer outputBuffer;

Expand All @@ -65,6 +68,7 @@ public final class DefaultCodec implements Codec {
/**
* Creates a {@code DefaultCodec}.
*
* @param context The {@link Context}.
* @param configurationFormat The {@link Format} to configure the {@code DefaultCodec}. See {@link
* #getConfigurationFormat()}. The {@link Format#sampleMimeType sampleMimeType} must not be
* {@code null}.
Expand All @@ -75,6 +79,7 @@ public final class DefaultCodec implements Codec {
* @param outputSurface The output {@link Surface} if the {@link MediaCodec} outputs to a surface.
*/
public DefaultCodec(
Context context,
Format configurationFormat,
MediaFormat configurationMediaFormat,
String mediaCodecName,
Expand Down Expand Up @@ -110,6 +115,7 @@ public DefaultCodec(
}
this.mediaCodec = mediaCodec;
this.inputSurface = inputSurface;
decoderNeedsFrameDroppingWorkaround = decoderNeedsFrameDroppingWorkaround(context);
}

@Override
Expand All @@ -124,15 +130,12 @@ public Surface getInputSurface() {

@Override
public int getMaxPendingFrameCount() {
if (SDK_INT < 29) {
// Prior to API 29, decoders may drop frames to keep their output surface from growing out of
// bounds. From API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame
// dropping even when the surface is full. Frame dropping is never desired, so allow a maximum
// of one frame to be pending at a time.
if (decoderNeedsFrameDroppingWorkaround) {
// Allow a maximum of one frame to be pending at a time to prevent frame dropping.
// TODO(b/226330223): Investigate increasing this limit.
return 1;
}
if (Ascii.toUpperCase(mediaCodec.getCodecInfo().getCanonicalName()).startsWith("OMX.")) {
if (Ascii.toUpperCase(getName()).startsWith("OMX.")) {
// Some OMX decoders don't correctly track their number of output buffers available, and get
// stuck if too many frames are rendered without being processed, so limit the number of
// pending frames to avoid getting stuck. This value is experimentally determined. See also
Expand Down Expand Up @@ -261,7 +264,7 @@ public void release() {
* {@inheritDoc}
*
* <p>This name is of the actual codec, which may not be the same as the {@code mediaCodecName}
* passed to {@link #DefaultCodec(Format, MediaFormat, String, boolean, Surface)}.
* passed to {@link #DefaultCodec(Context, Format, MediaFormat, String, boolean, Surface)}.
*
* @see MediaCodec#getCanonicalName()
*/
Expand Down Expand Up @@ -424,4 +427,13 @@ private static void startCodec(MediaCodec codec) {
codec.start();
TraceUtil.endSection();
}

private static boolean decoderNeedsFrameDroppingWorkaround(Context context) {
// Prior to API 29, decoders may drop frames to keep their output surface from growing out of
// bounds. From API 29, if the app targets API 29 or later, the {@link
// MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame dropping even when the surface is full.
// Frame dropping is never desired, so a workaround is needed for older API levels.
return SDK_INT < 29
|| context.getApplicationContext().getApplicationInfo().targetSdkVersion < 29;
}
}
30 changes: 25 additions & 5 deletions libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java
View file
Expand Up @@ -19,6 +19,8 @@
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Util.SDK_INT;

import android.annotation.SuppressLint;
import android.content.Context;
import android.media.MediaFormat;
import android.view.Surface;
import androidx.annotation.Nullable;
Expand All @@ -30,6 +32,18 @@
/** A default implementation of {@link Codec.DecoderFactory}. */
/* package */ final class DefaultDecoderFactory implements Codec.DecoderFactory {

private final Context context;

private final boolean decoderSupportsKeyAllowFrameDrop;

public DefaultDecoderFactory(Context context) {
this.context = context;

decoderSupportsKeyAllowFrameDrop =
SDK_INT >= 29
&& context.getApplicationContext().getApplicationInfo().targetSdkVersion >= 29;
}

@Override
public Codec createForAudioDecoding(Format format) throws TransformationException {
MediaFormat mediaFormat =
Expand All @@ -45,9 +59,15 @@ public Codec createForAudioDecoding(Format format) throws TransformationExceptio
throw createTransformationException(format);
}
return new DefaultCodec(
format, mediaFormat, mediaCodecName, /* isDecoder= */ true, /* outputSurface= */ null);
context,
format,
mediaFormat,
mediaCodecName,
/* isDecoder= */ true,
/* outputSurface= */ null);
}

@SuppressLint("InlinedApi")
@Override
public Codec createForVideoDecoding(
Format format, Surface outputSurface, boolean enableRequestSdrToneMapping)
Expand All @@ -59,9 +79,9 @@ public Codec createForVideoDecoding(
MediaFormatUtil.maybeSetInteger(
mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize);
MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData);
if (SDK_INT >= 29) {
// On API levels over 29, Transformer decodes as many frames as possible in one render
// cycle. This key ensures no frame dropping when the decoder's output surface is full.
if (decoderSupportsKeyAllowFrameDrop) {
// This key ensures no frame dropping when the decoder's output surface is full. This allows
// transformer to decode as many frames as possible in one render cycle.
mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0);
}
if (SDK_INT >= 31 && enableRequestSdrToneMapping) {
Expand All @@ -75,7 +95,7 @@ public Codec createForVideoDecoding(
throw createTransformationException(format);
}
return new DefaultCodec(
format, mediaFormat, mediaCodecName, /* isDecoder= */ true, outputSurface);
context, format, mediaFormat, mediaCodecName, /* isDecoder= */ true, outputSurface);
}

@RequiresNonNull("#1.sampleMimeType")
Expand Down

0 comments on commit a105d03

Please sign in to comment.