Skip to content

Commit

Permalink
Utilize AudioProcessingPipeline in Transformer.
Browse files Browse the repository at this point in the history 
Provides an API for applications to set AudioProcessors for use
in Transformer.

PiperOrigin-RevId: 488621242
  • Loading branch information
@Samrobbo @microkatz
Samrobbo authored and microkatz committed Nov 16, 2022
1 parent 20151b9 commit 01e0db6
Show file tree
Hide file tree
Showing 4 changed files with 109 additions and 80 deletions.
134 changes: 63 additions & 71 deletions ...c/main/java/com/google/android/exoplayer2/transformer/AudioTranscodingSamplePipeline.java
View file
Expand Up @@ -16,40 +16,38 @@

package com.google.android.exoplayer2.transformer;

import static com.google.android.exoplayer2.decoder.DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import static java.lang.Math.min;

import androidx.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.audio.AudioProcessingPipeline;
import com.google.android.exoplayer2.audio.AudioProcessor;
import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.Util;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
import org.checkerframework.dataflow.qual.Pure;

/**
* Pipeline to decode audio samples, apply transformations on the raw samples, and re-encode them.
* Pipeline to decode audio samples, apply audio processing to the raw samples, and re-encode them.
*/
/* package */ final class AudioTranscodingSamplePipeline extends BaseSamplePipeline {

private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024;

private final Codec decoder;
private final DecoderInputBuffer decoderInputBuffer;

@Nullable private final SpeedChangingAudioProcessor speedChangingAudioProcessor;

private final AudioProcessingPipeline audioProcessingPipeline;
private final Codec encoder;
private final AudioFormat encoderInputAudioFormat;
private final DecoderInputBuffer encoderInputBuffer;
private final DecoderInputBuffer encoderOutputBuffer;

private ByteBuffer processorOutputBuffer;

private long nextEncoderInputBufferTimeUs;
private long encoderBufferDurationRemainder;

Expand All @@ -58,6 +56,7 @@ public AudioTranscodingSamplePipeline(
long streamStartPositionUs,
long streamOffsetUs,
TransformationRequest transformationRequest,
ImmutableList<AudioProcessor> audioProcessors,
Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory,
MuxerWrapper muxerWrapper,
Expand All @@ -70,37 +69,38 @@ public AudioTranscodingSamplePipeline(
transformationRequest.flattenForSlowMotion,
muxerWrapper);

decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
encoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
encoderOutputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
encoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
encoderOutputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);

decoder = decoderFactory.createForAudioDecoding(inputFormat);

AudioFormat encoderInputAudioFormat =
if (transformationRequest.flattenForSlowMotion) {
audioProcessors =
new ImmutableList.Builder<AudioProcessor>()
.add(new SpeedChangingAudioProcessor(new SegmentSpeedProvider(inputFormat)))
.addAll(audioProcessors)
.build();
}

audioProcessingPipeline = new AudioProcessingPipeline(audioProcessors);
AudioFormat pipelineInputAudioFormat =
new AudioFormat(
inputFormat.sampleRate,
inputFormat.channelCount,
// The decoder uses ENCODING_PCM_16BIT by default.
// https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers
C.ENCODING_PCM_16BIT);
if (transformationRequest.flattenForSlowMotion) {
speedChangingAudioProcessor =
new SpeedChangingAudioProcessor(new SegmentSpeedProvider(inputFormat));
try {
encoderInputAudioFormat = speedChangingAudioProcessor.configure(encoderInputAudioFormat);
} catch (AudioProcessor.UnhandledAudioFormatException impossible) {
throw new IllegalStateException(impossible);
}
speedChangingAudioProcessor.flush();
} else {
speedChangingAudioProcessor = null;

try {
encoderInputAudioFormat = audioProcessingPipeline.configure(pipelineInputAudioFormat);
} catch (AudioProcessor.UnhandledAudioFormatException unhandledAudioFormatException) {
throw TransformationException.createForAudioProcessing(
unhandledAudioFormatException, pipelineInputAudioFormat);
}
processorOutputBuffer = AudioProcessor.EMPTY_BUFFER;

this.encoderInputAudioFormat = encoderInputAudioFormat;
audioProcessingPipeline.flush();

Format requestedOutputFormat =
new Format.Builder()
.setSampleMimeType(
Expand All @@ -125,9 +125,7 @@ public AudioTranscodingSamplePipeline(

@Override
public void release() {
if (speedChangingAudioProcessor != null) {
speedChangingAudioProcessor.reset();
}
audioProcessingPipeline.reset();
decoder.release();
encoder.release();
}
Expand All @@ -145,8 +143,8 @@ protected void queueInputBufferInternal() throws TransformationException {

@Override
protected boolean processDataUpToMuxer() throws TransformationException {
if (speedChangingAudioProcessor != null) {
return feedEncoderFromProcessor() || feedProcessorFromDecoder();
if (audioProcessingPipeline.isOperational()) {
return feedEncoderFromProcessingPipeline() || feedProcessingPipelineFromDecoder();
} else {
return feedEncoderFromDecoder();
}
Expand Down Expand Up @@ -207,57 +205,51 @@ private boolean feedEncoderFromDecoder() throws TransformationException {
}

/**
* Attempts to pass audio processor output data to the encoder, and returns whether it may be
* possible to pass more data immediately by calling this method again.
* Attempts to feed audio processor output data to the encoder.
*
* @return Whether more data can be fed immediately, by calling this method again.
*/
@RequiresNonNull("speedChangingAudioProcessor")
private boolean feedEncoderFromProcessor() throws TransformationException {
private boolean feedEncoderFromProcessingPipeline() throws TransformationException {
if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) {
return false;
}

if (!processorOutputBuffer.hasRemaining()) {
processorOutputBuffer = speedChangingAudioProcessor.getOutput();
if (!processorOutputBuffer.hasRemaining()) {
if (decoder.isEnded() && speedChangingAudioProcessor.isEnded()) {
queueEndOfStreamToEncoder();
}
return false;
ByteBuffer processingPipelineOutputBuffer = audioProcessingPipeline.getOutput();

if (!processingPipelineOutputBuffer.hasRemaining()) {
if (audioProcessingPipeline.isEnded()) {
queueEndOfStreamToEncoder();
}
return false;
}

feedEncoder(processorOutputBuffer);
feedEncoder(processingPipelineOutputBuffer);
return true;
}

/**
* Attempts to process decoder output data, and returns whether it may be possible to process more
* data immediately by calling this method again.
* Attempts to feed decoder output data to the {@link AudioProcessingPipeline}.
*
* @return Whether it may be possible to feed more data immediately by calling this method again.
*/
@RequiresNonNull("speedChangingAudioProcessor")
private boolean feedProcessorFromDecoder() throws TransformationException {
// Audio processors invalidate any previous output buffer when more input is queued, so we don't
// queue if there is output still to be processed.
if (processorOutputBuffer.hasRemaining()
|| speedChangingAudioProcessor.getOutput().hasRemaining()) {
return false;
}

private boolean feedProcessingPipelineFromDecoder() throws TransformationException {
if (decoder.isEnded()) {
speedChangingAudioProcessor.queueEndOfStream();
audioProcessingPipeline.queueEndOfStream();
return false;
}
checkState(!speedChangingAudioProcessor.isEnded());
checkState(!audioProcessingPipeline.isEnded());

@Nullable ByteBuffer decoderOutputBuffer = decoder.getOutputBuffer();
if (decoderOutputBuffer == null) {
return false;
}

speedChangingAudioProcessor.queueInput(decoderOutputBuffer);
if (!decoderOutputBuffer.hasRemaining()) {
decoder.releaseOutputBuffer(/* render= */ false);
audioProcessingPipeline.queueInput(decoderOutputBuffer);
if (decoderOutputBuffer.hasRemaining()) {
return false;
}
// Decoder output buffer was fully consumed by the processing pipeline.
decoder.releaseOutputBuffer(/* render= */ false);
return true;
}

Expand Down Expand Up @@ -290,6 +282,17 @@ private void queueEndOfStreamToEncoder() throws TransformationException {
encoder.queueInputBuffer(encoderInputBuffer);
}

@Pure
private static TransformationRequest createFallbackTransformationRequest(
TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) {
// TODO(b/210591626): Also update bitrate and other params once encoder configuration and
// fallback are implemented.
if (Util.areEqual(requestedFormat.sampleMimeType, actualFormat.sampleMimeType)) {
return transformationRequest;
}
return transformationRequest.buildUpon().setAudioMimeType(actualFormat.sampleMimeType).build();
}

private void computeNextEncoderInputBufferTimeUs(
long bytesWritten, int bytesPerFrame, int sampleRate) {
// The calculation below accounts for remainders and rounding. Without that it corresponds to
Expand All @@ -307,15 +310,4 @@ private void computeNextEncoderInputBufferTimeUs(
}
nextEncoderInputBufferTimeUs += bufferDurationUs;
}

@Pure
private static TransformationRequest createFallbackTransformationRequest(
TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) {
// TODO(b/210591626): Also update bitrate and other params once encoder configuration and
// fallback are implemented.
if (Util.areEqual(requestedFormat.sampleMimeType, actualFormat.sampleMimeType)) {
return transformationRequest;
}
return transformationRequest.buildUpon().setAudioMimeType(actualFormat.sampleMimeType).build();
}
}
25 changes: 16 additions & 9 deletions ...rmer/src/main/java/com/google/android/exoplayer2/transformer/TransformationException.java
View file
Expand Up @@ -24,7 +24,6 @@
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.PlaybackException;
import com.google.android.exoplayer2.audio.AudioProcessor;
import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
import com.google.android.exoplayer2.util.Clock;
import com.google.android.exoplayer2.util.FrameProcessingException;
Expand Down Expand Up @@ -71,6 +70,7 @@ public final class TransformationException extends Exception {
ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED,
ERROR_CODE_HDR_ENCODING_UNSUPPORTED,
ERROR_CODE_FRAME_PROCESSING_FAILED,
ERROR_CODE_AUDIO_PROCESSING_FAILED,
ERROR_CODE_MUXING_FAILED,
})
public @interface ErrorCode {}
Expand Down Expand Up @@ -161,9 +161,15 @@ public final class TransformationException extends Exception {
/** Caused by a frame processing failure. */
public static final int ERROR_CODE_FRAME_PROCESSING_FAILED = 5001;

// Muxing errors (6xxx).
// Audio processing errors (6xxx).

/** Caused by an audio processing failure. */
public static final int ERROR_CODE_AUDIO_PROCESSING_FAILED = 6001;

// Muxing errors (7xxx).

/** Caused by a failure while muxing media samples. */
public static final int ERROR_CODE_MUXING_FAILED = 6001;
public static final int ERROR_CODE_MUXING_FAILED = 7001;

private static final ImmutableBiMap<String, @ErrorCode Integer> NAME_TO_ERROR_CODE =
new ImmutableBiMap.Builder<String, @ErrorCode Integer>()
Expand All @@ -186,6 +192,7 @@ public final class TransformationException extends Exception {
.put("ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED", ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED)
.put("ERROR_CODE_HDR_ENCODING_UNSUPPORTED", ERROR_CODE_HDR_ENCODING_UNSUPPORTED)
.put("ERROR_CODE_FRAME_PROCESSING_FAILED", ERROR_CODE_FRAME_PROCESSING_FAILED)
.put("ERROR_CODE_AUDIO_PROCESSING_FAILED", ERROR_CODE_AUDIO_PROCESSING_FAILED)
.put("ERROR_CODE_MUXING_FAILED", ERROR_CODE_MUXING_FAILED)
.buildOrThrow();

Expand Down Expand Up @@ -262,18 +269,18 @@ public static TransformationException createForCodec(
}

/**
* Creates an instance for an {@link AudioProcessor} related exception.
* Creates an instance for an audio processing related exception.
*
* @param cause The cause of the failure.
* @param componentName The name of the {@link AudioProcessor} used.
* @param audioFormat The {@link AudioFormat} used.
* @param errorCode See {@link #errorCode}.
* @return The created instance.
*/
public static TransformationException createForAudioProcessor(
Throwable cause, String componentName, AudioFormat audioFormat, int errorCode) {
public static TransformationException createForAudioProcessing(
Throwable cause, AudioFormat audioFormat) {
return new TransformationException(
componentName + " error, audio_format = " + audioFormat, cause, errorCode);
"Audio processing error, audio_format = " + audioFormat,
cause,
ERROR_CODE_AUDIO_PROCESSING_FAILED);
}

/**
Expand Down

0 comments on commit 01e0db6

Please sign in to comment.