Skip to content

Commit

Permalink
Make minor fixes to HDR handling
Browse files Browse the repository at this point in the history 
- Update profile selection logic to pick an HDR-compatible profile when doing HDR editing on H.264/AVC videos.
- Handle doing the capabilities check for all MIME types that support HDR (not just H.265/HEVC).
- Fix a bug where we would pass an HDR input color format to the encoder when using tone-mapping.
- Tweak how `EncoderWrapper` works so decisions at made at construction time.

Manually tested cases:
- Transformation of an SDR video.
- Transformation of an HDR video to AVC (which triggers fallback/tone-mapping on a device that doesn't support HDR editing for AVC).
- Transformation of an HDR video with HDR editing.

PiperOrigin-RevId: 461572973
  • Loading branch information
@andrewlewis @rohitjoins
andrewlewis authored and rohitjoins committed Jul 21, 2022
1 parent 405be80 commit 0db07c6
Show file tree
Hide file tree
Showing 4 changed files with 136 additions and 73 deletions.
13 changes: 11 additions & 2 deletions ...former/src/main/java/com/google/android/exoplayer2/transformer/DefaultEncoderFactory.java
View file
Expand Up @@ -273,7 +273,7 @@ public Codec createForVideoEncoding(Format format, List<String> allowedMimeTypes
}

if (mimeType.equals(MimeTypes.VIDEO_H264)) {
adjustMediaFormatForH264EncoderSettings(mediaFormat, encoderInfo);
adjustMediaFormatForH264EncoderSettings(format.colorInfo, encoderInfo, mediaFormat);
}

MediaFormatUtil.maybeSetColorInfo(mediaFormat, encoderSupportedFormat.colorInfo);
Expand Down Expand Up @@ -523,12 +523,21 @@ private static void adjustMediaFormatForEncoderPerformanceSettings(MediaFormat m
* <p>The adjustment is applied in-place to {@code mediaFormat}.
*/
private static void adjustMediaFormatForH264EncoderSettings(
MediaFormat mediaFormat, MediaCodecInfo encoderInfo) {
@Nullable ColorInfo colorInfo, MediaCodecInfo encoderInfo, MediaFormat mediaFormat) {
// TODO(b/210593256): Remove overriding profile/level (before API 29) after switching to in-app
// muxing.
String mimeType = MimeTypes.VIDEO_H264;
if (Util.SDK_INT >= 29) {
int expectedEncodingProfile = MediaCodecInfo.CodecProfileLevel.AVCProfileHigh;
if (colorInfo != null) {
int colorTransfer = colorInfo.colorTransfer;
ImmutableList<Integer> codecProfiles =
EncoderUtil.getCodecProfilesForHdrFormat(mimeType, colorTransfer);
if (!codecProfiles.isEmpty()) {
// Default to the most compatible profile, which is first in the list.
expectedEncodingProfile = codecProfiles.get(0);
}
}
int supportedEncodingLevel =
EncoderUtil.findHighestSupportedEncodingLevel(
encoderInfo, mimeType, expectedEncodingProfile);
Expand Down
80 changes: 80 additions & 0 deletions library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java
View file
Expand Up @@ -31,10 +31,13 @@
import androidx.annotation.DoNotInline;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.C.ColorTransfer;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.util.MediaFormatUtil;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.ColorInfo;
import com.google.common.base.Ascii;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
Expand Down Expand Up @@ -65,6 +68,83 @@ public static ImmutableSet<String> getSupportedVideoMimeTypes() {
return checkNotNull(MIME_TYPE_TO_ENCODERS.get()).keySet();
}

/**
* Returns the names of encoders that support HDR editing for the given format, or an empty list
* if the format is unknown or not supported for HDR encoding.
*/
public static ImmutableList<String> getSupportedEncoderNamesForHdrEditing(
String mimeType, @Nullable ColorInfo colorInfo) {
if (Util.SDK_INT < 31 || colorInfo == null) {
return ImmutableList.of();
}

@ColorTransfer int colorTransfer = colorInfo.colorTransfer;
ImmutableList<Integer> profiles = getCodecProfilesForHdrFormat(mimeType, colorTransfer);
ImmutableList.Builder<String> resultBuilder = ImmutableList.builder();
ImmutableList<MediaCodecInfo> mediaCodecInfos =
EncoderSelector.DEFAULT.selectEncoderInfos(mimeType);
for (int i = 0; i < mediaCodecInfos.size(); i++) {
MediaCodecInfo mediaCodecInfo = mediaCodecInfos.get(i);
if (mediaCodecInfo.isAlias()
|| !EncoderUtil.isFeatureSupported(
mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)) {
continue;
}
for (MediaCodecInfo.CodecProfileLevel codecProfileLevel :
mediaCodecInfo.getCapabilitiesForType(mimeType).profileLevels) {
if (profiles.contains(codecProfileLevel.profile)) {
resultBuilder.add(mediaCodecInfo.getName());
}
}
}
return resultBuilder.build();
}

/**
* Returns the {@linkplain MediaCodecInfo.CodecProfileLevel#profile profile} constants that can be
* used to encode the given HDR format, if supported by the device (this method does not check
* device capabilities). If multiple profiles are returned, they are ordered by expected level of
* compatibility, with the most widely compatible profile first.
*/
@SuppressWarnings("InlinedApi") // Safe use of inlined constants from newer API versions.
public static ImmutableList<Integer> getCodecProfilesForHdrFormat(
String mimeType, @ColorTransfer int colorTransfer) {
// TODO(b/239174610): Add a way to determine profiles for DV and HDR10+.
switch (mimeType) {
case MimeTypes.VIDEO_VP9:
if (colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084) {
// Profiles support both HLG and PQ.
return ImmutableList.of(
MediaCodecInfo.CodecProfileLevel.VP9Profile2HDR,
MediaCodecInfo.CodecProfileLevel.VP9Profile3HDR);
}
break;
case MimeTypes.VIDEO_H264:
if (colorTransfer == C.COLOR_TRANSFER_HLG) {
return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AVCProfileHigh10);
}
break;
case MimeTypes.VIDEO_H265:
if (colorTransfer == C.COLOR_TRANSFER_HLG) {
return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10);
} else if (colorTransfer == C.COLOR_TRANSFER_ST2084) {
return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10HDR10);
}
break;
case MimeTypes.VIDEO_AV1:
if (colorTransfer == C.COLOR_TRANSFER_HLG) {
return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10);
} else if (colorTransfer == C.COLOR_TRANSFER_ST2084) {
return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10HDR10);
}
break;
default:
break;
}
// There are no profiles defined for the HDR format, or it's invalid.
return ImmutableList.of();
}

/** Returns whether the {@linkplain MediaCodecInfo encoder} supports the given resolution. */
public static boolean isSizeSupported(
MediaCodecInfo encoderInfo, String mimeType, int width, int height) {
Expand Down
113 changes: 43 additions & 70 deletions ...c/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
View file
Expand Up @@ -17,25 +17,20 @@
package com.google.android.exoplayer2.transformer;

import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState;

import android.content.Context;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.ColorInfo;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.dataflow.qual.Pure;
Expand Down Expand Up @@ -105,17 +100,6 @@ public VideoTranscodingSamplePipeline(
transformationRequest,
fallbackListener);

boolean enableRequestSdrToneMapping = transformationRequest.enableRequestSdrToneMapping;
// TODO(b/237674316): While HLG10 is correctly reported, HDR10 currently will be incorrectly
// processed as SDR, because the inputFormat.colorInfo reports the wrong value.
boolean useHdr =
transformationRequest.enableHdrEditing && ColorInfo.isHdr(inputFormat.colorInfo);
if (useHdr && !encoderWrapper.supportsHdr()) {
useHdr = false;
enableRequestSdrToneMapping = true;
encoderWrapper.signalFallbackToSdr();
}

try {
frameProcessor =
GlEffectsFrameProcessor.create(
Expand Down Expand Up @@ -153,17 +137,19 @@ public void onFrameProcessingEnded() {
// HDR is only used if the MediaCodec encoder supports FEATURE_HdrEditing. This
// implies that the OpenGL EXT_YUV_target extension is supported and hence the
// GlEffectsFrameProcessor also supports HDR.
useHdr);
/* useHdr= */ encoderWrapper.isHdrEditingEnabled());
} catch (FrameProcessingException e) {
throw TransformationException.createForFrameProcessingException(
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
frameProcessor.setInputFrameInfo(
new FrameInfo(decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio));

boolean isToneMappingRequired =
ColorInfo.isHdr(inputFormat.colorInfo) && !encoderWrapper.isHdrEditingEnabled();
decoder =
decoderFactory.createForVideoDecoding(
inputFormat, frameProcessor.getInputSurface(), enableRequestSdrToneMapping);
inputFormat, frameProcessor.getInputSurface(), isToneMappingRequired);
// TODO(b/236316454): Check in the decoder output format whether tone-mapping was actually
// applied and throw an exception if not.
maxPendingFrameCount = decoder.getMaxPendingFrameCount();
Expand Down Expand Up @@ -331,29 +317,41 @@ private boolean isDecodeOnlyBuffer(long presentationTimeUs) {
private final List<String> allowedOutputMimeTypes;
private final TransformationRequest transformationRequest;
private final FallbackListener fallbackListener;
private final HashSet<String> hdrMediaCodecNames;
private final String requestedOutputMimeType;
private final ImmutableList<String> supportedEncoderNamesForHdrEditing;

private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo;

private volatile @MonotonicNonNull Codec encoder;
private volatile int outputRotationDegrees;
private volatile boolean releaseEncoder;
private boolean fallbackToSdr;

public EncoderWrapper(
Codec.EncoderFactory encoderFactory,
Format inputFormat,
List<String> allowedOutputMimeTypes,
TransformationRequest transformationRequest,
FallbackListener fallbackListener) {

this.encoderFactory = encoderFactory;
this.inputFormat = inputFormat;
this.allowedOutputMimeTypes = allowedOutputMimeTypes;
this.transformationRequest = transformationRequest;
this.fallbackListener = fallbackListener;

hdrMediaCodecNames = new HashSet<>();
requestedOutputMimeType =
transformationRequest.videoMimeType != null
? transformationRequest.videoMimeType
: checkNotNull(inputFormat.sampleMimeType);
supportedEncoderNamesForHdrEditing =
EncoderUtil.getSupportedEncoderNamesForHdrEditing(
requestedOutputMimeType, inputFormat.colorInfo);
}

/** Returns whether the wrapped encoder is expecting HDR input for the HDR editing use case. */
public boolean isHdrEditingEnabled() {
return transformationRequest.enableHdrEditing
&& !transformationRequest.enableRequestSdrToneMapping
&& !supportedEncoderNamesForHdrEditing.isEmpty();
}

@Nullable
Expand All @@ -378,37 +376,39 @@ public SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight)
outputRotationDegrees = 90;
}

boolean isInputToneMapped = ColorInfo.isHdr(inputFormat.colorInfo) && !isHdrEditingEnabled();
Format requestedEncoderFormat =
new Format.Builder()
.setWidth(requestedWidth)
.setHeight(requestedHeight)
.setRotationDegrees(0)
.setFrameRate(inputFormat.frameRate)
.setSampleMimeType(
transformationRequest.videoMimeType != null
? transformationRequest.videoMimeType
: inputFormat.sampleMimeType)
.setColorInfo(fallbackToSdr ? null : inputFormat.colorInfo)
.setSampleMimeType(requestedOutputMimeType)
.setColorInfo(isInputToneMapped ? null : inputFormat.colorInfo)
.build();

encoder =
encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes);
if (!hdrMediaCodecNames.isEmpty() && !hdrMediaCodecNames.contains(encoder.getName())) {
Log.d(
TAG,
"Selected encoder "
+ encoder.getName()
+ " does not report sufficient HDR capabilities");
}

Format encoderSupportedFormat = encoder.getConfigurationFormat();
if (isHdrEditingEnabled()) {
if (!requestedOutputMimeType.equals(encoderSupportedFormat.sampleMimeType)) {
throw createEncodingException(
new IllegalStateException("MIME type fallback unsupported with HDR editing"),
encoderSupportedFormat);
} else if (!supportedEncoderNamesForHdrEditing.contains(encoder.getName())) {
throw createEncodingException(
new IllegalStateException("Selected encoder doesn't support HDR editing"),
encoderSupportedFormat);
}
}
fallbackListener.onTransformationRequestFinalized(
createFallbackTransformationRequest(
transformationRequest,
/* hasOutputFormatRotation= */ flipOrientation,
requestedEncoderFormat,
encoderSupportedFormat,
fallbackToSdr));
isInputToneMapped));

encoderSurfaceInfo =
new SurfaceInfo(
Expand Down Expand Up @@ -468,41 +468,14 @@ public void release() {
releaseEncoder = true;
}

/**
* Checks whether at least one MediaCodec encoder on the device has sufficient capabilities to
* encode HDR (only checks support for HLG at this time).
*/
public boolean supportsHdr() {
if (Util.SDK_INT < 31) {
return false;
}

// The only output MIME type that Transformer currently supports that can be used with HDR
// is H265/HEVC. So we assume that the EncoderFactory will pick this if HDR is requested.
String mimeType = MimeTypes.VIDEO_H265;

List<MediaCodecInfo> mediaCodecInfos = EncoderSelector.DEFAULT.selectEncoderInfos(mimeType);
for (int i = 0; i < mediaCodecInfos.size(); i++) {
MediaCodecInfo mediaCodecInfo = mediaCodecInfos.get(i);
if (EncoderUtil.isFeatureSupported(
mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)) {
for (MediaCodecInfo.CodecProfileLevel capabilities :
mediaCodecInfo.getCapabilitiesForType(MimeTypes.VIDEO_H265).profileLevels) {
// TODO(b/227624622): What profile to check depends on the HDR format. Once other
// formats besides HLG are supported, check the corresponding profiles here.
if (capabilities.profile == MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10) {
return hdrMediaCodecNames.add(mediaCodecInfo.getCanonicalName());
}
}
}
}
return !hdrMediaCodecNames.isEmpty();
}

public void signalFallbackToSdr() {
checkState(encoder == null, "Fallback to SDR is only allowed before encoder initialization");
fallbackToSdr = true;
hdrMediaCodecNames.clear();
private TransformationException createEncodingException(Exception cause, Format format) {
return TransformationException.createForCodec(
cause,
/* isVideo= */ true,
/* isDecoder= */ false,
format,
checkNotNull(encoder).getName(),
TransformationException.ERROR_CODE_ENCODING_FAILED);
}
}
}
3 changes: 2 additions & 1 deletion ...rmer/src/test/java/com/google/android/exoplayer2/transformer/VideoEncoderWrapperTest.java
View file
Expand Up @@ -27,6 +27,7 @@
import com.google.android.exoplayer2.MediaItem;
import com.google.android.exoplayer2.util.Clock;
import com.google.android.exoplayer2.util.ListenerSet;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.common.collect.ImmutableList;
import java.util.List;
import org.junit.Before;
Expand All @@ -47,7 +48,7 @@ public final class VideoEncoderWrapperTest {
private final VideoTranscodingSamplePipeline.EncoderWrapper encoderWrapper =
new VideoTranscodingSamplePipeline.EncoderWrapper(
fakeEncoderFactory,
/* inputFormat= */ new Format.Builder().build(),
/* inputFormat= */ new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H265).build(),
/* allowedOutputMimeTypes= */ ImmutableList.of(),
emptyTransformationRequest,
fallbackListener);
Expand Down

0 comments on commit 0db07c6

Please sign in to comment.