From d2f807ebae7f7f8554e089a2a285b29940b0c0e9 Mon Sep 17 00:00:00 2001 From: Manisha Jajoo Date: Tue, 25 Jan 2022 13:57:18 +0530 Subject: [PATCH 1/5] Add support for RTSP MPEG4 Added MPEG4 RTP packet reader and added support for MPEG4 playback through RTSP Change-Id: I57c9a61b18471dbd2c368177ebfb89ee662f995b --- .../common/util/CodecSpecificDataUtil.java | 86 +++++++++ .../exoplayer/rtsp/RtpPayloadFormat.java | 4 + .../media3/exoplayer/rtsp/RtspMediaTrack.java | 27 +++ .../DefaultRtpPayloadReaderFactory.java | 2 + .../exoplayer/rtsp/reader/RtpMPEG4Reader.java | 166 ++++++++++++++++++ 5 files changed, 285 insertions(+) create mode 100644 libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java diff --git a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java index b83620df38..a8a19aee8a 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java @@ -31,6 +31,13 @@ public final class CodecSpecificDataUtil { private static final String[] HEVC_GENERAL_PROFILE_SPACE_STRINGS = new String[] {"", "A", "B", "C"}; + // MP4V-ES + private static final int VISUAL_OBJECT_LAYER = 1; + private static final int VISUAL_OBJECT_LAYER_START = 0x20; + private static final int EXTENDED_PAR = 0x0F; + private static final int RECTANGULAR = 0x00; + private static final int FINE_GRANULARITY_SCALABLE = 0x12; + /** * Parses an ALAC AudioSpecificConfig (i.e. an ALACSpecificConfig). @@ -72,6 +79,85 @@ public static boolean parseCea708InitializationData(List initializationD && initializationData.get(0)[0] == 1; } + /** + * Parses an MPEG-4 Visual configuration information, as defined in ISO/IEC14496-2 + * + * @param videoSpecificConfig A byte array containing the MPEG-4 Visual configuration information + * to parse. + * @return A pair consisting of the width and the height. + */ + public static Pair parseMpeg4VideoSpecificConfig(byte[] videoSpecificConfig) { + int offset = 0; + boolean foundVOL = false; + ParsableByteArray scdScratchBytes = new ParsableByteArray(videoSpecificConfig); + while (offset + 3 < videoSpecificConfig.length) { + if (scdScratchBytes.readUnsignedInt24() != VISUAL_OBJECT_LAYER + || (videoSpecificConfig[offset + 3] & 0xf0) != VISUAL_OBJECT_LAYER_START) { + scdScratchBytes.setPosition(scdScratchBytes.getPosition() - 2); + offset++; + continue; + } + foundVOL = true; + break; + } + + Assertions.checkArgument(foundVOL); + + ParsableBitArray scdScratchBits = new ParsableBitArray(videoSpecificConfig); + scdScratchBits.skipBits((offset + 4) * 8); + scdScratchBits.skipBits(1); // random_accessible_vol + + int videoObjectTypeIndication = scdScratchBits.readBits(8); + Assertions.checkArgument(videoObjectTypeIndication != FINE_GRANULARITY_SCALABLE); + + if (scdScratchBits.readBit()) { // object_layer_identifier + scdScratchBits.skipBits(4); // video_object_layer_verid + scdScratchBits.skipBits(3); // video_object_layer_priority + } + + int aspectRatioInfo = scdScratchBits.readBits(4); + if (aspectRatioInfo == EXTENDED_PAR) { + scdScratchBits.skipBits(8); // par_width + scdScratchBits.skipBits(8); // par_height + } + + if (scdScratchBits.readBit()) { // vol_control_parameters + scdScratchBits.skipBits(2); // chroma_format + scdScratchBits.skipBits(1); // low_delay + if (scdScratchBits.readBit()) { // vbv_parameters + scdScratchBits.skipBits(79); + } + } + + int videoObjectLayerShape = scdScratchBits.readBits(2); + Assertions.checkArgument(videoObjectLayerShape == RECTANGULAR); + + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + int vopTimeIncrementResolution = scdScratchBits.readBits(16); + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + + if (scdScratchBits.readBit()) { // fixed_vop_rate + Assertions.checkArgument(vopTimeIncrementResolution > 0); + --vopTimeIncrementResolution; + int numBits = 0; + while (vopTimeIncrementResolution > 0) { + ++numBits; + vopTimeIncrementResolution >>= 1; + } + scdScratchBits.skipBits(numBits); // fixed_vop_time_increment + } + + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + int videoObjectLayerWidth = scdScratchBits.readBits(13); + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + int videoObjectLayerHeight = scdScratchBits.readBits(13); + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + + scdScratchBits.skipBits(1); // interlaced + + return Pair.create(videoObjectLayerWidth, videoObjectLayerHeight); + } + /** * Builds an RFC 6381 AVC codec string using the provided parameters. * diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java index 4c4521e682..f44c68ba0a 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java @@ -38,6 +38,7 @@ public final class RtpPayloadFormat { private static final String RTP_MEDIA_AC3 = "AC3"; private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC"; + private static final String RTP_MEDIA_MPEG4_VIDEO = "MP4V-ES"; private static final String RTP_MEDIA_H264 = "H264"; /** Returns whether the format of a {@link MediaDescription} is supported. */ @@ -45,6 +46,7 @@ public static boolean isFormatSupported(MediaDescription mediaDescription) { switch (Ascii.toUpperCase(mediaDescription.rtpMapAttribute.mediaEncoding)) { case RTP_MEDIA_AC3: case RTP_MEDIA_H264: + case RTP_MEDIA_MPEG4_VIDEO: case RTP_MEDIA_MPEG4_GENERIC: return true; default: @@ -65,6 +67,8 @@ public static String getMimeTypeFromRtpMediaType(String mediaType) { return MimeTypes.AUDIO_AC3; case RTP_MEDIA_H264: return MimeTypes.VIDEO_H264; + case RTP_MEDIA_MPEG4_VIDEO: + return MimeTypes.VIDEO_MP4V; case RTP_MEDIA_MPEG4_GENERIC: return MimeTypes.AUDIO_AAC; default: diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 5b6b9a4607..02733aadcc 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -25,6 +25,7 @@ import android.net.Uri; import android.util.Base64; +import android.util.Pair; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; @@ -44,10 +45,14 @@ // Format specific parameter names. private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id"; private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets"; + private static final String PARAMETER_CONFIG = "config"; + /** Prefix for the RFC6381 codecs string for AAC formats. */ private static final String AAC_CODECS_PREFIX = "mp4a.40."; /** Prefix for the RFC6381 codecs string for AVC formats. */ private static final String H264_CODECS_PREFIX = "avc1."; + /** Prefix for the RFC6416 codecs string for MPEG4V-ES formats. */ + private static final String MPEG4_CODECS_PREFIX = "mp4v"; private static final String GENERIC_CONTROL_ATTR = "*"; @@ -116,6 +121,10 @@ public int hashCode() { checkArgument(!fmtpParameters.isEmpty()); processAacFmtpAttribute(formatBuilder, fmtpParameters, channelCount, clockRate); break; + case MimeTypes.VIDEO_MP4V: + checkArgument(!fmtpParameters.isEmpty()); + processMPEG4FmtpAttribute(formatBuilder, fmtpParameters); + break; case MimeTypes.VIDEO_H264: checkArgument(!fmtpParameters.isEmpty()); processH264FmtpAttribute(formatBuilder, fmtpParameters); @@ -160,6 +169,24 @@ private static void processAacFmtpAttribute( AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount))); } + private static void processMPEG4FmtpAttribute( + Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { + @Nullable String configInput = fmtpAttributes.get(PARAMETER_CONFIG); + if (configInput != null) { + byte[] csd = Util.getBytesFromHexString(configInput); + ImmutableList initializationData = ImmutableList.of(csd); + formatBuilder.setInitializationData(initializationData); + Pair dimensions = CodecSpecificDataUtil.parseMpeg4VideoSpecificConfig(csd); + formatBuilder.setWidth(dimensions.first); + formatBuilder.setHeight(dimensions.second); + } + @Nullable String profileLevel = fmtpAttributes.get(PARAMETER_PROFILE_LEVEL_ID); + if (profileLevel == null) { + profileLevel = "1"; // default + } + formatBuilder.setCodecs(MPEG4_CODECS_PREFIX + profileLevel); + } + private static void processH264FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_PARAMS)); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java index 8fe084c131..0d58957a34 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java @@ -38,6 +38,8 @@ public RtpPayloadReader createPayloadReader(RtpPayloadFormat payloadFormat) { return new RtpAacReader(payloadFormat); case MimeTypes.VIDEO_H264: return new RtpH264Reader(payloadFormat); + case MimeTypes.VIDEO_MP4V: + return new RtpMPEG4Reader(payloadFormat); default: // No supported reader, returning null. } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java new file mode 100644 index 0000000000..8d22cd82f3 --- /dev/null +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java @@ -0,0 +1,166 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.common.util.Util.castNonNull; + +import androidx.media3.common.C; +import androidx.media3.common.ParserException; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.TrackOutput; +import com.google.common.primitives.Bytes; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Parses an H265 byte stream carried on RTP packets, and extracts H265 Access Units. Refer to + * RFC6416 for more details. + */ +/* package */ final class RtpMPEG4Reader implements RtpPayloadReader { + private static final String TAG = "RtpMPEG4Reader"; + + private static final long MEDIA_CLOCK_FREQUENCY = 90_000; + + /** + * VOP unit type. + */ + private static final int I_VOP = 0; + + private final RtpPayloadFormat payloadFormat; + + private @MonotonicNonNull TrackOutput trackOutput; + @C.BufferFlags private int bufferFlags; + + private long firstReceivedTimestamp; + + private int previousSequenceNumber; + + private long startTimeOffsetUs; + + private int sampleLength; + + File output = null; + + FileOutputStream outputStream = null; + + /** Creates an instance. */ + public RtpMPEG4Reader(RtpPayloadFormat payloadFormat) { + this.payloadFormat = payloadFormat; + firstReceivedTimestamp = C.TIME_UNSET; + previousSequenceNumber = C.INDEX_UNSET; + sampleLength = 0; + try { + output = new File("/data/local/tmp/" + "mpeg4v_es.out"); + outputStream = new FileOutputStream(output); + } catch (IOException e) { + //do nothing; + } + } + + private static long toSampleUs( + long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { + return startTimeOffsetUs + + Util.scaleLargeTimestamp( + (rtpTimestamp - firstReceivedRtpTimestamp), + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ MEDIA_CLOCK_FREQUENCY); + } + + @Override + public void createTracks(ExtractorOutput extractorOutput, int trackId) { + trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); + castNonNull(trackOutput).format(payloadFormat.format); + } + + @Override + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) { + Log.i(TAG, "RtpMPEG4Reader onReceivingFirstPacket"); + } + + @Override + public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) + throws ParserException { + if (previousSequenceNumber != C.INDEX_UNSET && sequenceNumber != (previousSequenceNumber + 1)) { + Log.e(TAG, "Packet loss"); + } + checkStateNotNull(trackOutput); + + int limit = data.bytesLeft(); + trackOutput.sampleData(data, limit); + sampleLength += limit; + parseVopType(data); + + // Write the video sample + if (outputStream != null) { + try { + outputStream.write(data.getData()); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Marker (M) bit: The marker bit is set to 1 to indicate the last RTP + // packet(or only RTP packet) of a VOP. When multiple VOPs are carried + // in the same RTP packet, the marker bit is set to 1. + if (rtpMarker) { + if (firstReceivedTimestamp == C.TIME_UNSET) { + firstReceivedTimestamp = timestamp; + } + + long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); + trackOutput.sampleMetadata(timeUs, bufferFlags, sampleLength, 0, null); + sampleLength = 0; + } + + previousSequenceNumber = sequenceNumber; + } + + /** + * Parses VOP Coding type + * + * Sets {@link #bufferFlags} according to the VOP Coding type. + */ + private void parseVopType(ParsableByteArray data) { + // search for VOP_START_CODE (00 00 01 B6) + byte[] inputData = data.getData(); + byte[] startCode = {0x0, 0x0, 0x01, (byte) 0xB6}; + int vopStartCodePos = Bytes.indexOf(inputData, startCode); + if (vopStartCodePos != -1) { + data.setPosition(vopStartCodePos + 4); + int vopType = data.peekUnsignedByte() >> 6; + bufferFlags = getBufferFlagsFromVopType(vopType); + } + } + + @C.BufferFlags + private static int getBufferFlagsFromVopType(int vopType) { + return vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0; + } + + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + firstReceivedTimestamp = nextRtpTimestamp; + startTimeOffsetUs = timeUs; + sampleLength = 0; + } +} From 743437e34fe50f5ebd56b262c7e8080a2a98f128 Mon Sep 17 00:00:00 2001 From: Manisha Jajoo Date: Mon, 31 Jan 2022 13:36:37 +0530 Subject: [PATCH 2/5] Clean up RtpMpeg4Reader --- .../common/util/CodecSpecificDataUtil.java | 21 ++++++++---------- .../exoplayer/rtsp/reader/RtpMPEG4Reader.java | 22 ------------------- 2 files changed, 9 insertions(+), 34 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java index a8a19aee8a..b4b872a6da 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java @@ -36,7 +36,6 @@ public final class CodecSpecificDataUtil { private static final int VISUAL_OBJECT_LAYER_START = 0x20; private static final int EXTENDED_PAR = 0x0F; private static final int RECTANGULAR = 0x00; - private static final int FINE_GRANULARITY_SCALABLE = 0x12; /** * Parses an ALAC AudioSpecificConfig (i.e. an parseMpeg4VideoSpecificConfig(byte[] videoS break; } - Assertions.checkArgument(foundVOL); + Assertions.checkArgument(foundVOL, "Invalid input. VOL not found"); ParsableBitArray scdScratchBits = new ParsableBitArray(videoSpecificConfig); scdScratchBits.skipBits((offset + 4) * 8); scdScratchBits.skipBits(1); // random_accessible_vol - - int videoObjectTypeIndication = scdScratchBits.readBits(8); - Assertions.checkArgument(videoObjectTypeIndication != FINE_GRANULARITY_SCALABLE); + scdScratchBits.skipBits(8); // video_object_type_indication if (scdScratchBits.readBit()) { // object_layer_identifier scdScratchBits.skipBits(4); // video_object_layer_verid @@ -130,14 +127,14 @@ public static Pair parseMpeg4VideoSpecificConfig(byte[] videoS } int videoObjectLayerShape = scdScratchBits.readBits(2); - Assertions.checkArgument(videoObjectLayerShape == RECTANGULAR); + Assertions.checkArgument(videoObjectLayerShape == RECTANGULAR, "Unsupported feature"); - Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit int vopTimeIncrementResolution = scdScratchBits.readBits(16); - Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit if (scdScratchBits.readBit()) { // fixed_vop_rate - Assertions.checkArgument(vopTimeIncrementResolution > 0); + Assertions.checkArgument(vopTimeIncrementResolution > 0, "Invalid input"); --vopTimeIncrementResolution; int numBits = 0; while (vopTimeIncrementResolution > 0) { @@ -147,11 +144,11 @@ public static Pair parseMpeg4VideoSpecificConfig(byte[] videoS scdScratchBits.skipBits(numBits); // fixed_vop_time_increment } - Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit int videoObjectLayerWidth = scdScratchBits.readBits(13); - Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit int videoObjectLayerHeight = scdScratchBits.readBits(13); - Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit scdScratchBits.skipBits(1); // interlaced diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java index 8d22cd82f3..a34c1e14f7 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java @@ -27,9 +27,6 @@ import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.TrackOutput; import com.google.common.primitives.Bytes; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** @@ -59,22 +56,12 @@ private int sampleLength; - File output = null; - - FileOutputStream outputStream = null; - /** Creates an instance. */ public RtpMPEG4Reader(RtpPayloadFormat payloadFormat) { this.payloadFormat = payloadFormat; firstReceivedTimestamp = C.TIME_UNSET; previousSequenceNumber = C.INDEX_UNSET; sampleLength = 0; - try { - output = new File("/data/local/tmp/" + "mpeg4v_es.out"); - outputStream = new FileOutputStream(output); - } catch (IOException e) { - //do nothing; - } } private static long toSampleUs( @@ -110,15 +97,6 @@ public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, sampleLength += limit; parseVopType(data); - // Write the video sample - if (outputStream != null) { - try { - outputStream.write(data.getData()); - } catch (IOException e) { - e.printStackTrace(); - } - } - // Marker (M) bit: The marker bit is set to 1 to indicate the last RTP // packet(or only RTP packet) of a VOP. When multiple VOPs are carried // in the same RTP packet, the marker bit is set to 1. From dfef2d13872d7950f54a805a1a18ad83e6c510dd Mon Sep 17 00:00:00 2001 From: Manisha Jajoo Date: Tue, 8 Feb 2022 17:02:59 +0530 Subject: [PATCH 3/5] Some minor cleanup in RTPMpeg4Reader --- .../media3/exoplayer/rtsp/RtspMediaTrack.java | 4 +- .../exoplayer/rtsp/reader/RtpMPEG4Reader.java | 41 +++++++++---------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 02733aadcc..d0cc763720 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -45,7 +45,7 @@ // Format specific parameter names. private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id"; private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets"; - private static final String PARAMETER_CONFIG = "config"; + private static final String PARAMETER_MP4V_CONFIG = "config"; /** Prefix for the RFC6381 codecs string for AAC formats. */ private static final String AAC_CODECS_PREFIX = "mp4a.40."; @@ -171,7 +171,7 @@ private static void processAacFmtpAttribute( private static void processMPEG4FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { - @Nullable String configInput = fmtpAttributes.get(PARAMETER_CONFIG); + @Nullable String configInput = fmtpAttributes.get(PARAMETER_MP4V_CONFIG); if (configInput != null) { byte[] csd = Util.getBytesFromHexString(configInput); ImmutableList initializationData = ImmutableList.of(csd); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java index a34c1e14f7..a3dacd0f89 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java @@ -30,7 +30,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** - * Parses an H265 byte stream carried on RTP packets, and extracts H265 Access Units. Refer to + * Parses an MPEG4 byte stream carried on RTP packets, and extracts MPEG4 Access Units. Refer to * RFC6416 for more details. */ /* package */ final class RtpMPEG4Reader implements RtpPayloadReader { @@ -44,16 +44,11 @@ private static final int I_VOP = 0; private final RtpPayloadFormat payloadFormat; - private @MonotonicNonNull TrackOutput trackOutput; @C.BufferFlags private int bufferFlags; - private long firstReceivedTimestamp; - private int previousSequenceNumber; - private long startTimeOffsetUs; - private int sampleLength; /** Creates an instance. */ @@ -64,15 +59,6 @@ public RtpMPEG4Reader(RtpPayloadFormat payloadFormat) { sampleLength = 0; } - private static long toSampleUs( - long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { - return startTimeOffsetUs - + Util.scaleLargeTimestamp( - (rtpTimestamp - firstReceivedRtpTimestamp), - /* multiplier= */ C.MICROS_PER_SECOND, - /* divisor= */ MEDIA_CLOCK_FREQUENCY); - } - @Override public void createTracks(ExtractorOutput extractorOutput, int trackId) { trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); @@ -113,6 +99,15 @@ public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, previousSequenceNumber = sequenceNumber; } + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + firstReceivedTimestamp = nextRtpTimestamp; + startTimeOffsetUs = timeUs; + sampleLength = 0; + } + + // Internal methods. + /** * Parses VOP Coding type * @@ -130,15 +125,17 @@ private void parseVopType(ParsableByteArray data) { } } + private static long toSampleUs( + long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { + return startTimeOffsetUs + + Util.scaleLargeTimestamp( + (rtpTimestamp - firstReceivedRtpTimestamp), + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ MEDIA_CLOCK_FREQUENCY); + } + @C.BufferFlags private static int getBufferFlagsFromVopType(int vopType) { return vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0; } - - @Override - public void seek(long nextRtpTimestamp, long timeUs) { - firstReceivedTimestamp = nextRtpTimestamp; - startTimeOffsetUs = timeUs; - sampleLength = 0; - } } From e7567d2072dd3716bce7c7ff6b77b2d6184fd035 Mon Sep 17 00:00:00 2001 From: Manisha Jajoo Date: Wed, 9 Feb 2022 21:35:11 +0530 Subject: [PATCH 4/5] Fix review comments in RtpMPEG4Reader --- .../common/util/CodecSpecificDataUtil.java | 79 ++++++++++--------- .../media3/exoplayer/rtsp/RtspMediaTrack.java | 15 ++-- .../exoplayer/rtsp/reader/RtpMPEG4Reader.java | 45 ++++++----- 3 files changed, 73 insertions(+), 66 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java index b4b872a6da..821a7a2ba6 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java @@ -15,6 +15,8 @@ */ package androidx.media3.common.util; +import static androidx.media3.common.util.Assertions.checkArgument; + import android.util.Pair; import androidx.annotation.Nullable; import androidx.media3.common.C; @@ -85,14 +87,15 @@ public static boolean parseCea708InitializationData(List initializationD * to parse. * @return A pair consisting of the width and the height. */ - public static Pair parseMpeg4VideoSpecificConfig(byte[] videoSpecificConfig) { + public static Pair getVideoResolutionFromMpeg4VideoConfig( + byte[] videoSpecificConfig) { int offset = 0; boolean foundVOL = false; - ParsableByteArray scdScratchBytes = new ParsableByteArray(videoSpecificConfig); + ParsableByteArray scratchBytes = new ParsableByteArray(videoSpecificConfig); while (offset + 3 < videoSpecificConfig.length) { - if (scdScratchBytes.readUnsignedInt24() != VISUAL_OBJECT_LAYER + if (scratchBytes.readUnsignedInt24() != VISUAL_OBJECT_LAYER || (videoSpecificConfig[offset + 3] & 0xf0) != VISUAL_OBJECT_LAYER_START) { - scdScratchBytes.setPosition(scdScratchBytes.getPosition() - 2); + scratchBytes.setPosition(scratchBytes.getPosition() - 2); offset++; continue; } @@ -100,57 +103,59 @@ public static Pair parseMpeg4VideoSpecificConfig(byte[] videoS break; } - Assertions.checkArgument(foundVOL, "Invalid input. VOL not found"); + checkArgument(foundVOL, "Invalid input: VOL not found."); - ParsableBitArray scdScratchBits = new ParsableBitArray(videoSpecificConfig); - scdScratchBits.skipBits((offset + 4) * 8); - scdScratchBits.skipBits(1); // random_accessible_vol - scdScratchBits.skipBits(8); // video_object_type_indication + ParsableBitArray scratchBits = new ParsableBitArray(videoSpecificConfig); + // Skip the start codecs from the bitstream + scratchBits.skipBits((offset + 4) * 8); + scratchBits.skipBits(1); // random_accessible_vol + scratchBits.skipBits(8); // video_object_type_indication - if (scdScratchBits.readBit()) { // object_layer_identifier - scdScratchBits.skipBits(4); // video_object_layer_verid - scdScratchBits.skipBits(3); // video_object_layer_priority + if (scratchBits.readBit()) { // object_layer_identifier + scratchBits.skipBits(4); // video_object_layer_verid + scratchBits.skipBits(3); // video_object_layer_priority } - int aspectRatioInfo = scdScratchBits.readBits(4); + int aspectRatioInfo = scratchBits.readBits(4); if (aspectRatioInfo == EXTENDED_PAR) { - scdScratchBits.skipBits(8); // par_width - scdScratchBits.skipBits(8); // par_height + scratchBits.skipBits(8); // par_width + scratchBits.skipBits(8); // par_height } - if (scdScratchBits.readBit()) { // vol_control_parameters - scdScratchBits.skipBits(2); // chroma_format - scdScratchBits.skipBits(1); // low_delay - if (scdScratchBits.readBit()) { // vbv_parameters - scdScratchBits.skipBits(79); + if (scratchBits.readBit()) { // vol_control_parameters + scratchBits.skipBits(2); // chroma_format + scratchBits.skipBits(1); // low_delay + if (scratchBits.readBit()) { // vbv_parameters + scratchBits.skipBits(79); } } - int videoObjectLayerShape = scdScratchBits.readBits(2); - Assertions.checkArgument(videoObjectLayerShape == RECTANGULAR, "Unsupported feature"); + int videoObjectLayerShape = scratchBits.readBits(2); + checkArgument( + videoObjectLayerShape == RECTANGULAR, "Only supports rectangular video object layer shape"); - Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit - int vopTimeIncrementResolution = scdScratchBits.readBits(16); - Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit + checkArgument(scratchBits.readBit()); // marker_bit + int vopTimeIncrementResolution = scratchBits.readBits(16); + checkArgument(scratchBits.readBit()); // marker_bit - if (scdScratchBits.readBit()) { // fixed_vop_rate - Assertions.checkArgument(vopTimeIncrementResolution > 0, "Invalid input"); - --vopTimeIncrementResolution; - int numBits = 0; + if (scratchBits.readBit()) { // fixed_vop_rate + checkArgument(vopTimeIncrementResolution > 0); + vopTimeIncrementResolution--; + int numBitsToSkip = 0; while (vopTimeIncrementResolution > 0) { - ++numBits; + numBitsToSkip++; vopTimeIncrementResolution >>= 1; } - scdScratchBits.skipBits(numBits); // fixed_vop_time_increment + scratchBits.skipBits(numBitsToSkip); // fixed_vop_time_increment } - Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit - int videoObjectLayerWidth = scdScratchBits.readBits(13); - Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit - int videoObjectLayerHeight = scdScratchBits.readBits(13); - Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit + checkArgument(scratchBits.readBit()); // marker_bit + int videoObjectLayerWidth = scratchBits.readBits(13); + checkArgument(scratchBits.readBit()); // marker_bit + int videoObjectLayerHeight = scratchBits.readBits(13); + checkArgument(scratchBits.readBit()); // marker_bit - scdScratchBits.skipBits(1); // interlaced + scratchBits.skipBits(1); // interlaced return Pair.create(videoObjectLayerWidth, videoObjectLayerHeight); } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index d0cc763720..f8edb33311 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -174,17 +174,14 @@ private static void processMPEG4FmtpAttribute( @Nullable String configInput = fmtpAttributes.get(PARAMETER_MP4V_CONFIG); if (configInput != null) { byte[] csd = Util.getBytesFromHexString(configInput); - ImmutableList initializationData = ImmutableList.of(csd); - formatBuilder.setInitializationData(initializationData); - Pair dimensions = CodecSpecificDataUtil.parseMpeg4VideoSpecificConfig(csd); - formatBuilder.setWidth(dimensions.first); - formatBuilder.setHeight(dimensions.second); + formatBuilder.setInitializationData(ImmutableList.of(csd)); + Pair resolution = + CodecSpecificDataUtil.getVideoResolutionFromMpeg4VideoConfig(csd); + formatBuilder.setWidth(resolution.first); + formatBuilder.setHeight(resolution.second); } @Nullable String profileLevel = fmtpAttributes.get(PARAMETER_PROFILE_LEVEL_ID); - if (profileLevel == null) { - profileLevel = "1"; // default - } - formatBuilder.setCodecs(MPEG4_CODECS_PREFIX + profileLevel); + formatBuilder.setCodecs(MPEG4_CODECS_PREFIX + (profileLevel == null ? "1" : profileLevel)); } private static void processH264FmtpAttribute( diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java index a3dacd0f89..8154b9379b 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java @@ -23,6 +23,7 @@ import androidx.media3.common.util.Log; import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPacket; import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.TrackOutput; @@ -38,9 +39,7 @@ private static final long MEDIA_CLOCK_FREQUENCY = 90_000; - /** - * VOP unit type. - */ + /** VOP unit type. */ private static final int I_VOP = 0; private final RtpPayloadFormat payloadFormat; @@ -66,22 +65,31 @@ public void createTracks(ExtractorOutput extractorOutput, int trackId) { } @Override - public void onReceivingFirstPacket(long timestamp, int sequenceNumber) { - Log.i(TAG, "RtpMPEG4Reader onReceivingFirstPacket"); - } + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {} @Override public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) throws ParserException { - if (previousSequenceNumber != C.INDEX_UNSET && sequenceNumber != (previousSequenceNumber + 1)) { - Log.e(TAG, "Packet loss"); - } checkStateNotNull(trackOutput); + // Check that this packet is in the sequence of the previous packet. + if (previousSequenceNumber != C.INDEX_UNSET) { + int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber); + if (sequenceNumber != expectedSequenceNumber) { + Log.w( + TAG, + Util.formatInvariant( + "Received RTP packet with unexpected sequence number. Expected: %d; received: %d." + + " Dropping packet.", + expectedSequenceNumber, sequenceNumber)); + return; + } + } + // Parse VOP Type and get the buffer flags int limit = data.bytesLeft(); trackOutput.sampleData(data, limit); + if (sampleLength == 0) bufferFlags = getBufferFlagsFromVop(data); sampleLength += limit; - parseVopType(data); // Marker (M) bit: The marker bit is set to 1 to indicate the last RTP // packet(or only RTP packet) of a VOP. When multiple VOPs are carried @@ -95,7 +103,6 @@ public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, trackOutput.sampleMetadata(timeUs, bufferFlags, sampleLength, 0, null); sampleLength = 0; } - previousSequenceNumber = sequenceNumber; } @@ -109,20 +116,23 @@ public void seek(long nextRtpTimestamp, long timeUs) { // Internal methods. /** - * Parses VOP Coding type + * Parses VOP Coding type. * * Sets {@link #bufferFlags} according to the VOP Coding type. */ - private void parseVopType(ParsableByteArray data) { + @C.BufferFlags + private static int getBufferFlagsFromVop(ParsableByteArray data) { + int flags = 0; // search for VOP_START_CODE (00 00 01 B6) byte[] inputData = data.getData(); - byte[] startCode = {0x0, 0x0, 0x01, (byte) 0xB6}; + byte[] startCode = new byte[] {0x0, 0x0, 0x1, (byte) 0xB6}; int vopStartCodePos = Bytes.indexOf(inputData, startCode); if (vopStartCodePos != -1) { data.setPosition(vopStartCodePos + 4); int vopType = data.peekUnsignedByte() >> 6; - bufferFlags = getBufferFlagsFromVopType(vopType); + flags = vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0; } + return flags; } private static long toSampleUs( @@ -133,9 +143,4 @@ private static long toSampleUs( /* multiplier= */ C.MICROS_PER_SECOND, /* divisor= */ MEDIA_CLOCK_FREQUENCY); } - - @C.BufferFlags - private static int getBufferFlagsFromVopType(int vopType) { - return vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0; - } } From ef9393a33719793f587f34dd02c1be2088e1b8ff Mon Sep 17 00:00:00 2001 From: Manisha Jajoo Date: Mon, 21 Feb 2022 13:20:12 +0530 Subject: [PATCH 5/5] Fix review comments on RtpMPEG4Reader --- .../media3/exoplayer/rtsp/RtspMediaTrack.java | 14 ++++++++------ .../exoplayer/rtsp/reader/RtpMPEG4Reader.java | 9 +++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 97900c1e1d..f4b10981ca 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -56,7 +56,7 @@ /** Prefix for the RFC6381 codecs string for AVC formats. */ private static final String H264_CODECS_PREFIX = "avc1."; /** Prefix for the RFC6416 codecs string for MPEG4V-ES formats. */ - private static final String MPEG4_CODECS_PREFIX = "mp4v"; + private static final String MPEG4_CODECS_PREFIX = "mp4v."; private static final String GENERIC_CONTROL_ATTR = "*"; @@ -181,12 +181,14 @@ private static void processMPEG4FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { @Nullable String configInput = fmtpAttributes.get(PARAMETER_MP4V_CONFIG); if (configInput != null) { - byte[] csd = Util.getBytesFromHexString(configInput); - formatBuilder.setInitializationData(ImmutableList.of(csd)); + byte[] configBuffer = Util.getBytesFromHexString(configInput); + formatBuilder.setInitializationData(ImmutableList.of(configBuffer)); Pair resolution = - CodecSpecificDataUtil.getVideoResolutionFromMpeg4VideoConfig(csd); - formatBuilder.setWidth(resolution.first); - formatBuilder.setHeight(resolution.second); + CodecSpecificDataUtil.getVideoResolutionFromMpeg4VideoConfig(configBuffer); + formatBuilder.setWidth(resolution.first).setHeight(resolution.second); + } else { + // set the default width and height + formatBuilder.setWidth(352).setHeight(288); } @Nullable String profileLevel = fmtpAttributes.get(PARAMETER_PROFILE_LEVEL_ID); formatBuilder.setCodecs(MPEG4_CODECS_PREFIX + (profileLevel == null ? "1" : profileLevel)); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java index 8154b9379b..82556057c6 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java @@ -88,7 +88,9 @@ public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, // Parse VOP Type and get the buffer flags int limit = data.bytesLeft(); trackOutput.sampleData(data, limit); - if (sampleLength == 0) bufferFlags = getBufferFlagsFromVop(data); + if (sampleLength == 0) { + bufferFlags = getBufferFlagsFromVop(data); + } sampleLength += limit; // Marker (M) bit: The marker bit is set to 1 to indicate the last RTP @@ -122,7 +124,6 @@ public void seek(long nextRtpTimestamp, long timeUs) { */ @C.BufferFlags private static int getBufferFlagsFromVop(ParsableByteArray data) { - int flags = 0; // search for VOP_START_CODE (00 00 01 B6) byte[] inputData = data.getData(); byte[] startCode = new byte[] {0x0, 0x0, 0x1, (byte) 0xB6}; @@ -130,9 +131,9 @@ private static int getBufferFlagsFromVop(ParsableByteArray data) { if (vopStartCodePos != -1) { data.setPosition(vopStartCodePos + 4); int vopType = data.peekUnsignedByte() >> 6; - flags = vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0; + return (vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0); } - return flags; + return 0; } private static long toSampleUs(