diff --git a/RELEASENOTES.md b/RELEASENOTES.md index ea2390e291..b1dfb314b4 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -87,6 +87,8 @@ Release notes * RTSP: * Add H263 fragmented packet handling ([#119](https://github.com/androidx/media/pull/119)). + * Add support for MP4A-LATM + ([#162](https://github.com/androidx/media/pull/162)). * IMA: * Add timeout for loading ad information to handle cases where the IMA SDK gets stuck loading an ad diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java index 55bb804642..17d1df7495 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java @@ -39,22 +39,23 @@ @UnstableApi public final class RtpPayloadFormat { - private static final String RTP_MEDIA_AC3 = "AC3"; - private static final String RTP_MEDIA_AMR = "AMR"; - private static final String RTP_MEDIA_AMR_WB = "AMR-WB"; - private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC"; - private static final String RTP_MEDIA_MPEG4_VIDEO = "MP4V-ES"; - private static final String RTP_MEDIA_H263_1998 = "H263-1998"; - private static final String RTP_MEDIA_H263_2000 = "H263-2000"; - private static final String RTP_MEDIA_H264 = "H264"; - private static final String RTP_MEDIA_H265 = "H265"; - private static final String RTP_MEDIA_OPUS = "OPUS"; - private static final String RTP_MEDIA_PCM_L8 = "L8"; - private static final String RTP_MEDIA_PCM_L16 = "L16"; - private static final String RTP_MEDIA_PCMA = "PCMA"; - private static final String RTP_MEDIA_PCMU = "PCMU"; - private static final String RTP_MEDIA_VP8 = "VP8"; - private static final String RTP_MEDIA_VP9 = "VP9"; + public static final String RTP_MEDIA_AC3 = "AC3"; + public static final String RTP_MEDIA_AMR = "AMR"; + public static final String RTP_MEDIA_AMR_WB = "AMR-WB"; + public static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC"; + public static final String RTP_MEDIA_MPEG4_LATM_AUDIO = "MP4A-LATM"; + public static final String RTP_MEDIA_MPEG4_VIDEO = "MP4V-ES"; + public static final String RTP_MEDIA_H263_1998 = "H263-1998"; + public static final String RTP_MEDIA_H263_2000 = "H263-2000"; + public static final String RTP_MEDIA_H264 = "H264"; + public static final String RTP_MEDIA_H265 = "H265"; + public static final String RTP_MEDIA_OPUS = "OPUS"; + public static final String RTP_MEDIA_PCM_L8 = "L8"; + public static final String RTP_MEDIA_PCM_L16 = "L16"; + public static final String RTP_MEDIA_PCMA = "PCMA"; + public static final String RTP_MEDIA_PCMU = "PCMU"; + public static final String RTP_MEDIA_VP8 = "VP8"; + public static final String RTP_MEDIA_VP9 = "VP9"; /** Returns whether the format of a {@link MediaDescription} is supported. */ public static boolean isFormatSupported(MediaDescription mediaDescription) { @@ -66,8 +67,9 @@ public static boolean isFormatSupported(MediaDescription mediaDescription) { case RTP_MEDIA_H263_2000: case RTP_MEDIA_H264: case RTP_MEDIA_H265: - case RTP_MEDIA_MPEG4_VIDEO: case RTP_MEDIA_MPEG4_GENERIC: + case RTP_MEDIA_MPEG4_LATM_AUDIO: + case RTP_MEDIA_MPEG4_VIDEO: case RTP_MEDIA_OPUS: case RTP_MEDIA_PCM_L8: case RTP_MEDIA_PCM_L16: @@ -97,6 +99,7 @@ public static String getMimeTypeFromRtpMediaType(String mediaType) { case RTP_MEDIA_AMR_WB: return MimeTypes.AUDIO_AMR_WB; case RTP_MEDIA_MPEG4_GENERIC: + case RTP_MEDIA_MPEG4_LATM_AUDIO: return MimeTypes.AUDIO_AAC; case RTP_MEDIA_OPUS: return MimeTypes.AUDIO_OPUS; @@ -142,6 +145,8 @@ public static String getMimeTypeFromRtpMediaType(String mediaType) { public final Format format; /** The format parameters, mapped from the SDP FMTP attribute (RFC2327 Page 22). */ public final ImmutableMap fmtpParameters; + /** The RTP media encoding. */ + public final String mediaEncoding; /** * Creates a new instance. @@ -153,13 +158,19 @@ public static String getMimeTypeFromRtpMediaType(String mediaType) { * @param fmtpParameters The format parameters, from the SDP FMTP attribute (RFC2327 Page 22), * empty if unset. The keys and values are specified in the RFCs for specific formats. For * instance, RFC3640 Section 4.1 defines keys like profile-level-id and config. + * @param mediaEncoding The RTP media encoding. */ public RtpPayloadFormat( - Format format, int rtpPayloadType, int clockRate, Map fmtpParameters) { + Format format, + int rtpPayloadType, + int clockRate, + Map fmtpParameters, + String mediaEncoding) { this.rtpPayloadType = rtpPayloadType; this.clockRate = clockRate; this.format = format; this.fmtpParameters = ImmutableMap.copyOf(fmtpParameters); + this.mediaEncoding = mediaEncoding; } @Override @@ -174,7 +185,8 @@ public boolean equals(@Nullable Object o) { return rtpPayloadType == that.rtpPayloadType && clockRate == that.clockRate && format.equals(that.format) - && fmtpParameters.equals(that.fmtpParameters); + && fmtpParameters.equals(that.fmtpParameters) + && mediaEncoding.equals(that.mediaEncoding); } @Override @@ -184,6 +196,7 @@ public int hashCode() { result = 31 * result + clockRate; result = 31 * result + format.hashCode(); result = 31 * result + fmtpParameters.hashCode(); + result = 31 * result + mediaEncoding.hashCode(); return result; } } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index c8de624326..b3d79404a2 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -31,7 +31,9 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; +import androidx.media3.common.ParserException; import androidx.media3.common.util.CodecSpecificDataUtil; +import androidx.media3.common.util.ParsableBitArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.extractor.AacUtil; @@ -52,7 +54,8 @@ private static final String PARAMETER_H265_SPROP_PPS = "sprop-pps"; private static final String PARAMETER_H265_SPROP_VPS = "sprop-vps"; private static final String PARAMETER_H265_SPROP_MAX_DON_DIFF = "sprop-max-don-diff"; - private static final String PARAMETER_MP4V_CONFIG = "config"; + private static final String PARAMETER_MP4A_CONFIG = "config"; + private static final String PARAMETER_MP4A_C_PRESENT = "cpresent"; /** Prefix for the RFC6381 codecs string for AAC formats. */ private static final String AAC_CODECS_PREFIX = "mp4a.40."; @@ -208,6 +211,23 @@ public int hashCode() { case MimeTypes.AUDIO_AAC: checkArgument(channelCount != C.INDEX_UNSET); checkArgument(!fmtpParameters.isEmpty()); + if (mediaEncoding.equals(RtpPayloadFormat.RTP_MEDIA_MPEG4_LATM_AUDIO)) { + // cpresent is defined in RFC3016 Section 5.3. cpresent=0 means the config fmtp parameter + // must exist. + checkArgument( + fmtpParameters.containsKey(PARAMETER_MP4A_C_PRESENT) + && fmtpParameters.get(PARAMETER_MP4A_C_PRESENT).equals("0"), + "Only supports cpresent=0 in AAC audio."); + @Nullable String config = fmtpParameters.get(PARAMETER_MP4A_CONFIG); + checkNotNull(config, "AAC audio stream must include config fmtp parameter"); + // config is a hex string. + checkArgument(config.length() % 2 == 0, "Malformat MPEG4 config: " + config); + AacUtil.Config aacConfig = parseAacStreamMuxConfig(config); + formatBuilder + .setSampleRate(aacConfig.sampleRateHz) + .setChannelCount(aacConfig.channelCount) + .setCodecs(aacConfig.codecs); + } processAacFmtpAttribute(formatBuilder, fmtpParameters, channelCount, clockRate); break; case MimeTypes.AUDIO_AMR_NB: @@ -267,7 +287,8 @@ public int hashCode() { } checkArgument(clockRate > 0); - return new RtpPayloadFormat(formatBuilder.build(), rtpPayloadType, clockRate, fmtpParameters); + return new RtpPayloadFormat( + formatBuilder.build(), rtpPayloadType, clockRate, fmtpParameters, mediaEncoding); } private static int inferChannelCount(int encodingParameter, String mimeType) { @@ -300,9 +321,29 @@ private static void processAacFmtpAttribute( AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount))); } + /** + * Returns the {@link AacUtil.Config} by parsing the MPEG4 Audio Stream Mux configuration. + * + *

fmtp attribute {@code config} includes the MPEG4 Audio Stream Mux configuration + * (ISO/IEC14496-3, Chapter 1.7.3). + */ + private static AacUtil.Config parseAacStreamMuxConfig(String streamMuxConfig) { + ParsableBitArray config = new ParsableBitArray(Util.getBytesFromHexString(streamMuxConfig)); + checkArgument(config.readBits(1) == 0, "Only supports audio mux version 0."); + checkArgument(config.readBits(1) == 1, "Only supports allStreamsSameTimeFraming."); + config.skipBits(6); + checkArgument(config.readBits(4) == 0, "Only supports one program."); + checkArgument(config.readBits(3) == 0, "Only supports one numLayer."); + try { + return AacUtil.parseAudioSpecificConfig(config, false); + } catch (ParserException e) { + throw new IllegalArgumentException(e); + } + } + private static void processMPEG4FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { - @Nullable String configInput = fmtpAttributes.get(PARAMETER_MP4V_CONFIG); + @Nullable String configInput = fmtpAttributes.get(PARAMETER_MP4A_CONFIG); if (configInput != null) { byte[] configBuffer = Util.getBytesFromHexString(configInput); formatBuilder.setInitializationData(ImmutableList.of(configBuffer)); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java index 0c1ee768b5..63883dce25 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java @@ -35,7 +35,11 @@ public RtpPayloadReader createPayloadReader(RtpPayloadFormat payloadFormat) { case MimeTypes.AUDIO_AC3: return new RtpAc3Reader(payloadFormat); case MimeTypes.AUDIO_AAC: - return new RtpAacReader(payloadFormat); + if (payloadFormat.mediaEncoding.equals(RtpPayloadFormat.RTP_MEDIA_MPEG4_LATM_AUDIO)) { + return new RtpMp4aReader(payloadFormat); + } else { + return new RtpAacReader(payloadFormat); + } case MimeTypes.AUDIO_AMR_NB: case MimeTypes.AUDIO_AMR_WB: return new RtpAmrReader(payloadFormat); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMp4aReader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMp4aReader.java new file mode 100644 index 0000000000..28886f2024 --- /dev/null +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMp4aReader.java @@ -0,0 +1,182 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.common.util.Util.castNonNull; +import static androidx.media3.exoplayer.rtsp.reader.RtpReaderUtils.toSampleTimeUs; + +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.ParserException; +import androidx.media3.common.util.ParsableBitArray; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPacket; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.TrackOutput; +import com.google.common.collect.ImmutableMap; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Parses an MP4A-LATM byte stream carried on RTP packets, and extracts MP4A-LATM Access Units. + * + *

Refer to RFC3016 for more details. The LATM byte stream format is defined in ISO/IEC14496-3. + */ +@UnstableApi +/* package */ final class RtpMp4aReader implements RtpPayloadReader { + private static final String TAG = "RtpMp4aReader"; + + private static final String PARAMETER_MP4A_CONFIG = "config"; + + private final RtpPayloadFormat payloadFormat; + private final int numberOfSubframes; + private @MonotonicNonNull TrackOutput trackOutput; + private long firstReceivedTimestamp; + private int previousSequenceNumber; + /** The combined size of a sample that is fragmented into multiple subFrames. */ + private int fragmentedSampleSizeBytes; + + private long startTimeOffsetUs; + private long fragmentedSampleTimeUs; + + /** + * Creates an instance. + * + * @throws IllegalArgumentException If {@link RtpPayloadFormat payloadFormat} is malformed. + */ + public RtpMp4aReader(RtpPayloadFormat payloadFormat) { + this.payloadFormat = payloadFormat; + try { + numberOfSubframes = getNumOfSubframesFromMpeg4AudioConfig(payloadFormat.fmtpParameters); + } catch (ParserException e) { + throw new IllegalArgumentException(e); + } + firstReceivedTimestamp = C.TIME_UNSET; + previousSequenceNumber = C.INDEX_UNSET; + fragmentedSampleSizeBytes = 0; + // The start time offset must be 0 until the first seek. + startTimeOffsetUs = 0; + fragmentedSampleTimeUs = C.TIME_UNSET; + } + + @Override + public void createTracks(ExtractorOutput extractorOutput, int trackId) { + trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); + castNonNull(trackOutput).format(payloadFormat.format); + } + + @Override + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) { + checkState(firstReceivedTimestamp == C.TIME_UNSET); + firstReceivedTimestamp = timestamp; + } + + @Override + public void consume( + ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) { + checkStateNotNull(trackOutput); + + int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber); + if (fragmentedSampleSizeBytes > 0 && expectedSequenceNumber < sequenceNumber) { + outputSampleMetadataForFragmentedPackets(); + } + + for (int subFrameIndex = 0; subFrameIndex < numberOfSubframes; subFrameIndex++) { + int sampleLength = 0; + // Implements PayloadLengthInfo() in ISO/IEC14496-3 Chapter 1.7.3.1, it only supports one + // program and one layer. Each subframe starts with a variable length encoding. + while (data.getPosition() < data.limit()) { + int payloadMuxLength = data.readUnsignedByte(); + sampleLength += payloadMuxLength; + if (payloadMuxLength != 0xff) { + break; + } + } + + trackOutput.sampleData(data, sampleLength); + fragmentedSampleSizeBytes += sampleLength; + } + fragmentedSampleTimeUs = + toSampleTimeUs( + startTimeOffsetUs, timestamp, firstReceivedTimestamp, payloadFormat.clockRate); + if (rtpMarker) { + outputSampleMetadataForFragmentedPackets(); + } + previousSequenceNumber = sequenceNumber; + } + + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + firstReceivedTimestamp = nextRtpTimestamp; + fragmentedSampleSizeBytes = 0; + startTimeOffsetUs = timeUs; + } + + // Internal methods. + + /** + * Parses an MPEG-4 Audio Stream Mux configuration, as defined in ISO/IEC14496-3. + * + *

FMTP attribute {@code config} contains the MPEG-4 Audio Stream Mux configuration. + * + * @param fmtpAttributes The format parameters, mapped from the SDP FMTP attribute. + * @return The number of subframes that is carried in each RTP packet. + */ + private static int getNumOfSubframesFromMpeg4AudioConfig( + ImmutableMap fmtpAttributes) throws ParserException { + @Nullable String configInput = fmtpAttributes.get(PARAMETER_MP4A_CONFIG); + int numberOfSubframes = 0; + if (configInput != null && configInput.length() % 2 == 0) { + byte[] configBuffer = Util.getBytesFromHexString(configInput); + ParsableBitArray scratchBits = new ParsableBitArray(configBuffer); + int audioMuxVersion = scratchBits.readBits(1); + if (audioMuxVersion == 0) { + checkArgument(scratchBits.readBits(1) == 1, "Only supports allStreamsSameTimeFraming."); + numberOfSubframes = scratchBits.readBits(6); + checkArgument(scratchBits.readBits(4) == 0, "Only suppors one program."); + checkArgument(scratchBits.readBits(3) == 0, "Only suppors one layer."); + } else { + throw ParserException.createForMalformedDataOfUnknownType( + "unsupported audio mux version: " + audioMuxVersion, null); + } + } + // ISO/IEC14496-3 Chapter 1.7.3.2.3: The minimum value is 0 indicating 1 subframe. + return numberOfSubframes + 1; + } + + /** + * Outputs sample metadata. + * + *

Call this method only after receiving the end of an MPEG4 partition. + */ + private void outputSampleMetadataForFragmentedPackets() { + checkNotNull(trackOutput) + .sampleMetadata( + fragmentedSampleTimeUs, + C.BUFFER_FLAG_KEY_FRAME, + fragmentedSampleSizeBytes, + /* offset= */ 0, + /* cryptoData= */ null); + fragmentedSampleSizeBytes = 0; + fragmentedSampleTimeUs = C.TIME_UNSET; + } +} diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspClientTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspClientTest.java index 4fdc1fc274..f7b7ab41b8 100644 --- a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspClientTest.java +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspClientTest.java @@ -70,8 +70,8 @@ public void setUp() throws Exception { ImmutableList.of( RtspTestUtils.readRtpPacketStreamDump("media/rtsp/h264-dump.json"), RtspTestUtils.readRtpPacketStreamDump("media/rtsp/aac-dump.json"), - // MP4A-LATM is not supported at the moment. - RtspTestUtils.readRtpPacketStreamDump("media/rtsp/mp4a-latm-dump.json")); + // MPEG2TS is not supported at the moment. + RtspTestUtils.readRtpPacketStreamDump("media/rtsp/mpeg2ts-dump.json")); } @After diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspMediaTrackTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspMediaTrackTest.java index 6f5b6fbc02..bfd2308c42 100644 --- a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspMediaTrackTest.java +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspMediaTrackTest.java @@ -76,7 +76,8 @@ public void generatePayloadFormat_withH264MediaDescription_succeeds() { /* fmtpParameters= */ ImmutableMap.of( "packetization-mode", "1", "profile-level-id", "64001F", - "sprop-parameter-sets", "Z2QAH6zZQPARabIAAAMACAAAAwGcHjBjLA==,aOvjyyLA")); + "sprop-parameter-sets", "Z2QAH6zZQPARabIAAAMACAAAAwGcHjBjLA==,aOvjyyLA"), + RtpPayloadFormat.RTP_MEDIA_H264); assertThat(format).isEqualTo(expectedFormat); } @@ -101,7 +102,8 @@ public void generatePayloadFormat_withPcmuMediaDescription_succeeds() { .build(), /* rtpPayloadType= */ 0, /* clockRate= */ 8_000, - /* fmtpParameters= */ ImmutableMap.of()); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_PCMU); assertThat(format).isEqualTo(expectedFormat); } @@ -134,7 +136,8 @@ public void generatePayloadFormat_withPcmaMediaDescription_succeeds() { .build(), /* rtpPayloadType= */ pcmaPayloadType, /* clockRate= */ pcmaClockRate, - /* fmtpParameters= */ ImmutableMap.of()); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_PCMA); assertThat(format).isEqualTo(expectedFormat); } @@ -168,7 +171,8 @@ public void generatePayloadFormat_withL16StereoMediaDescription_succeeds() { .build(), /* rtpPayloadType= */ l16StereoPayloadType, /* clockRate= */ l16StereoClockRate, - /* fmtpParameters= */ ImmutableMap.of()); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_PCM_L16); assertThat(format).isEqualTo(expectedFormat); } @@ -202,7 +206,8 @@ public void generatePayloadFormat_withL16MonoMediaDescription_succeeds() { .build(), /* rtpPayloadType= */ l16MonoPayloadType, /* clockRate= */ l16MonoClockRate, - /* fmtpParameters= */ ImmutableMap.of()); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_PCM_L16); assertThat(format).isEqualTo(expectedFormat); } @@ -244,7 +249,8 @@ public void generatePayloadFormat_withFmtpTrailingSemicolon_succeeds() { /* fmtpParameters= */ ImmutableMap.of( "packetization-mode", "1", "profile-level-id", "64001F", - "sprop-parameter-sets", "Z2QAH6zZQPARabIAAAMACAAAAwGcHjBjLA==,aOvjyyLA")); + "sprop-parameter-sets", "Z2QAH6zZQPARabIAAAMACAAAAwGcHjBjLA==,aOvjyyLA"), + RtpPayloadFormat.RTP_MEDIA_H264); assertThat(format).isEqualTo(expectedFormat); } @@ -288,7 +294,8 @@ public void generatePayloadFormat_withAacMediaDescription_succeeds() { .put("indexlength", "3") .put("indexdeltalength", "3") .put("config", "1208") - .buildOrThrow()); + .buildOrThrow(), + RtpPayloadFormat.RTP_MEDIA_MPEG4_GENERIC); assertThat(format).isEqualTo(expectedFormat); } @@ -315,7 +322,8 @@ public void generatePayloadFormat_withAc3MediaDescriptionWithDefaultChannelCount .build(), /* rtpPayloadType= */ 97, /* clockRate= */ 48000, - /* fmtpParameters= */ ImmutableMap.of()); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_AC3); assertThat(format).isEqualTo(expectedFormat); } @@ -342,7 +350,8 @@ public void generatePayloadFormat_withAc3MediaDescription_succeeds() { .build(), /* rtpPayloadType= */ 97, /* clockRate= */ 48000, - /* fmtpParameters= */ ImmutableMap.of()); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_AC3); assertThat(format).isEqualTo(expectedFormat); } diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspPlaybackTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspPlaybackTest.java index 0c93acc61f..dc44ce154c 100644 --- a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspPlaybackTest.java +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/RtspPlaybackTest.java @@ -72,7 +72,7 @@ public final class RtspPlaybackTest { private RtpPacketStreamDump aacRtpPacketStreamDump; // ExoPlayer does not support extracting MP4A-LATM RTP payload at the moment. - private RtpPacketStreamDump mp4aLatmRtpPacketStreamDump; + private RtpPacketStreamDump mpeg2tsRtpPacketStreamDump; /** Creates a new instance. */ public RtspPlaybackTest() { @@ -90,8 +90,8 @@ public RtspPlaybackTest() { @Before public void setUp() throws Exception { aacRtpPacketStreamDump = RtspTestUtils.readRtpPacketStreamDump("media/rtsp/aac-dump.json"); - mp4aLatmRtpPacketStreamDump = - RtspTestUtils.readRtpPacketStreamDump("media/rtsp/mp4a-latm-dump.json"); + mpeg2tsRtpPacketStreamDump = + RtspTestUtils.readRtpPacketStreamDump("media/rtsp/mpeg2ts-dump.json"); } @Test @@ -99,7 +99,7 @@ public void prepare_withSupportedTrack_playsTrackUntilEnded() throws Exception { ResponseProvider responseProvider = new ResponseProvider( clock, - ImmutableList.of(aacRtpPacketStreamDump, mp4aLatmRtpPacketStreamDump), + ImmutableList.of(aacRtpPacketStreamDump, mpeg2tsRtpPacketStreamDump), fakeRtpDataChannel); try (RtspServer rtspServer = new RtspServer(responseProvider)) { @@ -124,7 +124,7 @@ public void prepare_noSupportedTrack_throwsPreparationError() throws Exception { try (RtspServer rtspServer = new RtspServer( new ResponseProvider( - clock, ImmutableList.of(mp4aLatmRtpPacketStreamDump), fakeRtpDataChannel))) { + clock, ImmutableList.of(mpeg2tsRtpPacketStreamDump), fakeRtpDataChannel))) { ExoPlayer player = createExoPlayer(rtspServer.startAndGetPortNumber(), rtpDataChannelFactory); AtomicReference playbackError = new AtomicReference<>(); diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpAc3ReaderTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpAc3ReaderTest.java index 2404b669b5..de2cd91dc8 100644 --- a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpAc3ReaderTest.java +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpAc3ReaderTest.java @@ -75,7 +75,8 @@ public final class RtpAc3ReaderTest { .build(), /* rtpPayloadType= */ 97, /* clockRate= */ 48_000, - /* fmtpParameters= */ ImmutableMap.of()); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_AC3); @Rule public final MockitoRule mockito = MockitoJUnit.rule(); diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReaderTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReaderTest.java index cce1a3db44..858ea07ef6 100644 --- a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReaderTest.java +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpAmrReaderTest.java @@ -215,6 +215,9 @@ private static RtpPayloadFormat createRtpPayloadFormat(String mimeType, int samp .build(), /* rtpPayloadType= */ 97, /* clockRate= */ sampleRate, - /* fmtpParameters= */ ImmutableMap.of()); + /* fmtpParameters= */ ImmutableMap.of(), + MimeTypes.AUDIO_AMR.equals(mimeType) + ? RtpPayloadFormat.RTP_MEDIA_AMR + : RtpPayloadFormat.RTP_MEDIA_AMR_WB); } } diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpH263ReaderTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpH263ReaderTest.java index 4c1f4efde0..a623f6fb92 100644 --- a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpH263ReaderTest.java +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpH263ReaderTest.java @@ -109,7 +109,8 @@ public final class RtpH263ReaderTest { .build(), /* rtpPayloadType= */ 96, /* clockRate= */ (int) MEDIA_CLOCK_FREQUENCY, - /* fmtpParameters= */ ImmutableMap.of()); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_H263_1998); private FakeExtractorOutput extractorOutput; diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpMp4aReaderTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpMp4aReaderTest.java new file mode 100644 index 0000000000..c25e0bd4b9 --- /dev/null +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpMp4aReaderTest.java @@ -0,0 +1,160 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Util.getBytesFromHexString; +import static com.google.common.truth.Truth.assertThat; + +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.ParserException; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.exoplayer.rtsp.RtpPacket; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.test.utils.FakeExtractorOutput; +import androidx.media3.test.utils.FakeTrackOutput; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableMap; +import com.google.common.primitives.Bytes; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit test for {@link RtpMp4aReader}. */ +@RunWith(AndroidJUnit4.class) +public final class RtpMp4aReaderTest { + private static final byte[] FRAME_1_FRAGMENT_1_DATA = getBytesFromHexString("0102"); + private static final RtpPacket FRAME_1_FRAGMENT_1 = + new RtpPacket.Builder() + .setTimestamp(2599168056L) + .setSequenceNumber(40289) + .setMarker(false) + .setPayloadData( + Bytes.concat(/* payload size */ getBytesFromHexString("02"), FRAME_1_FRAGMENT_1_DATA)) + .build(); + private static final byte[] FRAME_1_FRAGMENT_2_DATA = getBytesFromHexString("030405"); + private static final RtpPacket FRAME_1_FRAGMENT_2 = + new RtpPacket.Builder() + .setTimestamp(2599168056L) + .setSequenceNumber(40290) + .setMarker(true) + .setPayloadData( + Bytes.concat(/* payload size */ getBytesFromHexString("03"), FRAME_1_FRAGMENT_2_DATA)) + .build(); + private static final byte[] FRAME_1_DATA = + Bytes.concat(FRAME_1_FRAGMENT_1_DATA, FRAME_1_FRAGMENT_2_DATA); + + private static final byte[] FRAME_2_FRAGMENT_1_DATA = getBytesFromHexString("0607"); + private static final RtpPacket FRAME_2_FRAGMENT_1 = + new RtpPacket.Builder() + .setTimestamp(2599168344L) + .setSequenceNumber(40291) + .setMarker(false) + .setPayloadData( + Bytes.concat(/* payload size */ getBytesFromHexString("02"), FRAME_2_FRAGMENT_1_DATA)) + .build(); + private static final byte[] FRAME_2_FRAGMENT_2_DATA = getBytesFromHexString("0809"); + private static final RtpPacket FRAME_2_FRAGMENT_2 = + new RtpPacket.Builder() + .setTimestamp(2599168344L) + .setSequenceNumber(40292) + .setMarker(true) + .setPayloadData( + Bytes.concat(/* payload size */ getBytesFromHexString("02"), FRAME_2_FRAGMENT_2_DATA)) + .build(); + private static final byte[] FRAME_2_DATA = + Bytes.concat(FRAME_2_FRAGMENT_1_DATA, FRAME_2_FRAGMENT_2_DATA); + + private static final RtpPayloadFormat MP4A_LATM_FORMAT = + new RtpPayloadFormat( + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setChannelCount(1).build(), + /* rtpPayloadType= */ 97, + /* clockRate= */ 44_100, + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_MPEG4_LATM_AUDIO); + + private FakeExtractorOutput extractorOutput; + + @Before + public void setUp() { + extractorOutput = new FakeExtractorOutput(); + } + + @Test + public void consume_validPackets() throws ParserException { + RtpMp4aReader mp4aLatmReader = new RtpMp4aReader(MP4A_LATM_FORMAT); + mp4aLatmReader.createTracks(extractorOutput, /* trackId= */ 0); + mp4aLatmReader.onReceivingFirstPacket( + FRAME_1_FRAGMENT_1.timestamp, FRAME_1_FRAGMENT_1.sequenceNumber); + consume(mp4aLatmReader, FRAME_1_FRAGMENT_1); + consume(mp4aLatmReader, FRAME_1_FRAGMENT_2); + consume(mp4aLatmReader, FRAME_2_FRAGMENT_1); + consume(mp4aLatmReader, FRAME_2_FRAGMENT_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_DATA); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(6530); + } + + @Test + public void consume_fragmentedFrameMissingFirstFragment() throws ParserException { + RtpMp4aReader mp4aLatmReader = new RtpMp4aReader(MP4A_LATM_FORMAT); + mp4aLatmReader.createTracks(extractorOutput, /* trackId= */ 0); + mp4aLatmReader.onReceivingFirstPacket( + FRAME_1_FRAGMENT_1.timestamp, FRAME_1_FRAGMENT_1.sequenceNumber); + consume(mp4aLatmReader, FRAME_1_FRAGMENT_2); + consume(mp4aLatmReader, FRAME_2_FRAGMENT_1); + consume(mp4aLatmReader, FRAME_2_FRAGMENT_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_FRAGMENT_2_DATA); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(6530); + } + + @Test + public void consume_fragmentedFrameMissingBoundaryFragment() throws ParserException { + RtpMp4aReader mp4aLatmReader = new RtpMp4aReader(MP4A_LATM_FORMAT); + mp4aLatmReader.createTracks(extractorOutput, /* trackId= */ 0); + mp4aLatmReader.onReceivingFirstPacket( + FRAME_1_FRAGMENT_1.timestamp, FRAME_1_FRAGMENT_1.sequenceNumber); + consume(mp4aLatmReader, FRAME_1_FRAGMENT_1); + consume(mp4aLatmReader, FRAME_2_FRAGMENT_1); + consume(mp4aLatmReader, FRAME_2_FRAGMENT_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_FRAGMENT_1_DATA); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(6530); + } + + private static void consume(RtpMp4aReader mpeg4Reader, RtpPacket rtpPacket) { + ParsableByteArray packetData = new ParsableByteArray(); + packetData.reset(rtpPacket.payloadData); + mpeg4Reader.consume( + packetData, + rtpPacket.timestamp, + rtpPacket.sequenceNumber, + /* isFrameBoundary= */ rtpPacket.marker); + } +} diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReaderTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReaderTest.java index 1b2ed3a50b..db8cd75e50 100644 --- a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReaderTest.java +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReaderTest.java @@ -51,7 +51,8 @@ public final class RtpOpusReaderTest { .build(), /* rtpPayloadType= */ 97, /* clockRate= */ 48_000, - /* fmtpParameters= */ ImmutableMap.of()); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_OPUS); private static final RtpPacket OPUS_HEADER = createRtpPacket( diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpPcmReaderTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpPcmReaderTest.java index bba419e3d6..81ea6810ac 100644 --- a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpPcmReaderTest.java +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpPcmReaderTest.java @@ -69,7 +69,8 @@ public void consume_twoDualChannelWav8bitPackets() { .build(), /* rtpPayloadType= */ RTP_PAYLOAD_TYPE, /* clockRate= */ 48_000, - /* fmtpParameters= */ ImmutableMap.of())); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_PCM_L8)); pcmReader.createTracks(extractorOutput, /* trackId= */ 0); pcmReader.onReceivingFirstPacket(PACKET_1.timestamp, PACKET_1.sequenceNumber); @@ -97,7 +98,8 @@ public void consume_twoSingleChannelWav16bitPackets() { .build(), /* rtpPayloadType= */ RTP_PAYLOAD_TYPE, /* clockRate= */ 60_000, - /* fmtpParameters= */ ImmutableMap.of())); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_PCM_L16)); pcmReader.createTracks(extractorOutput, /* trackId= */ 0); pcmReader.onReceivingFirstPacket(PACKET_1.timestamp, PACKET_1.sequenceNumber); @@ -124,7 +126,8 @@ public void consume_twoDualChannelAlawPackets() { .build(), /* rtpPayloadType= */ RTP_PAYLOAD_TYPE, /* clockRate= */ 16_000, - /* fmtpParameters= */ ImmutableMap.of())); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_PCMA)); pcmReader.createTracks(extractorOutput, /* trackId= */ 0); pcmReader.onReceivingFirstPacket(PACKET_1.timestamp, PACKET_1.sequenceNumber); @@ -151,7 +154,8 @@ public void consume_twoDualChannelMlawPackets() { .build(), /* rtpPayloadType= */ RTP_PAYLOAD_TYPE, /* clockRate= */ 24_000, - /* fmtpParameters= */ ImmutableMap.of())); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_PCMU)); pcmReader.createTracks(extractorOutput, /* trackId= */ 0); pcmReader.onReceivingFirstPacket(PACKET_1.timestamp, PACKET_1.sequenceNumber); diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpVp8ReaderTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpVp8ReaderTest.java index 73ffe05fc5..d45f274167 100644 --- a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpVp8ReaderTest.java +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpVp8ReaderTest.java @@ -190,7 +190,8 @@ private static RtpVp8Reader createVp8Reader() { new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_VP8).build(), /* rtpPayloadType= */ 96, /* clockRate= */ (int) MEDIA_CLOCK_FREQUENCY, - /* fmtpParameters= */ ImmutableMap.of())); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_VP8)); } private static void consume(RtpVp8Reader vp8Reader, RtpPacket rtpPacket) { diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpVp9ReaderTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpVp9ReaderTest.java index 22f87ff702..4cb7110877 100644 --- a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpVp9ReaderTest.java +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpVp9ReaderTest.java @@ -185,7 +185,8 @@ private static RtpVp9Reader createVp9Reader() { new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_VP9).build(), /* rtpPayloadType= */ 96, /* clockRate= */ (int) MEDIA_CLOCK_FREQUENCY, - /* fmtpParameters= */ ImmutableMap.of())); + /* fmtpParameters= */ ImmutableMap.of(), + RtpPayloadFormat.RTP_MEDIA_VP9)); } private static void consume(RtpVp9Reader vp9Reader, RtpPacket rtpPacket) { diff --git a/libraries/test_data/src/test/assets/media/rtsp/mp4a-latm-dump.json b/libraries/test_data/src/test/assets/media/rtsp/mp4a-latm-dump.json deleted file mode 100644 index a7ee934a83..0000000000 --- a/libraries/test_data/src/test/assets/media/rtsp/mp4a-latm-dump.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "trackName": "track3", - "firstSequenceNumber": 0, - "firstTimestamp": 0, - "transmitIntervalMs": 30, - "mediaDescription": "m=audio 0 RTP/AVP 97\r\nc=IN IP4 0.0.0.0\r\nb=AS:61\r\na=rtpmap:97 MP4A-LATM/44100/2\r\na=fmtp:97 profile-level-id=15;object=2;cpresent=0;config=400024203FC0\r\na=control:track3\r\n", - "packets": [ - ] -} diff --git a/libraries/test_data/src/test/assets/media/rtsp/mpeg2ts-dump.json b/libraries/test_data/src/test/assets/media/rtsp/mpeg2ts-dump.json new file mode 100644 index 0000000000..6546e52958 --- /dev/null +++ b/libraries/test_data/src/test/assets/media/rtsp/mpeg2ts-dump.json @@ -0,0 +1,9 @@ +{ + "trackName": "track3", + "firstSequenceNumber": 0, + "firstTimestamp": 0, + "transmitIntervalMs": 30, + "mediaDescription": "m=video 30000 RTP/AVP 32\r\nc=IN IP4 0.0.0.0\r\na=rtpmap:98 MP4/90000\r\na=control:track3\r\n", + "packets": [ + ] +}