Add support for RTSP Mp4a-Latm

libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java

@@ -31,7 +31,9 @@
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
 import androidx.media3.common.MimeTypes;
+import androidx.media3.common.ParserException;
 import androidx.media3.common.util.CodecSpecificDataUtil;
+import androidx.media3.common.util.ParsableBitArray;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
 import androidx.media3.extractor.AacUtil;
@@ -53,6 +55,7 @@
   private static final String PARAMETER_H265_SPROP_VPS = "sprop-vps";
   private static final String PARAMETER_H265_SPROP_MAX_DON_DIFF = "sprop-max-don-diff";
   private static final String PARAMETER_MP4V_CONFIG = "config";
+  private static final String PARAMETER_MP4A_C_PRESENT = "cpresent";
 
   /** Prefix for the RFC6381 codecs string for AAC formats. */
   private static final String AAC_CODECS_PREFIX = "mp4a.40.";
@@ -208,6 +211,21 @@ public int hashCode() {
       case MimeTypes.AUDIO_AAC:
         checkArgument(channelCount != C.INDEX_UNSET);
         checkArgument(!fmtpParameters.isEmpty());
+        if(mediaEncoding.equals(RtpPayloadFormat.RTP_MEDIA_MPEG4_AUDIO)) {
+          boolean isConfigPresent  = true;
+          if (fmtpParameters.get(PARAMETER_MP4A_C_PRESENT) != null && fmtpParameters.get(
+              PARAMETER_MP4A_C_PRESENT).equals("0")) {
+            isConfigPresent  = false;
+          }
+          checkArgument(!isConfigPresent, "cpresent == 0 means we need to parse config");
+          @Nullable String configInput = fmtpParameters.get(PARAMETER_MP4V_CONFIG);
+          if (configInput != null && configInput.length() % 2 == 0) {
+            Pair<Integer, Integer> configParameters = getSampleRateAndChannelCount(configInput);
+            channelCount = configParameters.first;
+            clockRate = configParameters.second;
+            formatBuilder.setSampleRate(clockRate).setChannelCount(channelCount);
+          }
+        }
         processAacFmtpAttribute(formatBuilder, fmtpParameters, channelCount, clockRate);
         break;
       case MimeTypes.AUDIO_AMR_NB:
@@ -301,6 +319,32 @@ private static void processAacFmtpAttribute(
             AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount)));
   }
 
+  /**
+   * Returns a {@link Pair} of sample rate and channel count, by parsing the
+   *  MPEG4 Audio Stream Mux configuration.
+   *
+   * <p>fmtp attribute {@code config} includes the MPEG4 Audio Stream Mux
+   * configuration (ISO/IEC14496-3, Chapter 1.7.3).
+   */
+  private static Pair<Integer, Integer> getSampleRateAndChannelCount(String configInput) {
+    ParsableBitArray config  = new ParsableBitArray(Util.getBytesFromHexString(configInput));
+    int audioMuxVersion = config .readBits(1);
+    if (audioMuxVersion == 0) {
+      checkArgument(config .readBits(1) == 1, "Only supports one allStreamsSameTimeFraming.");
+      config .readBits(6);
+      checkArgument(config .readBits(4) == 0, "Only supports one program.");
+      checkArgument(config .readBits(3) == 0, "Only supports one numLayer.");
+      @Nullable AacUtil.Config aacConfig = null;
+      try {
+        aacConfig = AacUtil.parseAudioSpecificConfig(config , false);
+      } catch (ParserException e) {
+        throw new IllegalArgumentException(e);
+      }
+      return Pair.create(aacConfig.channelCount, aacConfig.sampleRateHz);
+    }
+    throw new IllegalArgumentException ("Only support audio mux version 0");
+  }
+
   private static void processMPEG4FmtpAttribute(
       Format.Builder formatBuilder, ImmutableMap<String, String> fmtpAttributes) {
     @Nullable String configInput = fmtpAttributes.get(PARAMETER_MP4V_CONFIG);

libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMp4aReader.java

@@ -54,7 +54,7 @@
   private int fragmentedSampleSizeBytes;
   private long startTimeOffsetUs;
   private long sampleTimeUsOfFragmentedSample;
-  private int numSubFrames;
+  private int numberOfSubframes;
 
   /** Creates an instance. */
   public RtpMp4aReader(RtpPayloadFormat payloadFormat) {
@@ -78,9 +78,9 @@ public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {
     checkState(firstReceivedTimestamp == C.TIME_UNSET);
     firstReceivedTimestamp = timestamp;
     try {
-      numSubFrames = getNumOfSubframesFromMpeg4AudioConfig(payloadFormat.fmtpParameters);
+      numberOfSubframes = getNumOfSubframesFromMpeg4AudioConfig(payloadFormat.fmtpParameters);
     } catch (ParserException e) {
-      e.printStackTrace();
+      throw new IllegalArgumentException(e);
     }
 }
 
@@ -94,23 +94,24 @@ public void consume(
     if(fragmentedSampleSizeBytes > 0 && expectedSequenceNumber < sequenceNumber) {
       outputSampleMetadataForFragmentedPackets();
     }
-    int sampleOffset = 0;
-    for (int subFrame = 0; subFrame <= numSubFrames; subFrame++) {
+    for (int subFrame = 0; subFrame < numberOfSubframes; subFrame++) {
       int sampleLength = 0;
 
-      /* Each subframe starts with a variable length encoding */
-      for (; sampleOffset < data.bytesLeft(); sampleOffset++) {
-        sampleLength += data.getData()[sampleOffset] & 0xff;
-        if ((data.getData()[sampleOffset] & 0xff) != 0xff) {
+      /**
+       * This implements PayloadLengthInfo() in Chapter 1.7.3.1, it's only support one program and
+       * one layer.
+       * Each subframe starts with a variable length encoding.
+       */
+      while(data.getPosition() < data.limit()) {
+        int payloadMuxLength = data.readUnsignedByte();
+        sampleLength += payloadMuxLength;
+        if (payloadMuxLength != 0xff) {
           break;
         }
       }
-      sampleOffset++;
-      data.setPosition(sampleOffset);
 
       // Write the audio sample
       trackOutput.sampleData(data, sampleLength);
-      sampleOffset += sampleLength;
       fragmentedSampleSizeBytes += sampleLength;
     }
     sampleTimeUsOfFragmentedSample = toSampleTimeUs(startTimeOffsetUs, timestamp,
@@ -140,25 +141,25 @@ public void seek(long nextRtpTimestamp, long timeUs) {
    * @throws ParserException If the MPEG-4 Audio Stream Mux configuration cannot be parsed due to
    *     unsupported audioMuxVersion.
    */
-  private static Integer getNumOfSubframesFromMpeg4AudioConfig(
+  private static int getNumOfSubframesFromMpeg4AudioConfig(
       ImmutableMap<String, String> fmtpAttributes) throws ParserException {
     @Nullable String configInput = fmtpAttributes.get(PARAMETER_MP4A_CONFIG);
-    int numSubFrames = 0;
+    int numberOfSubframes = 0;
     if (configInput != null && configInput.length() % 2 == 0) {
       byte[] configBuffer = Util.getBytesFromHexString(configInput);
       ParsableBitArray scratchBits = new ParsableBitArray(configBuffer);
       int audioMuxVersion = scratchBits.readBits(1);
       if (audioMuxVersion == 0) {
         checkArgument(scratchBits.readBits(1) == 1, "Invalid allStreamsSameTimeFraming.");
-        numSubFrames = scratchBits.readBits(6);
+        numberOfSubframes = scratchBits.readBits(6);
         checkArgument(scratchBits.readBits(4) == 0, "Invalid numProgram.");
         checkArgument(scratchBits.readBits(3) == 0, "Invalid numLayer.");
       } else {
         throw ParserException.createForMalformedDataOfUnknownType(
             "unsupported audio mux version: " + audioMuxVersion, null);
       }
     }
-    return numSubFrames;
+    return numberOfSubframes + 1;
   }
 
   /**

libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpMp4aReaderTest.java

@@ -0,0 +1,175 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.rtsp.reader;
+
+import static androidx.media3.common.util.Util.getBytesFromHexString;
+import static com.google.common.truth.Truth.assertThat;
+
+import androidx.media3.common.Format;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.ParserException;
+import androidx.media3.common.util.ParsableByteArray;
+import androidx.media3.exoplayer.rtsp.RtpPacket;
+import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
+import androidx.media3.test.utils.FakeExtractorOutput;
+import androidx.media3.test.utils.FakeTrackOutput;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.primitives.Bytes;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnit;
+import org.mockito.junit.MockitoRule;
+
+/** Unit test for {@link RtpMp4aReader}. */
+@RunWith(AndroidJUnit4.class)
+public final class RtpMp4aReaderTest {
+  private static final byte[] FRAME_1_FRAGMENT_1_DATA =
+      getBytesFromHexString("0102");
+  private static final RtpPacket FRAME_1_FRAGMENT_1 =
+      new RtpPacket.Builder()
+          .setTimestamp((int) 2599168056L)
+          .setSequenceNumber(40289)
+          .setMarker(false)
+          .setPayloadData(Bytes.concat(
+              /*payload size */ getBytesFromHexString("02"), FRAME_1_FRAGMENT_1_DATA))
+          .build();
+  private static final byte[] FRAME_1_FRAGMENT_2_DATA =
+      getBytesFromHexString("030405");
+  private static final RtpPacket FRAME_1_FRAGMENT_2 =
+      new RtpPacket.Builder()
+          .setTimestamp((int) 2599168056L)
+          .setSequenceNumber(40290)
+          .setMarker(true)
+          .setPayloadData(Bytes.concat(
+              /*payload size */ getBytesFromHexString("03"), FRAME_1_FRAGMENT_2_DATA))
+          .build();
+  private static final byte[] FRAME_1_DATA =
+      Bytes.concat(FRAME_1_FRAGMENT_1_DATA, FRAME_1_FRAGMENT_2_DATA);
+
+  private static final byte[] FRAME_2_FRAGMENT_1_DATA =
+      getBytesFromHexString("0607");
+  private static final RtpPacket FRAME_2_FRAGMENT_1 =
+      new RtpPacket.Builder()
+          .setTimestamp((int) 2599168344L)
+          .setSequenceNumber(40291)
+          .setMarker(false)
+          .setPayloadData(Bytes.concat(
+              /*payload size */ getBytesFromHexString("02"), FRAME_2_FRAGMENT_1_DATA))
+          .build();
+  private static final byte[] FRAME_2_FRAGMENT_2_DATA =
+      getBytesFromHexString("0809");
+  private static final RtpPacket FRAME_2_FRAGMENT_2 =
+      new RtpPacket.Builder()
+          .setTimestamp((int) 2599168344L)
+          .setSequenceNumber(40292)
+          .setMarker(true)
+          .setPayloadData(Bytes.concat(
+              /*payload size */ getBytesFromHexString("02"), FRAME_2_FRAGMENT_2_DATA))
+          .build();
+  private static final byte[] FRAME_2_DATA =
+      Bytes.concat(FRAME_2_FRAGMENT_1_DATA, FRAME_2_FRAGMENT_2_DATA);
+
+  private static final RtpPayloadFormat MP4ALATM_FORMAT =
+      new RtpPayloadFormat(
+          new Format.Builder()
+              .setSampleMimeType(MimeTypes.AUDIO_AAC)
+              .setWidth(352)
+              .setHeight(288)
+              .build(),
+          /* rtpPayloadType= */ 96,
+          /* clockRate= */ 90_000,
+          /* fmtpParameters= */ ImmutableMap.of(), RtpPayloadFormat.RTP_MEDIA_MPEG4_AUDIO);
+
+  @Rule public final MockitoRule mockito = MockitoJUnit.rule();
+
+  private FakeTrackOutput trackOutput;
+
+  private FakeExtractorOutput extractorOutput;
+
+  @Before
+  public void setUp() {
+    extractorOutput = new FakeExtractorOutput();
+  }
+
+  @Test
+  public void consume_validPackets() throws ParserException {
+    RtpMp4aReader mp4aLatmReader = new RtpMp4aReader(MP4ALATM_FORMAT);
+    mp4aLatmReader.createTracks(extractorOutput, /* trackId= */ 0);
+    mp4aLatmReader.onReceivingFirstPacket(
+        FRAME_1_FRAGMENT_1.timestamp, FRAME_1_FRAGMENT_1.sequenceNumber);
+    consume(mp4aLatmReader, FRAME_1_FRAGMENT_1);
+    consume(mp4aLatmReader, FRAME_1_FRAGMENT_2);
+    consume(mp4aLatmReader, FRAME_2_FRAGMENT_1);
+    consume(mp4aLatmReader, FRAME_2_FRAGMENT_2);
+
+    trackOutput = extractorOutput.trackOutputs.get(0);
+    assertThat(trackOutput.getSampleCount()).isEqualTo(2);
+    assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_DATA);
+    assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0);
+    assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA);
+    assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(3200);
+  }
+
+  @Test
+  public void consume_fragmentedFrameMissingFirstFragment() throws ParserException {
+    RtpMp4aReader mp4aLatmReader = new RtpMp4aReader(MP4ALATM_FORMAT);
+    mp4aLatmReader.createTracks(extractorOutput, /* trackId= */ 0);
+    mp4aLatmReader.onReceivingFirstPacket(
+        FRAME_1_FRAGMENT_1.timestamp, FRAME_1_FRAGMENT_1.sequenceNumber);
+    consume(mp4aLatmReader, FRAME_1_FRAGMENT_2);
+    consume(mp4aLatmReader, FRAME_2_FRAGMENT_1);
+    consume(mp4aLatmReader, FRAME_2_FRAGMENT_2);
+
+    trackOutput = extractorOutput.trackOutputs.get(0);
+    assertThat(trackOutput.getSampleCount()).isEqualTo(2);
+    assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_FRAGMENT_2_DATA);
+    assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0);
+    assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA);
+    assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(3200);
+  }
+
+  @Test
+  public void consume_fragmentedFrameMissingBoundaryFragment() throws ParserException {
+    RtpMp4aReader mp4aLatmReader = new RtpMp4aReader(MP4ALATM_FORMAT);
+    mp4aLatmReader.createTracks(extractorOutput, /* trackId= */ 0);
+    mp4aLatmReader.onReceivingFirstPacket(
+        FRAME_1_FRAGMENT_1.timestamp, FRAME_1_FRAGMENT_1.sequenceNumber);
+    consume(mp4aLatmReader, FRAME_1_FRAGMENT_1);
+    consume(mp4aLatmReader, FRAME_2_FRAGMENT_1);
+    consume(mp4aLatmReader, FRAME_2_FRAGMENT_2);
+
+    trackOutput = extractorOutput.trackOutputs.get(0);
+    assertThat(trackOutput.getSampleCount()).isEqualTo(2);
+    assertThat(trackOutput.getSampleData(0)).isEqualTo(FRAME_1_FRAGMENT_1_DATA);
+    assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0);
+    assertThat(trackOutput.getSampleData(1)).isEqualTo(FRAME_2_DATA);
+    assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(3200);
+  }
+
+  private static void consume(RtpMp4aReader mpeg4Reader, RtpPacket rtpPacket)
+      throws ParserException {
+    ParsableByteArray packetData = new ParsableByteArray();
+    packetData.reset(rtpPacket.payloadData);
+    mpeg4Reader.consume(
+        packetData,
+        rtpPacket.timestamp,
+        rtpPacket.sequenceNumber,
+        /* isFrameBoundary= */ rtpPacket.marker);
+  }
+}