1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 package org.chromium.media;
7 import android.media.AudioFormat;
8 import android.media.AudioManager;
9 import android.media.AudioTrack;
10 import android.media.MediaCodec;
11 import android.media.MediaCodecInfo;
12 import android.media.MediaCodecList;
13 import android.media.MediaCrypto;
14 import android.media.MediaFormat;
15 import android.os.Build;
16 import android.os.Bundle;
17 import android.util.Log;
18 import android.view.Surface;
20 import org.chromium.base.CalledByNative;
21 import org.chromium.base.JNINamespace;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.HashMap;
29 * A wrapper of the MediaCodec class to facilitate exception capturing and
32 @JNINamespace("media")
33 class MediaCodecBridge {
34 private static final String TAG = "MediaCodecBridge";
36 // Error code for MediaCodecBridge. Keep this value in sync with
37 // MediaCodecStatus in media_codec_bridge.h.
38 private static final int MEDIA_CODEC_OK = 0;
39 private static final int MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER = 1;
40 private static final int MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER = 2;
41 private static final int MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED = 3;
42 private static final int MEDIA_CODEC_OUTPUT_FORMAT_CHANGED = 4;
43 private static final int MEDIA_CODEC_INPUT_END_OF_STREAM = 5;
44 private static final int MEDIA_CODEC_OUTPUT_END_OF_STREAM = 6;
45 private static final int MEDIA_CODEC_NO_KEY = 7;
46 private static final int MEDIA_CODEC_STOPPED = 8;
47 private static final int MEDIA_CODEC_ERROR = 9;
49 // Codec direction. Keep this in sync with media_codec_bridge.h.
50 private static final int MEDIA_CODEC_DECODER = 0;
51 private static final int MEDIA_CODEC_ENCODER = 1;
53 // After a flush(), dequeueOutputBuffer() can often produce empty presentation timestamps
54 // for several frames. As a result, the player may find that the time does not increase
55 // after decoding a frame. To detect this, we check whether the presentation timestamp from
56 // dequeueOutputBuffer() is larger than input_timestamp - MAX_PRESENTATION_TIMESTAMP_SHIFT_US
57 // after a flush. And we set the presentation timestamp from dequeueOutputBuffer() to be
58 // non-decreasing for the remaining frames.
59 private static final long MAX_PRESENTATION_TIMESTAMP_SHIFT_US = 100000;
61 private ByteBuffer[] mInputBuffers;
62 private ByteBuffer[] mOutputBuffers;
64 private MediaCodec mMediaCodec;
65 private AudioTrack mAudioTrack;
66 private boolean mFlushed;
67 private long mLastPresentationTimeUs;
69 private static class DequeueInputResult {
70 private final int mStatus;
71 private final int mIndex;
73 private DequeueInputResult(int status, int index) {
78 @CalledByNative("DequeueInputResult")
79 private int status() { return mStatus; }
81 @CalledByNative("DequeueInputResult")
82 private int index() { return mIndex; }
86 * This class represents supported android codec information.
88 private static class CodecInfo {
89 private final String mCodecType; // e.g. "video/x-vnd.on2.vp8".
90 private final String mCodecName; // e.g. "OMX.google.vp8.decoder".
91 private final int mDirection;
93 private CodecInfo(String codecType, String codecName,
95 mCodecType = codecType;
96 mCodecName = codecName;
97 mDirection = direction;
100 @CalledByNative("CodecInfo")
101 private String codecType() { return mCodecType; }
103 @CalledByNative("CodecInfo")
104 private String codecName() { return mCodecName; }
106 @CalledByNative("CodecInfo")
107 private int direction() { return mDirection; }
110 private static class DequeueOutputResult {
111 private final int mStatus;
112 private final int mIndex;
113 private final int mFlags;
114 private final int mOffset;
115 private final long mPresentationTimeMicroseconds;
116 private final int mNumBytes;
118 private DequeueOutputResult(int status, int index, int flags, int offset,
119 long presentationTimeMicroseconds, int numBytes) {
124 mPresentationTimeMicroseconds = presentationTimeMicroseconds;
125 mNumBytes = numBytes;
128 @CalledByNative("DequeueOutputResult")
129 private int status() { return mStatus; }
131 @CalledByNative("DequeueOutputResult")
132 private int index() { return mIndex; }
134 @CalledByNative("DequeueOutputResult")
135 private int flags() { return mFlags; }
137 @CalledByNative("DequeueOutputResult")
138 private int offset() { return mOffset; }
140 @CalledByNative("DequeueOutputResult")
141 private long presentationTimeMicroseconds() { return mPresentationTimeMicroseconds; }
143 @CalledByNative("DequeueOutputResult")
144 private int numBytes() { return mNumBytes; }
148 * Get a list of supported android codec mimes.
151 private static CodecInfo[] getCodecsInfo() {
152 // Return the first (highest-priority) codec for each MIME type.
153 Map<String, CodecInfo> encoderInfoMap = new HashMap<String, CodecInfo>();
154 Map<String, CodecInfo> decoderInfoMap = new HashMap<String, CodecInfo>();
155 int count = MediaCodecList.getCodecCount();
156 for (int i = 0; i < count; ++i) {
157 MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i);
159 info.isEncoder() ? MEDIA_CODEC_ENCODER : MEDIA_CODEC_DECODER;
160 String codecString = info.getName();
161 String[] supportedTypes = info.getSupportedTypes();
162 for (int j = 0; j < supportedTypes.length; ++j) {
163 Map<String, CodecInfo> map = info.isEncoder() ? encoderInfoMap : decoderInfoMap;
164 if (!map.containsKey(supportedTypes[j])) {
165 map.put(supportedTypes[j], new CodecInfo(
166 supportedTypes[j], codecString, direction));
170 ArrayList<CodecInfo> codecInfos = new ArrayList<CodecInfo>(
171 decoderInfoMap.size() + encoderInfoMap.size());
172 codecInfos.addAll(encoderInfoMap.values());
173 codecInfos.addAll(decoderInfoMap.values());
174 return codecInfos.toArray(new CodecInfo[codecInfos.size()]);
177 private static String getSecureDecoderNameForMime(String mime) {
178 int count = MediaCodecList.getCodecCount();
179 for (int i = 0; i < count; ++i) {
180 MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i);
181 if (info.isEncoder()) {
185 String[] supportedTypes = info.getSupportedTypes();
186 for (int j = 0; j < supportedTypes.length; ++j) {
187 if (supportedTypes[j].equalsIgnoreCase(mime)) {
188 return info.getName() + ".secure";
196 private MediaCodecBridge(MediaCodec mediaCodec) {
197 assert mediaCodec != null;
198 mMediaCodec = mediaCodec;
199 mLastPresentationTimeUs = 0;
204 private static MediaCodecBridge create(String mime, boolean isSecure, int direction) {
205 // Creation of ".secure" codecs sometimes crash instead of throwing exceptions
206 // on pre-JBMR2 devices.
207 if (isSecure && Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN_MR2) {
210 MediaCodec mediaCodec = null;
212 // |isSecure| only applies to video decoders.
213 if (mime.startsWith("video") && isSecure && direction == MEDIA_CODEC_DECODER) {
214 mediaCodec = MediaCodec.createByCodecName(getSecureDecoderNameForMime(mime));
216 if (direction == MEDIA_CODEC_ENCODER) {
217 mediaCodec = MediaCodec.createEncoderByType(mime);
219 mediaCodec = MediaCodec.createDecoderByType(mime);
222 } catch (Exception e) {
223 Log.e(TAG, "Failed to create MediaCodec: " + mime + ", isSecure: "
224 + isSecure + ", direction: " + direction, e);
227 if (mediaCodec == null) {
231 return new MediaCodecBridge(mediaCodec);
235 private void release() {
237 mMediaCodec.release();
238 } catch(IllegalStateException e) {
239 // The MediaCodec is stuck in a wrong state, possibly due to losing
241 Log.e(TAG, "Cannot release media codec", e);
244 if (mAudioTrack != null) {
245 mAudioTrack.release();
250 private boolean start() {
253 mInputBuffers = mMediaCodec.getInputBuffers();
254 } catch (IllegalStateException e) {
255 Log.e(TAG, "Cannot start the media codec", e);
262 private DequeueInputResult dequeueInputBuffer(long timeoutUs) {
263 int status = MEDIA_CODEC_ERROR;
266 int indexOrStatus = mMediaCodec.dequeueInputBuffer(timeoutUs);
267 if (indexOrStatus >= 0) { // index!
268 status = MEDIA_CODEC_OK;
269 index = indexOrStatus;
270 } else if (indexOrStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
271 Log.e(TAG, "dequeueInputBuffer: MediaCodec.INFO_TRY_AGAIN_LATER");
272 status = MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER;
274 Log.e(TAG, "Unexpected index_or_status: " + indexOrStatus);
277 } catch (Exception e) {
278 Log.e(TAG, "Failed to dequeue input buffer", e);
280 return new DequeueInputResult(status, index);
284 private int flush() {
287 if (mAudioTrack != null) {
288 // Need to call pause() here, or otherwise flush() is a no-op.
293 } catch (IllegalStateException e) {
294 Log.e(TAG, "Failed to flush MediaCodec", e);
295 return MEDIA_CODEC_ERROR;
297 return MEDIA_CODEC_OK;
301 private void stop() {
303 if (mAudioTrack != null) {
309 private int getOutputHeight() {
310 return mMediaCodec.getOutputFormat().getInteger(MediaFormat.KEY_HEIGHT);
314 private int getOutputWidth() {
315 return mMediaCodec.getOutputFormat().getInteger(MediaFormat.KEY_WIDTH);
319 private ByteBuffer getInputBuffer(int index) {
320 return mInputBuffers[index];
324 private ByteBuffer getOutputBuffer(int index) {
325 return mOutputBuffers[index];
329 private int getInputBuffersCount() {
330 return mInputBuffers.length;
334 private int getOutputBuffersCount() {
335 return mOutputBuffers != null ? mOutputBuffers.length : -1;
339 private int getOutputBuffersCapacity() {
340 return mOutputBuffers != null ? mOutputBuffers[0].capacity() : -1;
344 private boolean getOutputBuffers() {
346 mOutputBuffers = mMediaCodec.getOutputBuffers();
347 } catch (IllegalStateException e) {
348 Log.e(TAG, "Cannot get output buffers", e);
355 private int queueInputBuffer(
356 int index, int offset, int size, long presentationTimeUs, int flags) {
357 resetLastPresentationTimeIfNeeded(presentationTimeUs);
359 mMediaCodec.queueInputBuffer(index, offset, size, presentationTimeUs, flags);
360 } catch (Exception e) {
361 Log.e(TAG, "Failed to queue input buffer", e);
362 return MEDIA_CODEC_ERROR;
364 return MEDIA_CODEC_OK;
368 private void setVideoBitrate(int bps) {
369 Bundle b = new Bundle();
370 b.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, bps);
371 mMediaCodec.setParameters(b);
375 private void requestKeyFrameSoon() {
376 Bundle b = new Bundle();
377 b.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0);
378 mMediaCodec.setParameters(b);
382 private int queueSecureInputBuffer(
383 int index, int offset, byte[] iv, byte[] keyId, int[] numBytesOfClearData,
384 int[] numBytesOfEncryptedData, int numSubSamples, long presentationTimeUs) {
385 resetLastPresentationTimeIfNeeded(presentationTimeUs);
387 MediaCodec.CryptoInfo cryptoInfo = new MediaCodec.CryptoInfo();
388 cryptoInfo.set(numSubSamples, numBytesOfClearData, numBytesOfEncryptedData,
389 keyId, iv, MediaCodec.CRYPTO_MODE_AES_CTR);
390 mMediaCodec.queueSecureInputBuffer(index, offset, cryptoInfo, presentationTimeUs, 0);
391 } catch (MediaCodec.CryptoException e) {
392 Log.e(TAG, "Failed to queue secure input buffer", e);
393 if (e.getErrorCode() == MediaCodec.CryptoException.ERROR_NO_KEY) {
394 Log.e(TAG, "MediaCodec.CryptoException.ERROR_NO_KEY");
395 return MEDIA_CODEC_NO_KEY;
397 Log.e(TAG, "MediaCodec.CryptoException with error code " + e.getErrorCode());
398 return MEDIA_CODEC_ERROR;
399 } catch (IllegalStateException e) {
400 Log.e(TAG, "Failed to queue secure input buffer", e);
401 return MEDIA_CODEC_ERROR;
403 return MEDIA_CODEC_OK;
407 private void releaseOutputBuffer(int index, boolean render) {
409 mMediaCodec.releaseOutputBuffer(index, render);
410 } catch(IllegalStateException e) {
411 // TODO(qinmin): May need to report the error to the caller. crbug.com/356498.
412 Log.e(TAG, "Failed to release output buffer", e);
417 private DequeueOutputResult dequeueOutputBuffer(long timeoutUs) {
418 MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
419 int status = MEDIA_CODEC_ERROR;
422 int indexOrStatus = mMediaCodec.dequeueOutputBuffer(info, timeoutUs);
423 if (info.presentationTimeUs < mLastPresentationTimeUs) {
424 // TODO(qinmin): return a special code through DequeueOutputResult
425 // to notify the native code the the frame has a wrong presentation
426 // timestamp and should be skipped.
427 info.presentationTimeUs = mLastPresentationTimeUs;
429 mLastPresentationTimeUs = info.presentationTimeUs;
431 if (indexOrStatus >= 0) { // index!
432 status = MEDIA_CODEC_OK;
433 index = indexOrStatus;
434 } else if (indexOrStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
435 status = MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED;
436 } else if (indexOrStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
437 status = MEDIA_CODEC_OUTPUT_FORMAT_CHANGED;
438 } else if (indexOrStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
439 status = MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER;
441 Log.e(TAG, "Unexpected index_or_status: " + indexOrStatus);
444 } catch (IllegalStateException e) {
445 Log.e(TAG, "Failed to dequeue output buffer", e);
448 return new DequeueOutputResult(
449 status, index, info.flags, info.offset, info.presentationTimeUs, info.size);
453 private boolean configureVideo(MediaFormat format, Surface surface, MediaCrypto crypto,
456 mMediaCodec.configure(format, surface, crypto, flags);
458 } catch (IllegalStateException e) {
459 Log.e(TAG, "Cannot configure the video codec", e);
465 private static MediaFormat createAudioFormat(String mime, int sampleRate, int channelCount) {
466 return MediaFormat.createAudioFormat(mime, sampleRate, channelCount);
470 private static MediaFormat createVideoDecoderFormat(String mime, int width, int height) {
471 return MediaFormat.createVideoFormat(mime, width, height);
475 private static MediaFormat createVideoEncoderFormat(String mime, int width, int height,
476 int bitRate, int frameRate, int iFrameInterval, int colorFormat) {
477 MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
478 format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
479 format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate);
480 format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval);
481 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
486 private static void setCodecSpecificData(MediaFormat format, int index, byte[] bytes) {
490 } else if (index == 1) {
494 format.setByteBuffer(name, ByteBuffer.wrap(bytes));
499 private static void setFrameHasADTSHeader(MediaFormat format) {
500 format.setInteger(MediaFormat.KEY_IS_ADTS, 1);
504 private boolean configureAudio(MediaFormat format, MediaCrypto crypto, int flags,
507 mMediaCodec.configure(format, null, crypto, flags);
509 int sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE);
510 int channelCount = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
511 int channelConfig = getAudioFormat(channelCount);
512 // Using 16bit PCM for output. Keep this value in sync with
513 // kBytesPerAudioOutputSample in media_codec_bridge.cc.
514 int minBufferSize = AudioTrack.getMinBufferSize(sampleRate, channelConfig,
515 AudioFormat.ENCODING_PCM_16BIT);
516 mAudioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, sampleRate, channelConfig,
517 AudioFormat.ENCODING_PCM_16BIT, minBufferSize, AudioTrack.MODE_STREAM);
518 if (mAudioTrack.getState() == AudioTrack.STATE_UNINITIALIZED) {
524 } catch (IllegalStateException e) {
525 Log.e(TAG, "Cannot configure the audio codec", e);
531 * Play the audio buffer that is passed in.
533 * @param buf Audio buffer to be rendered.
534 * @return The number of frames that have already been consumed by the
535 * hardware. This number resets to 0 after each flush call.
538 private long playOutputBuffer(byte[] buf) {
539 if (mAudioTrack == null) {
543 if (AudioTrack.PLAYSTATE_PLAYING != mAudioTrack.getPlayState()) {
546 int size = mAudioTrack.write(buf, 0, buf.length);
547 if (buf.length != size) {
548 Log.i(TAG, "Failed to send all data to audio output, expected size: " +
549 buf.length + ", actual size: " + size);
551 // TODO(qinmin): Returning the head position allows us to estimate
552 // the current presentation time in native code. However, it is
553 // better to use AudioTrack.getCurrentTimestamp() to get the last
554 // known time when a frame is played. However, we will need to
555 // convert the java nano time to C++ timestamp.
556 // If the stream runs too long, getPlaybackHeadPosition() could
557 // overflow. AudioTimestampHelper in MediaSourcePlayer has the same
558 // issue. See http://crbug.com/358801.
559 return mAudioTrack.getPlaybackHeadPosition();
563 private void setVolume(double volume) {
564 if (mAudioTrack != null) {
565 mAudioTrack.setStereoVolume((float) volume, (float) volume);
569 private void resetLastPresentationTimeIfNeeded(long presentationTimeUs) {
571 mLastPresentationTimeUs =
572 Math.max(presentationTimeUs - MAX_PRESENTATION_TIMESTAMP_SHIFT_US, 0);
577 private int getAudioFormat(int channelCount) {
578 switch (channelCount) {
580 return AudioFormat.CHANNEL_OUT_MONO;
582 return AudioFormat.CHANNEL_OUT_STEREO;
584 return AudioFormat.CHANNEL_OUT_QUAD;
586 return AudioFormat.CHANNEL_OUT_5POINT1;
588 return AudioFormat.CHANNEL_OUT_7POINT1;
590 return AudioFormat.CHANNEL_OUT_DEFAULT;