1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 package org.chromium.media;
7 import android.media.AudioFormat;
8 import android.media.AudioManager;
9 import android.media.AudioTrack;
10 import android.media.MediaCodec;
11 import android.media.MediaCodecInfo;
12 import android.media.MediaCodecList;
13 import android.media.MediaCrypto;
14 import android.media.MediaFormat;
15 import android.os.Build;
16 import android.os.Bundle;
17 import android.util.Log;
18 import android.view.Surface;
20 import org.chromium.base.CalledByNative;
21 import org.chromium.base.JNINamespace;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.HashMap;
29 * A wrapper of the MediaCodec class to facilitate exception capturing and
32 @JNINamespace("media")
33 class MediaCodecBridge {
34 private static final String TAG = "MediaCodecBridge";
36 // Error code for MediaCodecBridge. Keep this value in sync with
37 // MediaCodecStatus in media_codec_bridge.h.
38 private static final int MEDIA_CODEC_OK = 0;
39 private static final int MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER = 1;
40 private static final int MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER = 2;
41 private static final int MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED = 3;
42 private static final int MEDIA_CODEC_OUTPUT_FORMAT_CHANGED = 4;
43 private static final int MEDIA_CODEC_INPUT_END_OF_STREAM = 5;
44 private static final int MEDIA_CODEC_OUTPUT_END_OF_STREAM = 6;
45 private static final int MEDIA_CODEC_NO_KEY = 7;
46 private static final int MEDIA_CODEC_STOPPED = 8;
47 private static final int MEDIA_CODEC_ERROR = 9;
49 // Codec direction. Keep this in sync with media_codec_bridge.h.
50 private static final int MEDIA_CODEC_DECODER = 0;
51 private static final int MEDIA_CODEC_ENCODER = 1;
53 // Max adaptive playback size to be supplied to the decoder.
54 private static final int MAX_ADAPTIVE_PLAYBACK_WIDTH = 1920;
55 private static final int MAX_ADAPTIVE_PLAYBACK_HEIGHT = 1080;
57 // After a flush(), dequeueOutputBuffer() can often produce empty presentation timestamps
58 // for several frames. As a result, the player may find that the time does not increase
59 // after decoding a frame. To detect this, we check whether the presentation timestamp from
60 // dequeueOutputBuffer() is larger than input_timestamp - MAX_PRESENTATION_TIMESTAMP_SHIFT_US
61 // after a flush. And we set the presentation timestamp from dequeueOutputBuffer() to be
62 // non-decreasing for the remaining frames.
63 private static final long MAX_PRESENTATION_TIMESTAMP_SHIFT_US = 100000;
65 private ByteBuffer[] mInputBuffers;
66 private ByteBuffer[] mOutputBuffers;
68 private MediaCodec mMediaCodec;
69 private AudioTrack mAudioTrack;
70 private boolean mFlushed;
71 private long mLastPresentationTimeUs;
73 private boolean mAdaptivePlaybackSupported;
75 private static class DequeueInputResult {
76 private final int mStatus;
77 private final int mIndex;
79 private DequeueInputResult(int status, int index) {
84 @CalledByNative("DequeueInputResult")
85 private int status() { return mStatus; }
87 @CalledByNative("DequeueInputResult")
88 private int index() { return mIndex; }
92 * This class represents supported android codec information.
94 private static class CodecInfo {
95 private final String mCodecType; // e.g. "video/x-vnd.on2.vp8".
96 private final String mCodecName; // e.g. "OMX.google.vp8.decoder".
97 private final int mDirection;
99 private CodecInfo(String codecType, String codecName,
101 mCodecType = codecType;
102 mCodecName = codecName;
103 mDirection = direction;
106 @CalledByNative("CodecInfo")
107 private String codecType() { return mCodecType; }
109 @CalledByNative("CodecInfo")
110 private String codecName() { return mCodecName; }
112 @CalledByNative("CodecInfo")
113 private int direction() { return mDirection; }
116 private static class DequeueOutputResult {
117 private final int mStatus;
118 private final int mIndex;
119 private final int mFlags;
120 private final int mOffset;
121 private final long mPresentationTimeMicroseconds;
122 private final int mNumBytes;
124 private DequeueOutputResult(int status, int index, int flags, int offset,
125 long presentationTimeMicroseconds, int numBytes) {
130 mPresentationTimeMicroseconds = presentationTimeMicroseconds;
131 mNumBytes = numBytes;
134 @CalledByNative("DequeueOutputResult")
135 private int status() { return mStatus; }
137 @CalledByNative("DequeueOutputResult")
138 private int index() { return mIndex; }
140 @CalledByNative("DequeueOutputResult")
141 private int flags() { return mFlags; }
143 @CalledByNative("DequeueOutputResult")
144 private int offset() { return mOffset; }
146 @CalledByNative("DequeueOutputResult")
147 private long presentationTimeMicroseconds() { return mPresentationTimeMicroseconds; }
149 @CalledByNative("DequeueOutputResult")
150 private int numBytes() { return mNumBytes; }
154 * Get a list of supported android codec mimes.
156 @SuppressWarnings("deprecation")
158 private static CodecInfo[] getCodecsInfo() {
159 // Return the first (highest-priority) codec for each MIME type.
160 Map<String, CodecInfo> encoderInfoMap = new HashMap<String, CodecInfo>();
161 Map<String, CodecInfo> decoderInfoMap = new HashMap<String, CodecInfo>();
162 int count = MediaCodecList.getCodecCount();
163 for (int i = 0; i < count; ++i) {
164 MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i);
166 info.isEncoder() ? MEDIA_CODEC_ENCODER : MEDIA_CODEC_DECODER;
167 String codecString = info.getName();
168 String[] supportedTypes = info.getSupportedTypes();
169 for (int j = 0; j < supportedTypes.length; ++j) {
170 Map<String, CodecInfo> map = info.isEncoder() ? encoderInfoMap : decoderInfoMap;
171 if (!map.containsKey(supportedTypes[j])) {
172 map.put(supportedTypes[j], new CodecInfo(
173 supportedTypes[j], codecString, direction));
177 ArrayList<CodecInfo> codecInfos = new ArrayList<CodecInfo>(
178 decoderInfoMap.size() + encoderInfoMap.size());
179 codecInfos.addAll(encoderInfoMap.values());
180 codecInfos.addAll(decoderInfoMap.values());
181 return codecInfos.toArray(new CodecInfo[codecInfos.size()]);
184 @SuppressWarnings("deprecation")
185 private static String getDecoderNameForMime(String mime) {
186 int count = MediaCodecList.getCodecCount();
187 for (int i = 0; i < count; ++i) {
188 MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i);
189 if (info.isEncoder()) {
193 String[] supportedTypes = info.getSupportedTypes();
194 for (int j = 0; j < supportedTypes.length; ++j) {
195 if (supportedTypes[j].equalsIgnoreCase(mime)) {
196 return info.getName();
204 private MediaCodecBridge(
205 MediaCodec mediaCodec, String mime, boolean adaptivePlaybackSupported) {
206 assert mediaCodec != null;
207 mMediaCodec = mediaCodec;
209 mLastPresentationTimeUs = 0;
211 mAdaptivePlaybackSupported = adaptivePlaybackSupported;
215 private static MediaCodecBridge create(String mime, boolean isSecure, int direction) {
216 // Creation of ".secure" codecs sometimes crash instead of throwing exceptions
217 // on pre-JBMR2 devices.
218 if (isSecure && Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN_MR2) {
221 MediaCodec mediaCodec = null;
222 boolean adaptivePlaybackSupported = false;
224 // |isSecure| only applies to video decoders.
225 if (mime.startsWith("video") && isSecure && direction == MEDIA_CODEC_DECODER) {
226 String decoderName = getDecoderNameForMime(mime);
227 if (decoderName == null) {
230 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) {
231 // To work around an issue that we cannot get the codec info from the secure
232 // decoder, create an insecure decoder first so that we can query its codec
233 // info. http://b/15587335.
234 MediaCodec insecureCodec = MediaCodec.createByCodecName(decoderName);
235 adaptivePlaybackSupported = codecSupportsAdaptivePlayback(insecureCodec, mime);
236 insecureCodec.release();
238 mediaCodec = MediaCodec.createByCodecName(decoderName + ".secure");
240 if (direction == MEDIA_CODEC_ENCODER) {
241 mediaCodec = MediaCodec.createEncoderByType(mime);
243 mediaCodec = MediaCodec.createDecoderByType(mime);
244 adaptivePlaybackSupported = codecSupportsAdaptivePlayback(mediaCodec, mime);
247 } catch (Exception e) {
248 Log.e(TAG, "Failed to create MediaCodec: " + mime + ", isSecure: "
249 + isSecure + ", direction: " + direction, e);
252 if (mediaCodec == null) {
255 return new MediaCodecBridge(mediaCodec, mime, adaptivePlaybackSupported);
259 private void release() {
261 mMediaCodec.release();
262 } catch (IllegalStateException e) {
263 // The MediaCodec is stuck in a wrong state, possibly due to losing
265 Log.e(TAG, "Cannot release media codec", e);
268 if (mAudioTrack != null) {
269 mAudioTrack.release();
273 @SuppressWarnings("deprecation")
275 private boolean start() {
278 mInputBuffers = mMediaCodec.getInputBuffers();
279 } catch (IllegalStateException e) {
280 Log.e(TAG, "Cannot start the media codec", e);
287 private DequeueInputResult dequeueInputBuffer(long timeoutUs) {
288 int status = MEDIA_CODEC_ERROR;
291 int indexOrStatus = mMediaCodec.dequeueInputBuffer(timeoutUs);
292 if (indexOrStatus >= 0) { // index!
293 status = MEDIA_CODEC_OK;
294 index = indexOrStatus;
295 } else if (indexOrStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
296 Log.e(TAG, "dequeueInputBuffer: MediaCodec.INFO_TRY_AGAIN_LATER");
297 status = MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER;
299 Log.e(TAG, "Unexpected index_or_status: " + indexOrStatus);
302 } catch (Exception e) {
303 Log.e(TAG, "Failed to dequeue input buffer", e);
305 return new DequeueInputResult(status, index);
309 private int flush() {
312 if (mAudioTrack != null) {
313 // Need to call pause() here, or otherwise flush() is a no-op.
318 } catch (IllegalStateException e) {
319 Log.e(TAG, "Failed to flush MediaCodec", e);
320 return MEDIA_CODEC_ERROR;
322 return MEDIA_CODEC_OK;
326 private void stop() {
328 if (mAudioTrack != null) {
334 private int getOutputHeight() {
335 return mMediaCodec.getOutputFormat().getInteger(MediaFormat.KEY_HEIGHT);
339 private int getOutputWidth() {
340 return mMediaCodec.getOutputFormat().getInteger(MediaFormat.KEY_WIDTH);
344 private ByteBuffer getInputBuffer(int index) {
345 return mInputBuffers[index];
349 private ByteBuffer getOutputBuffer(int index) {
350 return mOutputBuffers[index];
354 private int getInputBuffersCount() {
355 return mInputBuffers.length;
359 private int getOutputBuffersCount() {
360 return mOutputBuffers != null ? mOutputBuffers.length : -1;
364 private int getOutputBuffersCapacity() {
365 return mOutputBuffers != null ? mOutputBuffers[0].capacity() : -1;
368 @SuppressWarnings("deprecation")
370 private boolean getOutputBuffers() {
372 mOutputBuffers = mMediaCodec.getOutputBuffers();
373 } catch (IllegalStateException e) {
374 Log.e(TAG, "Cannot get output buffers", e);
381 private int queueInputBuffer(
382 int index, int offset, int size, long presentationTimeUs, int flags) {
383 resetLastPresentationTimeIfNeeded(presentationTimeUs);
385 mMediaCodec.queueInputBuffer(index, offset, size, presentationTimeUs, flags);
386 } catch (Exception e) {
387 Log.e(TAG, "Failed to queue input buffer", e);
388 return MEDIA_CODEC_ERROR;
390 return MEDIA_CODEC_OK;
394 private void setVideoBitrate(int bps) {
395 Bundle b = new Bundle();
396 b.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, bps);
397 mMediaCodec.setParameters(b);
401 private void requestKeyFrameSoon() {
402 Bundle b = new Bundle();
403 b.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0);
404 mMediaCodec.setParameters(b);
408 private int queueSecureInputBuffer(
409 int index, int offset, byte[] iv, byte[] keyId, int[] numBytesOfClearData,
410 int[] numBytesOfEncryptedData, int numSubSamples, long presentationTimeUs) {
411 resetLastPresentationTimeIfNeeded(presentationTimeUs);
413 MediaCodec.CryptoInfo cryptoInfo = new MediaCodec.CryptoInfo();
414 cryptoInfo.set(numSubSamples, numBytesOfClearData, numBytesOfEncryptedData,
415 keyId, iv, MediaCodec.CRYPTO_MODE_AES_CTR);
416 mMediaCodec.queueSecureInputBuffer(index, offset, cryptoInfo, presentationTimeUs, 0);
417 } catch (MediaCodec.CryptoException e) {
418 Log.e(TAG, "Failed to queue secure input buffer", e);
419 if (e.getErrorCode() == MediaCodec.CryptoException.ERROR_NO_KEY) {
420 Log.e(TAG, "MediaCodec.CryptoException.ERROR_NO_KEY");
421 return MEDIA_CODEC_NO_KEY;
423 Log.e(TAG, "MediaCodec.CryptoException with error code " + e.getErrorCode());
424 return MEDIA_CODEC_ERROR;
425 } catch (IllegalStateException e) {
426 Log.e(TAG, "Failed to queue secure input buffer", e);
427 return MEDIA_CODEC_ERROR;
429 return MEDIA_CODEC_OK;
433 private void releaseOutputBuffer(int index, boolean render) {
435 mMediaCodec.releaseOutputBuffer(index, render);
436 } catch (IllegalStateException e) {
437 // TODO(qinmin): May need to report the error to the caller. crbug.com/356498.
438 Log.e(TAG, "Failed to release output buffer", e);
442 @SuppressWarnings("deprecation")
444 private DequeueOutputResult dequeueOutputBuffer(long timeoutUs) {
445 MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
446 int status = MEDIA_CODEC_ERROR;
449 int indexOrStatus = mMediaCodec.dequeueOutputBuffer(info, timeoutUs);
450 if (info.presentationTimeUs < mLastPresentationTimeUs) {
451 // TODO(qinmin): return a special code through DequeueOutputResult
452 // to notify the native code the the frame has a wrong presentation
453 // timestamp and should be skipped.
454 info.presentationTimeUs = mLastPresentationTimeUs;
456 mLastPresentationTimeUs = info.presentationTimeUs;
458 if (indexOrStatus >= 0) { // index!
459 status = MEDIA_CODEC_OK;
460 index = indexOrStatus;
461 } else if (indexOrStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
462 status = MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED;
463 } else if (indexOrStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
464 status = MEDIA_CODEC_OUTPUT_FORMAT_CHANGED;
465 } else if (indexOrStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
466 status = MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER;
468 Log.e(TAG, "Unexpected index_or_status: " + indexOrStatus);
471 } catch (IllegalStateException e) {
472 Log.e(TAG, "Failed to dequeue output buffer", e);
475 return new DequeueOutputResult(
476 status, index, info.flags, info.offset, info.presentationTimeUs, info.size);
480 private boolean configureVideo(MediaFormat format, Surface surface, MediaCrypto crypto,
483 if (mAdaptivePlaybackSupported) {
484 format.setInteger(MediaFormat.KEY_MAX_WIDTH, MAX_ADAPTIVE_PLAYBACK_WIDTH);
485 format.setInteger(MediaFormat.KEY_MAX_HEIGHT, MAX_ADAPTIVE_PLAYBACK_HEIGHT);
487 mMediaCodec.configure(format, surface, crypto, flags);
489 } catch (IllegalStateException e) {
490 Log.e(TAG, "Cannot configure the video codec", e);
496 private static MediaFormat createAudioFormat(String mime, int sampleRate, int channelCount) {
497 return MediaFormat.createAudioFormat(mime, sampleRate, channelCount);
501 private static MediaFormat createVideoDecoderFormat(String mime, int width, int height) {
502 return MediaFormat.createVideoFormat(mime, width, height);
506 private static MediaFormat createVideoEncoderFormat(String mime, int width, int height,
507 int bitRate, int frameRate, int iFrameInterval, int colorFormat) {
508 MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
509 format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
510 format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate);
511 format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval);
512 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
517 private boolean isAdaptivePlaybackSupported(int width, int height) {
518 if (!mAdaptivePlaybackSupported)
520 return width <= MAX_ADAPTIVE_PLAYBACK_WIDTH && height <= MAX_ADAPTIVE_PLAYBACK_HEIGHT;
523 private static boolean codecSupportsAdaptivePlayback(MediaCodec mediaCodec, String mime) {
524 if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT || mediaCodec == null) {
528 MediaCodecInfo info = mediaCodec.getCodecInfo();
529 if (info.isEncoder()) {
532 MediaCodecInfo.CodecCapabilities capabilities = info.getCapabilitiesForType(mime);
533 return (capabilities != null) && capabilities.isFeatureSupported(
534 MediaCodecInfo.CodecCapabilities.FEATURE_AdaptivePlayback);
535 } catch (IllegalArgumentException e) {
536 Log.e(TAG, "Cannot retrieve codec information", e);
542 private static void setCodecSpecificData(MediaFormat format, int index, byte[] bytes) {
546 } else if (index == 1) {
550 format.setByteBuffer(name, ByteBuffer.wrap(bytes));
555 private static void setFrameHasADTSHeader(MediaFormat format) {
556 format.setInteger(MediaFormat.KEY_IS_ADTS, 1);
560 private boolean configureAudio(MediaFormat format, MediaCrypto crypto, int flags,
563 mMediaCodec.configure(format, null, crypto, flags);
565 int sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE);
566 int channelCount = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
567 int channelConfig = getAudioFormat(channelCount);
568 // Using 16bit PCM for output. Keep this value in sync with
569 // kBytesPerAudioOutputSample in media_codec_bridge.cc.
570 int minBufferSize = AudioTrack.getMinBufferSize(sampleRate, channelConfig,
571 AudioFormat.ENCODING_PCM_16BIT);
572 mAudioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, sampleRate, channelConfig,
573 AudioFormat.ENCODING_PCM_16BIT, minBufferSize, AudioTrack.MODE_STREAM);
574 if (mAudioTrack.getState() == AudioTrack.STATE_UNINITIALIZED) {
580 } catch (IllegalStateException e) {
581 Log.e(TAG, "Cannot configure the audio codec", e);
587 * Play the audio buffer that is passed in.
589 * @param buf Audio buffer to be rendered.
590 * @return The number of frames that have already been consumed by the
591 * hardware. This number resets to 0 after each flush call.
594 private long playOutputBuffer(byte[] buf) {
595 if (mAudioTrack == null) {
599 if (AudioTrack.PLAYSTATE_PLAYING != mAudioTrack.getPlayState()) {
602 int size = mAudioTrack.write(buf, 0, buf.length);
603 if (buf.length != size) {
604 Log.i(TAG, "Failed to send all data to audio output, expected size: " +
605 buf.length + ", actual size: " + size);
607 // TODO(qinmin): Returning the head position allows us to estimate
608 // the current presentation time in native code. However, it is
609 // better to use AudioTrack.getCurrentTimestamp() to get the last
610 // known time when a frame is played. However, we will need to
611 // convert the java nano time to C++ timestamp.
612 // If the stream runs too long, getPlaybackHeadPosition() could
613 // overflow. AudioTimestampHelper in MediaSourcePlayer has the same
614 // issue. See http://crbug.com/358801.
615 return mAudioTrack.getPlaybackHeadPosition();
618 @SuppressWarnings("deprecation")
620 private void setVolume(double volume) {
621 if (mAudioTrack != null) {
622 mAudioTrack.setStereoVolume((float) volume, (float) volume);
626 private void resetLastPresentationTimeIfNeeded(long presentationTimeUs) {
628 mLastPresentationTimeUs =
629 Math.max(presentationTimeUs - MAX_PRESENTATION_TIMESTAMP_SHIFT_US, 0);
634 private int getAudioFormat(int channelCount) {
635 switch (channelCount) {
637 return AudioFormat.CHANNEL_OUT_MONO;
639 return AudioFormat.CHANNEL_OUT_STEREO;
641 return AudioFormat.CHANNEL_OUT_QUAD;
643 return AudioFormat.CHANNEL_OUT_5POINT1;
645 return AudioFormat.CHANNEL_OUT_7POINT1;
647 return AudioFormat.CHANNEL_OUT_DEFAULT;