1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/base/android/media_codec_bridge.h"
10 #include "base/android/build_info.h"
11 #include "base/android/jni_android.h"
12 #include "base/android/jni_array.h"
13 #include "base/android/jni_string.h"
14 #include "base/basictypes.h"
15 #include "base/lazy_instance.h"
16 #include "base/logging.h"
17 #include "base/numerics/safe_conversions.h"
18 #include "base/strings/string_util.h"
19 #include "base/strings/stringprintf.h"
20 #include "jni/MediaCodecBridge_jni.h"
21 #include "media/base/bit_reader.h"
22 #include "media/base/decrypt_config.h"
24 using base::android::AttachCurrentThread;
25 using base::android::ConvertJavaStringToUTF8;
26 using base::android::ConvertUTF8ToJavaString;
27 using base::android::ScopedJavaLocalRef;
32 kBufferFlagSyncFrame = 1, // BUFFER_FLAG_SYNC_FRAME
33 kBufferFlagEndOfStream = 4, // BUFFER_FLAG_END_OF_STREAM
34 kConfigureFlagEncode = 1, // CONFIGURE_FLAG_ENCODE
37 static const std::string AudioCodecToAndroidMimeType(const AudioCodec& codec) {
42 return "audio/vorbis";
44 return "audio/mp4a-latm";
50 static const std::string VideoCodecToAndroidMimeType(const VideoCodec& codec) {
55 return "video/x-vnd.on2.vp8";
57 return "video/x-vnd.on2.vp9";
63 static const std::string CodecTypeToAndroidMimeType(const std::string& codec) {
64 // TODO(xhwang): Shall we handle more detailed strings like "mp4a.40.2"?
68 return "audio/mp4a-latm";
69 if (codec == "vp8" || codec == "vp8.0")
70 return "video/x-vnd.on2.vp8";
71 if (codec == "vp9" || codec == "vp9.0")
72 return "video/x-vnd.on2.vp9";
73 if (codec == "vorbis")
74 return "audio/vorbis";
78 // TODO(qinmin): using a map to help all the conversions in this class.
79 static const std::string AndroidMimeTypeToCodecType(const std::string& mime) {
80 if (mime == "video/mp4v-es")
82 if (mime == "video/avc")
84 if (mime == "video/x-vnd.on2.vp8")
86 if (mime == "video/x-vnd.on2.vp9")
88 if (mime == "audio/mp4a-latm")
90 if (mime == "audio/mpeg")
92 if (mime == "audio/vorbis")
97 static ScopedJavaLocalRef<jintArray>
98 ToJavaIntArray(JNIEnv* env, scoped_ptr<jint[]> native_array, int size) {
99 ScopedJavaLocalRef<jintArray> j_array(env, env->NewIntArray(size));
100 env->SetIntArrayRegion(j_array.obj(), 0, size, native_array.get());
105 bool MediaCodecBridge::IsAvailable() {
106 // MediaCodec is only available on JB and greater.
107 return base::android::BuildInfo::GetInstance()->sdk_int() >= 16;
111 bool MediaCodecBridge::SupportsSetParameters() {
112 // MediaCodec.setParameters() is only available starting with K.
113 return base::android::BuildInfo::GetInstance()->sdk_int() >= 19;
117 std::vector<MediaCodecBridge::CodecsInfo> MediaCodecBridge::GetCodecsInfo() {
118 std::vector<CodecsInfo> codecs_info;
122 JNIEnv* env = AttachCurrentThread();
123 std::string mime_type;
124 ScopedJavaLocalRef<jobjectArray> j_codec_info_array =
125 Java_MediaCodecBridge_getCodecsInfo(env);
126 jsize len = env->GetArrayLength(j_codec_info_array.obj());
127 for (jsize i = 0; i < len; ++i) {
128 ScopedJavaLocalRef<jobject> j_info(
129 env, env->GetObjectArrayElement(j_codec_info_array.obj(), i));
130 ScopedJavaLocalRef<jstring> j_codec_type =
131 Java_CodecInfo_codecType(env, j_info.obj());
132 ConvertJavaStringToUTF8(env, j_codec_type.obj(), &mime_type);
133 ScopedJavaLocalRef<jstring> j_codec_name =
134 Java_CodecInfo_codecName(env, j_info.obj());
136 info.codecs = AndroidMimeTypeToCodecType(mime_type);
137 ConvertJavaStringToUTF8(env, j_codec_name.obj(), &info.name);
138 info.direction = static_cast<MediaCodecDirection>(
139 Java_CodecInfo_direction(env, j_info.obj()));
140 codecs_info.push_back(info);
146 bool MediaCodecBridge::CanDecode(const std::string& codec, bool is_secure) {
150 JNIEnv* env = AttachCurrentThread();
151 std::string mime = CodecTypeToAndroidMimeType(codec);
154 ScopedJavaLocalRef<jstring> j_mime = ConvertUTF8ToJavaString(env, mime);
155 ScopedJavaLocalRef<jobject> j_media_codec_bridge =
156 Java_MediaCodecBridge_create(env, j_mime.obj(), is_secure, false);
157 if (!j_media_codec_bridge.is_null()) {
158 Java_MediaCodecBridge_release(env, j_media_codec_bridge.obj());
165 bool MediaCodecBridge::IsKnownUnaccelerated(const std::string& mime_type,
166 MediaCodecDirection direction) {
170 std::string codec_type = AndroidMimeTypeToCodecType(mime_type);
171 std::vector<media::MediaCodecBridge::CodecsInfo> codecs_info =
172 MediaCodecBridge::GetCodecsInfo();
173 for (size_t i = 0; i < codecs_info.size(); ++i) {
174 if (codecs_info[i].codecs == codec_type &&
175 codecs_info[i].direction == direction) {
176 // It would be nice if MediaCodecInfo externalized some notion of
177 // HW-acceleration but it doesn't. Android Media guidance is that the
178 // prefix below is always used for SW decoders, so that's what we use.
179 return StartsWithASCII(codecs_info[i].name, "OMX.google.", true);
185 MediaCodecBridge::MediaCodecBridge(const std::string& mime,
187 MediaCodecDirection direction) {
188 JNIEnv* env = AttachCurrentThread();
190 DCHECK(!mime.empty());
191 ScopedJavaLocalRef<jstring> j_mime = ConvertUTF8ToJavaString(env, mime);
192 j_media_codec_.Reset(
193 Java_MediaCodecBridge_create(env, j_mime.obj(), is_secure, direction));
196 MediaCodecBridge::~MediaCodecBridge() {
197 JNIEnv* env = AttachCurrentThread();
199 if (j_media_codec_.obj())
200 Java_MediaCodecBridge_release(env, j_media_codec_.obj());
203 bool MediaCodecBridge::StartInternal() {
204 JNIEnv* env = AttachCurrentThread();
205 return Java_MediaCodecBridge_start(env, j_media_codec_.obj()) &&
209 MediaCodecStatus MediaCodecBridge::Reset() {
210 JNIEnv* env = AttachCurrentThread();
211 return static_cast<MediaCodecStatus>(
212 Java_MediaCodecBridge_flush(env, j_media_codec_.obj()));
215 void MediaCodecBridge::Stop() {
216 JNIEnv* env = AttachCurrentThread();
217 Java_MediaCodecBridge_stop(env, j_media_codec_.obj());
220 void MediaCodecBridge::GetOutputFormat(int* width, int* height) {
221 JNIEnv* env = AttachCurrentThread();
223 *width = Java_MediaCodecBridge_getOutputWidth(env, j_media_codec_.obj());
224 *height = Java_MediaCodecBridge_getOutputHeight(env, j_media_codec_.obj());
227 MediaCodecStatus MediaCodecBridge::QueueInputBuffer(
231 const base::TimeDelta& presentation_time) {
232 DVLOG(3) << __PRETTY_FUNCTION__ << index << ": " << data_size;
233 if (data_size > base::checked_cast<size_t>(kint32max))
234 return MEDIA_CODEC_ERROR;
235 if (data && !FillInputBuffer(index, data, data_size))
236 return MEDIA_CODEC_ERROR;
237 JNIEnv* env = AttachCurrentThread();
238 return static_cast<MediaCodecStatus>(
239 Java_MediaCodecBridge_queueInputBuffer(env,
240 j_media_codec_.obj(),
244 presentation_time.InMicroseconds(),
248 MediaCodecStatus MediaCodecBridge::QueueSecureInputBuffer(
256 const SubsampleEntry* subsamples,
258 const base::TimeDelta& presentation_time) {
259 DVLOG(3) << __PRETTY_FUNCTION__ << index << ": " << data_size;
260 if (data_size > base::checked_cast<size_t>(kint32max))
261 return MEDIA_CODEC_ERROR;
262 if (data && !FillInputBuffer(index, data, data_size))
263 return MEDIA_CODEC_ERROR;
265 JNIEnv* env = AttachCurrentThread();
266 ScopedJavaLocalRef<jbyteArray> j_key_id =
267 base::android::ToJavaByteArray(env, key_id, key_id_size);
268 ScopedJavaLocalRef<jbyteArray> j_iv =
269 base::android::ToJavaByteArray(env, iv, iv_size);
271 // MediaCodec.CryptoInfo documentations says passing NULL for |clear_array|
272 // to indicate that all data is encrypted. But it doesn't specify what
273 // |cypher_array| and |subsamples_size| should be in that case. Passing
274 // one subsample here just to be on the safe side.
275 int new_subsamples_size = subsamples_size == 0 ? 1 : subsamples_size;
277 scoped_ptr<jint[]> native_clear_array(new jint[new_subsamples_size]);
278 scoped_ptr<jint[]> native_cypher_array(new jint[new_subsamples_size]);
280 if (subsamples_size == 0) {
282 native_clear_array[0] = 0;
283 native_cypher_array[0] = data_size;
285 DCHECK_GT(subsamples_size, 0);
287 for (int i = 0; i < subsamples_size; ++i) {
288 DCHECK(subsamples[i].clear_bytes <= std::numeric_limits<uint16>::max());
289 if (subsamples[i].cypher_bytes >
290 static_cast<uint32>(std::numeric_limits<jint>::max())) {
291 return MEDIA_CODEC_ERROR;
294 native_clear_array[i] = subsamples[i].clear_bytes;
295 native_cypher_array[i] = subsamples[i].cypher_bytes;
299 ScopedJavaLocalRef<jintArray> clear_array =
300 ToJavaIntArray(env, native_clear_array.Pass(), new_subsamples_size);
301 ScopedJavaLocalRef<jintArray> cypher_array =
302 ToJavaIntArray(env, native_cypher_array.Pass(), new_subsamples_size);
304 return static_cast<MediaCodecStatus>(
305 Java_MediaCodecBridge_queueSecureInputBuffer(
307 j_media_codec_.obj(),
315 presentation_time.InMicroseconds()));
318 void MediaCodecBridge::QueueEOS(int input_buffer_index) {
319 DVLOG(3) << __PRETTY_FUNCTION__ << ": " << input_buffer_index;
320 JNIEnv* env = AttachCurrentThread();
321 Java_MediaCodecBridge_queueInputBuffer(env,
322 j_media_codec_.obj(),
327 kBufferFlagEndOfStream);
330 MediaCodecStatus MediaCodecBridge::DequeueInputBuffer(
331 const base::TimeDelta& timeout,
333 JNIEnv* env = AttachCurrentThread();
334 ScopedJavaLocalRef<jobject> result = Java_MediaCodecBridge_dequeueInputBuffer(
335 env, j_media_codec_.obj(), timeout.InMicroseconds());
336 *index = Java_DequeueInputResult_index(env, result.obj());
337 MediaCodecStatus status = static_cast<MediaCodecStatus>(
338 Java_DequeueInputResult_status(env, result.obj()));
339 DVLOG(3) << __PRETTY_FUNCTION__ << ": status: " << status
340 << ", index: " << *index;
344 MediaCodecStatus MediaCodecBridge::DequeueOutputBuffer(
345 const base::TimeDelta& timeout,
349 base::TimeDelta* presentation_time,
352 JNIEnv* env = AttachCurrentThread();
353 ScopedJavaLocalRef<jobject> result =
354 Java_MediaCodecBridge_dequeueOutputBuffer(
355 env, j_media_codec_.obj(), timeout.InMicroseconds());
356 *index = Java_DequeueOutputResult_index(env, result.obj());
357 *offset = base::checked_cast<size_t>(
358 Java_DequeueOutputResult_offset(env, result.obj()));
359 *size = base::checked_cast<size_t>(
360 Java_DequeueOutputResult_numBytes(env, result.obj()));
361 if (presentation_time) {
362 *presentation_time = base::TimeDelta::FromMicroseconds(
363 Java_DequeueOutputResult_presentationTimeMicroseconds(env,
366 int flags = Java_DequeueOutputResult_flags(env, result.obj());
368 *end_of_stream = flags & kBufferFlagEndOfStream;
370 *key_frame = flags & kBufferFlagSyncFrame;
371 MediaCodecStatus status = static_cast<MediaCodecStatus>(
372 Java_DequeueOutputResult_status(env, result.obj()));
373 DVLOG(3) << __PRETTY_FUNCTION__ << ": status: " << status
374 << ", index: " << *index << ", offset: " << *offset
375 << ", size: " << *size << ", flags: " << flags;
379 void MediaCodecBridge::ReleaseOutputBuffer(int index, bool render) {
380 DVLOG(3) << __PRETTY_FUNCTION__ << ": " << index;
381 JNIEnv* env = AttachCurrentThread();
384 Java_MediaCodecBridge_releaseOutputBuffer(
385 env, j_media_codec_.obj(), index, render);
388 int MediaCodecBridge::GetInputBuffersCount() {
389 JNIEnv* env = AttachCurrentThread();
390 return Java_MediaCodecBridge_getInputBuffersCount(env, j_media_codec_.obj());
393 int MediaCodecBridge::GetOutputBuffersCount() {
394 JNIEnv* env = AttachCurrentThread();
395 return Java_MediaCodecBridge_getOutputBuffersCount(env, j_media_codec_.obj());
398 size_t MediaCodecBridge::GetOutputBuffersCapacity() {
399 JNIEnv* env = AttachCurrentThread();
400 return Java_MediaCodecBridge_getOutputBuffersCapacity(env,
401 j_media_codec_.obj());
404 bool MediaCodecBridge::GetOutputBuffers() {
405 JNIEnv* env = AttachCurrentThread();
406 return Java_MediaCodecBridge_getOutputBuffers(env, j_media_codec_.obj());
409 void MediaCodecBridge::GetInputBuffer(int input_buffer_index,
412 JNIEnv* env = AttachCurrentThread();
413 ScopedJavaLocalRef<jobject> j_buffer(Java_MediaCodecBridge_getInputBuffer(
414 env, j_media_codec_.obj(), input_buffer_index));
415 *data = static_cast<uint8*>(env->GetDirectBufferAddress(j_buffer.obj()));
416 *capacity = base::checked_cast<size_t>(
417 env->GetDirectBufferCapacity(j_buffer.obj()));
420 bool MediaCodecBridge::CopyFromOutputBuffer(int index,
424 JNIEnv* env = AttachCurrentThread();
425 ScopedJavaLocalRef<jobject> j_buffer(
426 Java_MediaCodecBridge_getOutputBuffer(env, j_media_codec_.obj(), index));
428 reinterpret_cast<uint8*>(env->GetDirectBufferAddress(j_buffer.obj())) +
430 int src_capacity = env->GetDirectBufferCapacity(j_buffer.obj()) - offset;
431 if (src_capacity < dst_size)
433 memcpy(dst, src_data, dst_size);
437 bool MediaCodecBridge::FillInputBuffer(int index,
442 GetInputBuffer(index, &dst, &capacity);
445 if (size > capacity) {
446 LOG(ERROR) << "Input buffer size " << size
447 << " exceeds MediaCodec input buffer capacity: " << capacity;
451 memcpy(dst, data, size);
455 AudioCodecBridge::AudioCodecBridge(const std::string& mime)
456 // Audio codec doesn't care about security level and there is no need for
457 // audio encoding yet.
458 : MediaCodecBridge(mime, false, MEDIA_CODEC_DECODER) {}
460 bool AudioCodecBridge::Start(const AudioCodec& codec,
463 const uint8* extra_data,
464 size_t extra_data_size,
466 jobject media_crypto) {
467 JNIEnv* env = AttachCurrentThread();
472 std::string codec_string = AudioCodecToAndroidMimeType(codec);
473 if (codec_string.empty())
476 ScopedJavaLocalRef<jstring> j_mime =
477 ConvertUTF8ToJavaString(env, codec_string);
478 ScopedJavaLocalRef<jobject> j_format(Java_MediaCodecBridge_createAudioFormat(
479 env, j_mime.obj(), sample_rate, channel_count));
480 DCHECK(!j_format.is_null());
482 if (!ConfigureMediaFormat(j_format.obj(), codec, extra_data, extra_data_size))
485 if (!Java_MediaCodecBridge_configureAudio(
486 env, media_codec(), j_format.obj(), media_crypto, 0, play_audio)) {
490 return StartInternal();
493 bool AudioCodecBridge::ConfigureMediaFormat(jobject j_format,
494 const AudioCodec& codec,
495 const uint8* extra_data,
496 size_t extra_data_size) {
497 if (extra_data_size == 0)
500 JNIEnv* env = AttachCurrentThread();
503 if (extra_data[0] != 2) {
504 LOG(ERROR) << "Invalid number of vorbis headers before the codec "
505 << "header: " << extra_data[0];
509 size_t header_length[2];
510 // |total_length| keeps track of the total number of bytes before the last
512 size_t total_length = 1;
513 const uint8* current_pos = extra_data;
514 // Calculate the length of the first 2 headers.
515 for (int i = 0; i < 2; ++i) {
516 header_length[i] = 0;
517 while (total_length < extra_data_size) {
518 size_t size = *(++current_pos);
519 total_length += 1 + size;
520 if (total_length > 0x80000000) {
521 LOG(ERROR) << "Vorbis header size too large";
524 header_length[i] += size;
528 if (total_length >= extra_data_size) {
529 LOG(ERROR) << "Invalid vorbis header size in the extra data";
534 // The first header is identification header.
535 ScopedJavaLocalRef<jbyteArray> first_header =
536 base::android::ToJavaByteArray(env, current_pos, header_length[0]);
537 Java_MediaCodecBridge_setCodecSpecificData(
538 env, j_format, 0, first_header.obj());
539 // The last header is codec header.
540 ScopedJavaLocalRef<jbyteArray> last_header =
541 base::android::ToJavaByteArray(
542 env, extra_data + total_length, extra_data_size - total_length);
543 Java_MediaCodecBridge_setCodecSpecificData(
544 env, j_format, 1, last_header.obj());
548 media::BitReader reader(extra_data, extra_data_size);
550 // The following code is copied from aac.cc
551 // TODO(qinmin): refactor the code in aac.cc to make it more reusable.
553 uint8 frequency_index = 0;
554 uint8 channel_config = 0;
555 if (!reader.ReadBits(5, &profile) ||
556 !reader.ReadBits(4, &frequency_index)) {
557 LOG(ERROR) << "Unable to parse AAC header";
560 if (0xf == frequency_index && !reader.SkipBits(24)) {
561 LOG(ERROR) << "Unable to parse AAC header";
564 if (!reader.ReadBits(4, &channel_config)) {
565 LOG(ERROR) << "Unable to parse AAC header";
569 if (profile < 1 || profile > 4 || frequency_index == 0xf ||
570 channel_config > 7) {
571 LOG(ERROR) << "Invalid AAC header";
574 const size_t kCsdLength = 2;
575 uint8 csd[kCsdLength];
576 csd[0] = profile << 3 | frequency_index >> 1;
577 csd[1] = (frequency_index & 0x01) << 7 | channel_config << 3;
578 ScopedJavaLocalRef<jbyteArray> byte_array =
579 base::android::ToJavaByteArray(env, csd, kCsdLength);
580 Java_MediaCodecBridge_setCodecSpecificData(
581 env, j_format, 0, byte_array.obj());
583 // TODO(qinmin): pass an extra variable to this function to determine
584 // whether we need to call this.
585 Java_MediaCodecBridge_setFrameHasADTSHeader(env, j_format);
589 LOG(ERROR) << "Invalid header encountered for codec: "
590 << AudioCodecToAndroidMimeType(codec);
596 int64 AudioCodecBridge::PlayOutputBuffer(int index, size_t size) {
598 int numBytes = base::checked_cast<int>(size);
599 JNIEnv* env = AttachCurrentThread();
600 ScopedJavaLocalRef<jobject> buf =
601 Java_MediaCodecBridge_getOutputBuffer(env, media_codec(), index);
602 uint8* buffer = static_cast<uint8*>(env->GetDirectBufferAddress(buf.obj()));
604 ScopedJavaLocalRef<jbyteArray> byte_array =
605 base::android::ToJavaByteArray(env, buffer, numBytes);
606 return Java_MediaCodecBridge_playOutputBuffer(
607 env, media_codec(), byte_array.obj());
610 void AudioCodecBridge::SetVolume(double volume) {
611 JNIEnv* env = AttachCurrentThread();
612 Java_MediaCodecBridge_setVolume(env, media_codec(), volume);
616 AudioCodecBridge* AudioCodecBridge::Create(const AudioCodec& codec) {
617 if (!MediaCodecBridge::IsAvailable())
620 const std::string mime = AudioCodecToAndroidMimeType(codec);
621 return mime.empty() ? NULL : new AudioCodecBridge(mime);
625 bool AudioCodecBridge::IsKnownUnaccelerated(const AudioCodec& codec) {
626 return MediaCodecBridge::IsKnownUnaccelerated(
627 AudioCodecToAndroidMimeType(codec), MEDIA_CODEC_DECODER);
631 bool VideoCodecBridge::IsKnownUnaccelerated(const VideoCodec& codec,
632 MediaCodecDirection direction) {
633 return MediaCodecBridge::IsKnownUnaccelerated(
634 VideoCodecToAndroidMimeType(codec), direction);
638 VideoCodecBridge* VideoCodecBridge::CreateDecoder(const VideoCodec& codec,
640 const gfx::Size& size,
642 jobject media_crypto) {
643 if (!MediaCodecBridge::IsAvailable())
646 const std::string mime = VideoCodecToAndroidMimeType(codec);
650 scoped_ptr<VideoCodecBridge> bridge(
651 new VideoCodecBridge(mime, is_secure, MEDIA_CODEC_DECODER));
652 if (!bridge->media_codec())
655 JNIEnv* env = AttachCurrentThread();
656 ScopedJavaLocalRef<jstring> j_mime = ConvertUTF8ToJavaString(env, mime);
657 ScopedJavaLocalRef<jobject> j_format(
658 Java_MediaCodecBridge_createVideoDecoderFormat(
659 env, j_mime.obj(), size.width(), size.height()));
660 DCHECK(!j_format.is_null());
661 if (!Java_MediaCodecBridge_configureVideo(env,
662 bridge->media_codec(),
670 return bridge->StartInternal() ? bridge.release() : NULL;
674 VideoCodecBridge* VideoCodecBridge::CreateEncoder(const VideoCodec& codec,
675 const gfx::Size& size,
678 int i_frame_interval,
680 if (!MediaCodecBridge::IsAvailable())
683 const std::string mime = VideoCodecToAndroidMimeType(codec);
687 scoped_ptr<VideoCodecBridge> bridge(
688 new VideoCodecBridge(mime, false, MEDIA_CODEC_ENCODER));
689 if (!bridge->media_codec())
692 JNIEnv* env = AttachCurrentThread();
693 ScopedJavaLocalRef<jstring> j_mime = ConvertUTF8ToJavaString(env, mime);
694 ScopedJavaLocalRef<jobject> j_format(
695 Java_MediaCodecBridge_createVideoEncoderFormat(env,
703 DCHECK(!j_format.is_null());
704 if (!Java_MediaCodecBridge_configureVideo(env,
705 bridge->media_codec(),
709 kConfigureFlagEncode)) {
713 return bridge->StartInternal() ? bridge.release() : NULL;
716 VideoCodecBridge::VideoCodecBridge(const std::string& mime,
718 MediaCodecDirection direction)
719 : MediaCodecBridge(mime, is_secure, direction) {}
721 void VideoCodecBridge::SetVideoBitrate(int bps) {
722 JNIEnv* env = AttachCurrentThread();
723 Java_MediaCodecBridge_setVideoBitrate(env, media_codec(), bps);
726 void VideoCodecBridge::RequestKeyFrameSoon() {
727 JNIEnv* env = AttachCurrentThread();
728 Java_MediaCodecBridge_requestKeyFrameSoon(env, media_codec());
731 bool MediaCodecBridge::RegisterMediaCodecBridge(JNIEnv* env) {
732 return RegisterNativesImpl(env);