From 48b40d4ffd885b8a9444fa9e1b2140cd4c9b2a30 Mon Sep 17 00:00:00 2001 From: Sejun Park Date: Fri, 19 May 2017 17:20:10 +0900 Subject: [PATCH] Added audioconvert for supporting 16bit pcm when running audio encoder Change-Id: Ifc30775dc202210669112a677b64cb4147e7213e --- include/media_codec_port_gst.h | 3 + packaging/capi-media-codec.spec | 2 +- src/media_codec_ini.c | 3 + src/media_codec_port.c | 2 +- src/media_codec_port_gst.c | 155 ++++++++++++++++++++++++++++++++++++---- test/media_codec_test.c | 12 ++-- 6 files changed, 156 insertions(+), 21 deletions(-) diff --git a/include/media_codec_port_gst.h b/include/media_codec_port_gst.h index 98e93c0..84ff576 100755 --- a/include/media_codec_port_gst.h +++ b/include/media_codec_port_gst.h @@ -98,6 +98,7 @@ struct _mc_gst_core_t { gchar *format; GstElement *pipeline; GstElement *appsrc; + GstElement *audioconvert; GstElement *capsfilter; GstElement *parser; GstElement *fakesink; @@ -147,6 +148,7 @@ struct _mc_gst_core_t { GQueue *output_queue; void *codec_info; + GstBuffer *codec_data; void* user_cb[_MEDIACODEC_EVENT_TYPE_NUM]; void* user_data[_MEDIACODEC_EVENT_TYPE_NUM]; @@ -170,6 +172,7 @@ int __mc_fill_input_buffer_with_adec_packet(mc_gst_core_t *core, media_packet_h int __mc_fill_input_buffer_with_venc_packet(mc_gst_core_t *core, media_packet_h packet, GstMCBuffer *mcbuffer); int __mc_fill_packet_with_output_buffer(mc_gst_core_t *core, void *data, int size, media_packet_h *out_packet); +int __mc_fill_aenc_packet_with_output_buffer(mc_gst_core_t *core, void *data, int size, media_packet_h *out_packet); int __mc_fill_venc_packet_with_output_buffer(mc_gst_core_t *core, void *data, int size, media_packet_h *out_packet); int __mc_fill_vdec_packet_with_output_buffer(mc_gst_core_t *core, void *data, int size, media_packet_h *packet); diff --git a/packaging/capi-media-codec.spec b/packaging/capi-media-codec.spec index 6b69fe1..c58bbf9 100755 --- a/packaging/capi-media-codec.spec +++ b/packaging/capi-media-codec.spec @@ -4,7 +4,7 @@ Name: capi-media-codec Summary: A Media Codec library in Tizen Native API -Version: 0.5.3 +Version: 0.5.4 Release: 0 Group: Multimedia/API License: Apache-2.0 diff --git a/src/media_codec_ini.c b/src/media_codec_ini.c index 1bc2c86..676ae83 100755 --- a/src/media_codec_ini.c +++ b/src/media_codec_ini.c @@ -227,6 +227,9 @@ media_format_mimetype_e _mc_convert_media_format_str_to_int(char *sformat) } else if (!strcmp(sformat, "PCM")) { iformat = MEDIA_FORMAT_PCM; goto endf; + } else if (!strcmp(sformat, "PCM_F32LE")) { + iformat = MEDIA_FORMAT_PCM_F32LE; + goto endf; } else if (!strcmp(sformat, "H261")) { iformat = MEDIA_FORMAT_H261; goto endf; diff --git a/src/media_codec_port.c b/src/media_codec_port.c index f2afd34..48760d6 100755 --- a/src/media_codec_port.c +++ b/src/media_codec_port.c @@ -682,7 +682,7 @@ int _mediacodec_foreach_supported_codec(mediacodec_supported_codec_cb callback, { int ret = MEDIACODEC_NONE; int i; - int index; + int index = 0; gboolean codec[CODEC_NR_ITEMS] = {0,}; diff --git a/src/media_codec_port_gst.c b/src/media_codec_port_gst.c index ae66a6e..9e93383 100755 --- a/src/media_codec_port_gst.c +++ b/src/media_codec_port_gst.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -70,6 +71,8 @@ static gint __gst_handle_resource_error(mc_gst_core_t *core, int code); static gint __gst_handle_library_error(mc_gst_core_t *core, int code); static gint __gst_handle_core_error(mc_gst_core_t *core, int code); static const gchar * _mc_error_to_string(mc_ret_e err); +static const gchar * _mc_bit_to_string(int bit); +static int _mc_get_support_bit_from_format(media_format_mimetype_e format); static int _mc_link_vtable(mc_gst_core_t *core, mediacodec_codec_type_e id, gboolean is_encoder, gboolean is_hw); #ifdef TIZEN_PROFILE_LITE @@ -88,7 +91,7 @@ static void __csc_tiled_to_linear_crop(unsigned char *yuv420_dest, static void _mc_send_eos_signal(mc_gst_core_t *core); static void _mc_wait_for_eos(mc_gst_core_t *core); -static int _mediacodec_get_mime(mc_gst_core_t *core); +static int _mc_get_mime(mc_gst_core_t *core); /* video vtable */ int(*vdec_vtable[])() = {&__mc_fill_input_buffer_with_packet, &__mc_fill_packet_with_output_buffer}; @@ -132,7 +135,7 @@ int(*aenc_vtable[])() = {&__mc_fill_input_buffer_with_packet, &__mc_fill_packet_ int(*adec_vtable[])() = {&__mc_fill_input_buffer_with_packet, &__mc_fill_packet_with_output_buffer}; int(*aenc_aac_vtable[])() = {&__mc_fill_input_buffer_with_packet, /* AAC LC Encoder vtable */ - &__mc_fill_packet_with_output_buffer}; + &__mc_fill_aenc_packet_with_output_buffer}; int(*adec_aac_vtable[])() = {&__mc_fill_input_buffer_with_adec_packet, /* AAC LC Decoder Vtable */ &__mc_fill_packet_with_output_buffer}; @@ -571,6 +574,63 @@ int __mc_fill_venc_packet_with_output_buffer(mc_gst_core_t *core, void *data, in return ret; } +int __mc_fill_aenc_packet_with_output_buffer(mc_gst_core_t *core, void *data, int size, media_packet_h *out_packet) +{ + int ret = MC_ERROR_NONE; + GstPad *pad; + GstCaps *peercaps; + const GValue *codec_data = NULL; + void *packet_data = NULL; + GstMapInfo map; + guint8 *ptr; + gsize len; + + g_return_val_if_fail(core != NULL, MC_PARAM_ERROR); + + if (core->encoder && core->codec_id == MEDIACODEC_AAC) + media_format_set_audio_aac_type(core->output_fmt, FALSE); + + ret = media_packet_create_alloc(core->output_fmt, __mc_output_buffer_finalize_cb, core, out_packet); + if (ret != MEDIA_PACKET_ERROR_NONE) { + LOGW("media_packet_create_alloc failed"); + return MC_ERROR; + } + + media_packet_set_buffer_size(*out_packet, size); + media_packet_get_buffer_data_ptr(*out_packet, &packet_data); + memcpy(packet_data, data, size); + + if (!core->codec_data) { + pad = gst_element_get_static_pad(core->fakesink, "sink"); + peercaps = gst_pad_get_current_caps(pad); + if (peercaps) { + GstStructure *s; + if (gst_caps_is_empty(peercaps)) { + gst_caps_unref(peercaps); + gst_object_unref(pad); + LOGW("Empty caps"); + return ret; + } + + s = gst_caps_get_structure(peercaps, 0); + codec_data = gst_structure_get_value(s, "codec_data"); + core->codec_data = GST_BUFFER(g_value_dup_boxed(codec_data)); + gst_caps_unref(peercaps); + } + gst_object_unref(pad); + LOGD("codec data set"); + } + + if (core->codec_data) { + gst_buffer_map(core->codec_data, &map, GST_MAP_READ); + ptr = map.data; + len = map.size; + media_packet_set_codec_data(*out_packet, ptr, len); + gst_buffer_unmap(core->codec_data, &map); + } + + return ret; +} static GstCaps *__mc_gst_caps_set_buffer_array(GstCaps * caps, const gchar * name, GstBuffer * buf, ...) { @@ -827,10 +887,13 @@ int _mc_output_media_packet_new(mc_gst_core_t *core, bool video, bool encoder, m media_format_set_video_height(core->output_fmt, info->height); media_format_set_video_avg_bps(core->output_fmt, info->bitrate); } else { + int support_bit = 0; + support_bit = _mc_get_support_bit_from_format(out_mime); + media_format_set_audio_mime(core->output_fmt, out_mime); media_format_set_audio_channel(core->output_fmt, info->channel); media_format_set_audio_samplerate(core->output_fmt, info->samplerate); - media_format_set_audio_bit(core->output_fmt, info->bit); + media_format_set_audio_bit(core->output_fmt, support_bit); media_format_set_audio_avg_bps(core->output_fmt, info->bitrate); } } else { @@ -887,6 +950,7 @@ mc_gst_core_t *mc_gst_core_new() core->prepare_count = 0; core->etb_count = 0; core->ftb_count = 0; + core->codec_data = NULL; g_atomic_int_set(&core->available_queue->running, 1); core->available_queue->thread = g_thread_new("feed thread", &feed_task, core); @@ -923,6 +987,9 @@ void mc_gst_core_free(mc_gst_core_t *core) mc_async_queue_free(async_queue->input); g_free(async_queue); + if (core->codec_data) + gst_buffer_unref(core->codec_data); + if (core->ports[1] != NULL) { mc_gst_port_free(core->ports[1]); core->ports[1] = NULL; @@ -1077,6 +1144,7 @@ static int _mc_gst_update_caps(mc_gst_core_t *core, GstMCBuffer *mcbuffer, bool mc_encoder_info_t *enc_info = (mc_encoder_info_t *)core->codec_info; gst_caps_set_simple(core->caps, + "format", G_TYPE_STRING, _mc_bit_to_string(enc_info->bit), "rate", G_TYPE_INT, enc_info->samplerate, "channels", G_TYPE_INT, enc_info->channel, NULL); @@ -1280,7 +1348,6 @@ static int _mc_link_vtable(mc_gst_core_t *core, mediacodec_codec_type_e id, gboo if (encoder) { core->vtable = aenc_aac_vtable; core->caps = gst_caps_new_simple(core->mime, - "format", G_TYPE_STRING, "F32LE", "layout", G_TYPE_STRING, "interleaved", NULL); g_object_set(GST_OBJECT(core->codec), "compliance", -2, NULL); } else { @@ -1903,6 +1970,15 @@ mc_ret_e _mc_gst_create_pipeline(mc_gst_core_t *core, gchar *factory_name) goto ERROR; } + if (!core->video && core->encoder) { + core->audioconvert = gst_element_factory_make("audioconvert", NULL); + + if (!core->audioconvert) { + LOGE("capsfilter can't create"); + goto ERROR; + } + } + core->capsfilter = gst_element_factory_make("capsfilter", NULL); if (!core->capsfilter) { @@ -1919,12 +1995,22 @@ mc_ret_e _mc_gst_create_pipeline(mc_gst_core_t *core, gchar *factory_name) g_object_set(core->fakesink, "enable-last-sample", FALSE, NULL); /*__mc_link_elements(core);*/ - gst_bin_add_many(GST_BIN(core->pipeline), core->appsrc, core->capsfilter, core->codec, core->fakesink, NULL); + if (!core->video && core->encoder) { + gst_bin_add_many(GST_BIN(core->pipeline), core->appsrc, core->capsfilter, core->audioconvert, core->codec, core->fakesink, NULL); - /* link elements */ - if (!(gst_element_link_many(core->appsrc, core->capsfilter, core->codec, core->fakesink, NULL))) { - LOGE("gst_element_link_many is failed"); - goto ERROR; + /* link elements */ + if (!(gst_element_link_many(core->appsrc, core->capsfilter, core->audioconvert, core->codec, core->fakesink, NULL))) { + LOGE("gst_element_link_many is failed"); + goto ERROR; + } + } else { + gst_bin_add_many(GST_BIN(core->pipeline), core->appsrc, core->capsfilter, core->codec, core->fakesink, NULL); + + /* link elements */ + if (!(gst_element_link_many(core->appsrc, core->capsfilter, core->codec, core->fakesink, NULL))) { + LOGE("gst_element_link_many is failed"); + goto ERROR; + } } /* connect signals, bus watcher */ @@ -1971,6 +2057,9 @@ ERROR: if (core->capsfilter) gst_object_unref(GST_OBJECT(core->capsfilter)); + if (core->audioconvert) + gst_object_unref(GST_OBJECT(core->audioconvert)); + if (core->fakesink) gst_object_unref(GST_OBJECT(core->fakesink)); @@ -3185,7 +3274,47 @@ const gchar * _mc_error_to_string(mc_ret_e err) } } -int _mediacodec_get_mime(mc_gst_core_t *core) +const gchar * _mc_bit_to_string(int bit) +{ + switch (bit) { + case 16: + return "S16LE"; + case 32: + return "F32LE"; + default: + return "NULL"; + + } +} + +int _mc_get_support_bit_from_format(media_format_mimetype_e format) +{ + int bit = 0; + + switch (format) { + case MEDIA_FORMAT_PCM_S16LE: + case MEDIA_FORMAT_PCM_S16BE: + case MEDIA_FORMAT_PCM_U16LE: + case MEDIA_FORMAT_PCM_U16BE: + bit = 16; + break; + case MEDIA_FORMAT_PCM_S32LE: + case MEDIA_FORMAT_PCM_S32BE: + case MEDIA_FORMAT_PCM_U32LE: + case MEDIA_FORMAT_PCM_U32BE: + case MEDIA_FORMAT_PCM_F32LE: + case MEDIA_FORMAT_PCM_F32BE: + bit = 32; + break; + default: + LOGE("NOT SUPPORTED!!!!"); + break; + } + + return bit; +} + +int _mc_get_mime(mc_gst_core_t *core) { media_format_mimetype_e mime = MEDIA_FORMAT_MAX; @@ -3212,14 +3341,14 @@ int _mediacodec_get_mime(mc_gst_core_t *core) break; case MEDIACODEC_AAC: if (core->encoder) - mime = MEDIA_FORMAT_PCM; + mime = MEDIA_FORMAT_PCM_F32LE; /*FIXME need to change PCM supported by chipset */ else mime = MEDIA_FORMAT_AAC; break; case MEDIACODEC_AAC_HE: if (core->encoder) - mime = MEDIA_FORMAT_PCM; + mime = MEDIA_FORMAT_PCM_F32LE; /* FIXME need to change PCM supported by chipset */ else mime = MEDIA_FORMAT_AAC_HE; @@ -3281,7 +3410,7 @@ mc_ret_e mc_gst_get_packet_pool(mc_handle_t *mc_handle, media_packet_pool_h *pac return MC_ERROR; } - mime_format = _mediacodec_get_mime(core); + mime_format = _mc_get_mime(core); if (core->video) { if (core->encoder) { diff --git a/test/media_codec_test.c b/test/media_codec_test.c index 1f6096c..67f0f68 100755 --- a/test/media_codec_test.c +++ b/test/media_codec_test.c @@ -873,7 +873,7 @@ int _mediacodec_set_codec(App *app, int codecid, int flag, bool *hardware) case MEDIACODEC_AAC: if (encoder) { extractor = aacenc_extractor; - mime = MEDIA_FORMAT_PCM; + mime = MEDIA_FORMAT_PCM_F32LE; /* FIXME need to check according to verdor */ } else { extractor = aacdec_extractor; mime = MEDIA_FORMAT_AAC; @@ -882,7 +882,7 @@ int _mediacodec_set_codec(App *app, int codecid, int flag, bool *hardware) case MEDIACODEC_AAC_HE: if (encoder) { extractor = aacenc_extractor; - mime = MEDIA_FORMAT_PCM; + mime = MEDIA_FORMAT_PCM_F32LE; /* FIXME need to check according to verdor */ } else { extractor = extract_input_aacdec_m4a_test; mime = MEDIA_FORMAT_AAC_HE; @@ -909,7 +909,7 @@ int _mediacodec_set_codec(App *app, int codecid, int flag, bool *hardware) case MEDIACODEC_AMR_NB: if (encoder) { extractor = amrenc_extractor; - mime = MEDIA_FORMAT_PCM; + mime = MEDIA_FORMAT_PCM_F32LE; /* FIXME need to check according to verdor */ app->is_amr_nb = TRUE; } else { extractor = amrdec_extractor; @@ -919,7 +919,7 @@ int _mediacodec_set_codec(App *app, int codecid, int flag, bool *hardware) case MEDIACODEC_AMR_WB: if (encoder) { extractor = amrenc_extractor; - mime = MEDIA_FORMAT_PCM; + mime = MEDIA_FORMAT_PCM_F32LE; /* FIXME need to check according to verdor */ app->is_amr_nb = FALSE; } else { extractor = amrdec_extractor; @@ -1244,10 +1244,10 @@ static bool _mediacodec_eos_cb(void *user_data) return TRUE; } -void _foreach_cb(mediacodec_codec_type_e codec_type, void *user_data) +gboolean _foreach_cb(mediacodec_codec_type_e codec_type, void *user_data) { g_print("codec type : %x %s\n", codec_type, codec_type_to_string(codec_type)); - return; + return TRUE; } static void _mediacodec_prepare(App *app, bool frame_all) -- 2.7.4