src/media_codec_sync_internal.c

   1 /*
   2  * Copyright (c) 2019 Samsung Electronics Co., Ltd All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <inttypes.h>
  18 #include <dlog.h>
  19 #include <media_codec_sync_private.h>
  20
  21 #ifdef LOG_TAG
  22 #undef LOG_TAG
  23 #endif
  24 #define LOG_TAG "TIZEN_N_MEDIACODECSYNC"
  25
  26 #define PIPELINE_NAME_AUDIO "MEDIACODECSYNC_AUDIO"
  27 #define PIPELINE_NAME_VIDEO "MEDIACODECSYNC_VIDEO"
  28
  29 #define _MEDIACODECSYNC_PIPELINE_MAKE(handle, pipeline, name) \
  30         do { \
  31                 if (pipeline) { \
  32                         gst_object_unref(pipeline); \
  33                         pipeline = NULL; \
  34                 } \
  35                 pipeline = gst_pipeline_new(name); \
  36                 if (!pipeline) { \
  37                         LOGE("create pipeline[%s] failed", name); \
  38                         goto _CREATE_PIPELINE_FAILED; \
  39                 } \
  40                 g_object_weak_ref(G_OBJECT(pipeline), (GWeakNotify)__mediacodecsync_element_release_noti, handle); \
  41                 LOGD("create pipeline [%s] done", name); \
  42         } while (0)
  43
  44 #define _MEDIACODECSYNC_ELEMENT_MAKE(handle, element, name, prefix, nick) \
  45         do { \
  46                 char nick_name[24]; \
  47                 if (element) { \
  48                         gst_object_unref(element); \
  49                         element = NULL; \
  50                 } \
  51                 snprintf(nick_name, sizeof(nick_name), "%s%s", prefix, nick); \
  52                 element = gst_element_factory_make(name, nick_name); \
  53                 if (!element) { \
  54                         LOGE("create element[%s,%s] failed", name, nick_name); \
  55                         goto _CREATE_PIPELINE_FAILED; \
  56                 } \
  57                 g_object_weak_ref(G_OBJECT(element), (GWeakNotify)__mediacodecsync_element_release_noti, handle); \
  58                 LOGD("create element [%s,%s] done", name, nick_name); \
  59         } while (0)
  60
  61 #define _MEDIACODECSYNC_ELEMENT_REMOVE(element) \
  62         do { \
  63                 if (element != NULL) \
  64                         gst_object_unref(element); \
  65         } while (0)
  66
  67
  68
  69 static void __mediacodecsync_need_audio_data_cb(GstElement *appsrc, guint size, gpointer data)
  70 {
  71         /*mediacodecsync_s *handle = (mediacodecsync_s *)data;*/
  72
  73         LOGI("audio data is needed");
  74 }
  75
  76
  77 static void __mediacodecsync_enough_audio_data_cb(GstElement *appsrc, gpointer data)
  78 {
  79         /*mediacodecsync_s *handle = (mediacodecsync_s *)data;*/
  80
  81         LOGI("audio data is enough");
  82 }
  83
  84
  85 static void __mediacodecsync_need_video_data_cb(GstElement *appsrc, guint size, gpointer data)
  86 {
  87         /*mediacodecsync_s *handle = (mediacodecsync_s *)data;*/
  88
  89         LOGI("video data is needed");
  90 }
  91
  92
  93 static void __mediacodecsync_enough_video_data_cb(GstElement *appsrc, gpointer data)
  94 {
  95         /*mediacodecsync_s *handle = (mediacodecsync_s *)data;*/
  96
  97         LOGI("video data is enough");
  98 }
  99
 100
 101 static int _mediacodecsync_get_packet_info(media_packet_h packet, packet_info *info)
 102 {
 103         int ret = MEDIACODEC_ERROR_NONE;
 104         bool has_tbm_surface = false;
 105         void *data = NULL;
 106         uint64_t data_size = 0;
 107         uint64_t dts = 0;
 108         uint64_t pts = 0;
 109         uint64_t duration = 0;
 110
 111         if (!packet || !info) {
 112                 LOGE("NULL params %p %p", packet, info);
 113                 return MEDIACODEC_ERROR_INVALID_PARAMETER;
 114         }
 115
 116         /* get packet info */
 117         ret = media_packet_has_tbm_surface_buffer(packet, &has_tbm_surface);
 118         ret |= media_packet_get_buffer_data_ptr(packet, &data);
 119         ret |= media_packet_get_buffer_size(packet, &data_size);
 120         ret |= media_packet_get_dts(packet, &dts);
 121         ret |= media_packet_get_pts(packet, &pts);
 122         ret |= media_packet_get_duration(packet, &duration);
 123
 124         if (ret == MEDIACODEC_ERROR_NONE) {
 125                 info->has_tbm_surface = has_tbm_surface;
 126                 info->data = data;
 127                 info->data_size = data_size;
 128                 info->dts = dts;
 129                 info->pts = pts;
 130                 info->duration = duration;
 131
 132                 LOGI("packet info[%u], %p, size %"PRIu64", dts %"PRIu64", pts %"PRIu64", duration %"PRIu64,
 133                         has_tbm_surface, data, data_size, dts, pts, duration);
 134         }
 135
 136         return ret;
 137 }
 138
 139
 140 static void _mediacodecsync_gst_buffer_finalize(GstMCSBuffer *mcs_buffer)
 141 {
 142         mediacodecsync_s *handle = NULL;
 143
 144         if (!mcs_buffer || !mcs_buffer->handle) {
 145                 LOGE("NULL buffer(%p) or handle", mcs_buffer);
 146                 return;
 147         }
 148
 149         handle = mcs_buffer->handle;
 150
 151         LOGI("[type %d] mcs_buffer %p, gst buffer %p, packet %p",
 152                 mcs_buffer->type, mcs_buffer, mcs_buffer->gst_buffer, mcs_buffer->packet);
 153
 154         if (mcs_buffer->packet) {
 155                 if (handle->buffer_used_cb)
 156                         handle->buffer_used_cb(mcs_buffer->packet, mcs_buffer->type, handle->buffer_used_cb_userdata);
 157                 else
 158                         LOGE("No buffer used callback");
 159         }
 160
 161         free(mcs_buffer);
 162 }
 163
 164
 165 static GstMCSBuffer *_mediacodecsync_gst_buffer_new(mediacodecsync_s *handle, media_packet_h packet, int type)
 166 {
 167         GstMCSBuffer *new_buffer = NULL;
 168         GstMemory *memory = NULL;
 169         tbm_surface_h tbm_surface = NULL;
 170         packet_info info;
 171
 172         if (!handle || !packet) {
 173                 LOGE("invalid params %p, %p", handle, packet);
 174                 return NULL;
 175         }
 176
 177         /* get packet info */
 178         if (_mediacodecsync_get_packet_info(packet, &info) != MEDIA_PACKET_ERROR_NONE) {
 179                 LOGE("_mediacodecsync_get_packet_info failed");
 180                 return NULL;
 181         }
 182
 183         new_buffer = (GstMCSBuffer *)calloc(1, sizeof(GstMCSBuffer));
 184         if (!new_buffer) {
 185                 LOGE("GstMCSBuffer alloc failed");
 186                 return NULL;
 187         }
 188
 189         new_buffer->gst_buffer = gst_buffer_new();
 190         new_buffer->type = type;
 191         new_buffer->packet = packet;
 192         new_buffer->handle = handle;
 193
 194         if (info.has_tbm_surface) {
 195                 /* get tbm surface from packet */
 196                 if (media_packet_get_tbm_surface(packet, &tbm_surface) != MEDIA_PACKET_ERROR_NONE) {
 197                         LOGE("get tbm surface failed");
 198                         goto _BUFFER_NEW_FAILED;
 199                 }
 200
 201                 /* create tizen memory for gst buffer with tbm surface */
 202                 memory = gst_tizen_allocator_alloc_surface(handle->allocator,
 203                         &handle->video_info, tbm_surface, (gpointer)new_buffer,
 204                         (GDestroyNotify)_mediacodecsync_gst_buffer_finalize);
 205         } else {
 206                 /* If tbm is not used, the data from packet will be used. */
 207                 memory = gst_memory_new_wrapped(0, info.data, (gsize)info.data_size,
 208                         0, (gsize)info.data_size, (gpointer)new_buffer,
 209                         (GDestroyNotify)_mediacodecsync_gst_buffer_finalize);
 210         }
 211
 212         if (!memory) {
 213                 LOGE("GstMemory failed");
 214                 goto _BUFFER_NEW_FAILED;
 215         }
 216
 217         gst_buffer_append_memory(new_buffer->gst_buffer, memory);
 218
 219         /* set buffer meta */
 220         gst_buffer_set_size(new_buffer->gst_buffer, info.data_size);
 221         GST_BUFFER_DTS(new_buffer->gst_buffer) = info.dts;
 222         GST_BUFFER_PTS(new_buffer->gst_buffer) = info.pts;
 223         GST_BUFFER_DURATION(new_buffer->gst_buffer) = info.duration;
 224         GST_BUFFER_OFFSET(new_buffer->gst_buffer) = 0;
 225         GST_BUFFER_OFFSET_END(new_buffer->gst_buffer) = info.data_size;
 226
 227         return new_buffer;
 228
 229 _BUFFER_NEW_FAILED:
 230         gst_buffer_unref(new_buffer->gst_buffer);
 231         free(new_buffer);
 232         return NULL;
 233 }
 234
 235 static void __mediacodecsync_element_release_noti(gpointer data, GObject *obj)
 236 {
 237         int i = 0;
 238         mediacodecsync_s *handle = (mediacodecsync_s *)data;
 239
 240         if (!handle) {
 241                 LOGE("NULL handle");
 242                 return;
 243         }
 244
 245         for (i = 0 ; i < MEDIACODECSYNC_ELEMENT_NUM ; i++) {
 246                 if (handle->audio_pipe[i] && G_OBJECT(handle->audio_pipe[i]) == obj) {
 247                         LOGD("audio element[%d] was released", i);
 248                         handle->audio_pipe[i] = NULL;
 249                         return;
 250                 }
 251         }
 252
 253         for (i = 0 ; i < MEDIACODECSYNC_ELEMENT_NUM ; i++) {
 254                 if (handle->video_pipe[i] && G_OBJECT(handle->video_pipe[i]) == obj) {
 255                         LOGD("video element[%d] was released", i);
 256                         handle->video_pipe[i] = NULL;
 257                         return;
 258                 }
 259         }
 260
 261         LOGW("no matched element [%p]", obj);
 262 }
 263
 264
 265 static GstCaps *_mediacodecsync_make_caps_from_mediaformat(media_format_h format)
 266 {
 267         int i = 0;
 268         GstCaps *new_caps = NULL;
 269         media_format_type_e type = MEDIA_FORMAT_NONE;
 270         media_format_mimetype_e mime_type = 0;
 271
 272         format_table audio_table[AUDIO_FORMAT_TABLE_SIZE] = {
 273                 {MEDIA_FORMAT_PCM,       "S16LE"},
 274                 {MEDIA_FORMAT_PCM_S16LE, "S16LE"},
 275                 {MEDIA_FORMAT_PCM_S24LE, "S24LE"},
 276                 {MEDIA_FORMAT_PCM_S32LE, "S32LE"},
 277                 {MEDIA_FORMAT_PCM_S16BE, "S16BE"},
 278                 {MEDIA_FORMAT_PCM_S24BE, "S24BE"},
 279                 {MEDIA_FORMAT_PCM_S32BE, "S32BE"},
 280                 {MEDIA_FORMAT_PCM_F32LE, "F32LE"},
 281                 {MEDIA_FORMAT_PCM_F32BE, "F32BE"},
 282                 {MEDIA_FORMAT_PCM_U16LE, "U16LE"},
 283                 {MEDIA_FORMAT_PCM_U24LE, "U24LE"},
 284                 {MEDIA_FORMAT_PCM_U32LE, "U32LE"},
 285                 {MEDIA_FORMAT_PCM_U16BE, "U16BE"},
 286                 {MEDIA_FORMAT_PCM_U24BE, "U24BE"},
 287                 {MEDIA_FORMAT_PCM_U32BE, "U32BE"}
 288         };
 289         format_table video_table[VIDEO_FORMAT_TABLE_SIZE] = {
 290                 {MEDIA_FORMAT_I420,  "I420"},
 291                 {MEDIA_FORMAT_NV12,  "SN12"},
 292                 {MEDIA_FORMAT_NV12T, "NV12T"},
 293                 {MEDIA_FORMAT_YV12,  "YV12"},
 294                 {MEDIA_FORMAT_NV21,  "NV21"},
 295                 {MEDIA_FORMAT_NV16,  "NV16"},
 296                 {MEDIA_FORMAT_YUYV,  "YUYV"},
 297                 {MEDIA_FORMAT_UYVY,  "UYVY"}
 298         };
 299
 300         if (media_format_get_type(format, &type) != MEDIA_FORMAT_ERROR_NONE) {
 301                 LOGE("get type failed");
 302                 return NULL;
 303         }
 304
 305         if (type & MEDIA_FORMAT_AUDIO) {
 306                 int channel;
 307                 int samplerate;
 308                 int bit;
 309
 310                 if (media_format_get_audio_info(format, &mime_type, &channel,
 311                                 &samplerate, &bit, NULL) != MEDIA_FORMAT_ERROR_NONE) {
 312                         LOGE("get audio info failed");
 313                         return NULL;
 314                 }
 315
 316                 for (i = 0 ; i < AUDIO_FORMAT_TABLE_SIZE ; i++) {
 317                         if (audio_table[i].mime_type == mime_type) {
 318                                 new_caps = gst_caps_new_simple("audio/x-raw",
 319                                         "rate", G_TYPE_INT, samplerate,
 320                                         "channels", G_TYPE_INT, channel,
 321                                         "format", G_TYPE_STRING, audio_table[i].format_string,
 322                                         NULL);
 323                                 break;
 324                         }
 325                 }
 326         } else if (type & MEDIA_FORMAT_VIDEO) {
 327                 int width;
 328                 int height;
 329
 330                 if (media_format_get_video_info(format, &mime_type,
 331                                 &width, &height, NULL, NULL) != MEDIA_FORMAT_ERROR_NONE) {
 332                         LOGE("get video info failed");
 333                         return NULL;
 334                 }
 335
 336                 for (i = 0 ; i < VIDEO_FORMAT_TABLE_SIZE ; i++) {
 337                         if (video_table[i].mime_type == mime_type) {
 338                                 new_caps = gst_caps_new_simple("video/x-raw",
 339                                         "format", G_TYPE_STRING, video_table[i].format_string,
 340                                         "width", G_TYPE_INT, width,
 341                                         "height", G_TYPE_INT, height,
 342                                         "framerate", GST_TYPE_FRACTION, 30, 1,
 343                                         NULL);
 344                                 break;
 345                         }
 346                 }
 347         }
 348
 349         if (new_caps) {
 350                 gchar *caps_string = gst_caps_to_string(new_caps);
 351                 if (caps_string) {
 352                         LOGD("caps [%s]", caps_string);
 353                         g_free(caps_string);
 354                 }
 355         } else {
 356                 LOGE("new caps failed [type:0x%x,0x%x]", type, mime_type);
 357         }
 358
 359         return new_caps;
 360 }
 361
 362
 363 static void _mediacodecsync_destroy_pipeline(GstElement *pipeline[MEDIACODECSYNC_ELEMENT_NUM])
 364 {
 365         GstStateChangeReturn result = GST_STATE_CHANGE_SUCCESS;
 366
 367         if (!pipeline[MEDIACODECSYNC_PIPELINE]) {
 368                 LOGW("NULL pipeline");
 369                 return;
 370         }
 371
 372         result = gst_element_set_state(pipeline[MEDIACODECSYNC_PIPELINE], GST_STATE_NULL);
 373
 374         LOGD("set state NULL : %d", result);
 375
 376         _MEDIACODECSYNC_ELEMENT_REMOVE(pipeline[MEDIACODECSYNC_PIPELINE]);
 377         _MEDIACODECSYNC_ELEMENT_REMOVE(pipeline[MEDIACODECSYNC_ELEMENT_APPSRC]);
 378         _MEDIACODECSYNC_ELEMENT_REMOVE(pipeline[MEDIACODECSYNC_ELEMENT_CAPS]);
 379         _MEDIACODECSYNC_ELEMENT_REMOVE(pipeline[MEDIACODECSYNC_ELEMENT_QUE]);
 380         _MEDIACODECSYNC_ELEMENT_REMOVE(pipeline[MEDIACODECSYNC_ELEMENT_SINK]);
 381 }
 382
 383
 384 static int _mediacodecsync_create_pipeline(mediacodecsync_s *handle, media_format_h format,
 385         GstElement *pipeline[MEDIACODECSYNC_ELEMENT_NUM], const char *pipe_name, const char *nick_prefix,
 386         GCallback need_data_cb, GCallback enough_data_cb, GstCaps **out_caps)
 387 {
 388         GstCaps *caps = NULL;
 389
 390         LOGD("name : %s", pipe_name);
 391
 392         /* main pipeline */
 393         _MEDIACODECSYNC_PIPELINE_MAKE(handle, pipeline[MEDIACODECSYNC_PIPELINE], pipe_name);
 394
 395         /* elements */
 396         _MEDIACODECSYNC_ELEMENT_MAKE(handle, pipeline[MEDIACODECSYNC_ELEMENT_APPSRC], "appsrc", nick_prefix, "src");
 397         _MEDIACODECSYNC_ELEMENT_MAKE(handle, pipeline[MEDIACODECSYNC_ELEMENT_CAPS], "capsfilter", nick_prefix, "caps");
 398
 399         if (!strncmp(nick_prefix, "audio", 5)) {
 400                 _MEDIACODECSYNC_ELEMENT_MAKE(handle, pipeline[MEDIACODECSYNC_ELEMENT_QUE], "queue", nick_prefix, "queue");
 401                 _MEDIACODECSYNC_ELEMENT_MAKE(handle, pipeline[MEDIACODECSYNC_ELEMENT_SINK], "pulsesink", nick_prefix, "sink");
 402                 g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_SINK]), "qos", 1, NULL);
 403                 g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_SINK]), "provide-clock", 0, NULL);
 404                 g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_SINK]), "slave-method", 2, NULL); /* GST_AUDIO_BASE_SINK_SLAVE_NONE */
 405         } else {
 406                 _MEDIACODECSYNC_ELEMENT_MAKE(handle, pipeline[MEDIACODECSYNC_ELEMENT_QUE], "queue", nick_prefix, "queue");
 407                 _MEDIACODECSYNC_ELEMENT_MAKE(handle, pipeline[MEDIACODECSYNC_ELEMENT_SINK], "tizenwlsink", nick_prefix, "sink");
 408
 409                 g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_SINK]), "use-tbm", 1, NULL);
 410                 g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_SINK]), "enable-last-sample", 0, NULL);
 411                 g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_SINK]), "show-preroll-frame", 0, NULL);
 412         }
 413
 414         g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_APPSRC]), "max-bytes", (guint64)0, NULL); /* unlimited */
 415         g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_QUE]), "max-size-time", 0, NULL);
 416         g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_QUE]), "max-size-bytes", 0, NULL);
 417         g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_QUE]), "max-size-buffers", 0, NULL);
 418         g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_APPSRC]), "format", 3, NULL); /* GST_FORMAT_TIME */
 419         g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_SINK]), "sync", 1, NULL);
 420
 421         /* add elements to pipeline */
 422         gst_bin_add_many(GST_BIN(pipeline[MEDIACODECSYNC_PIPELINE]),
 423                 pipeline[MEDIACODECSYNC_ELEMENT_APPSRC],
 424                 pipeline[MEDIACODECSYNC_ELEMENT_CAPS],
 425                 pipeline[MEDIACODECSYNC_ELEMENT_QUE],
 426                 pipeline[MEDIACODECSYNC_ELEMENT_SINK],
 427                 NULL);
 428
 429         /* link elements */
 430         if (!(gst_element_link_many(pipeline[MEDIACODECSYNC_ELEMENT_APPSRC],
 431                 pipeline[MEDIACODECSYNC_ELEMENT_CAPS],
 432                 pipeline[MEDIACODECSYNC_ELEMENT_QUE],
 433                 pipeline[MEDIACODECSYNC_ELEMENT_SINK],
 434                 NULL))) {
 435                 LOGE("link elements failed");
 436                 goto _CREATE_PIPELINE_FAILED;
 437         }
 438
 439         /* set capsfilter */
 440         caps = _mediacodecsync_make_caps_from_mediaformat(format);
 441         if (!caps) {
 442                 LOGE("pipeline caps failed");
 443                 goto _CREATE_PIPELINE_FAILED;
 444         }
 445
 446         g_object_set(G_OBJECT(pipeline[MEDIACODECSYNC_ELEMENT_CAPS]), "caps", caps, NULL);
 447
 448         if (out_caps) {
 449                 if (*out_caps) {
 450                         LOGD("unref previous video_caps %p", *out_caps);
 451                         gst_caps_unref(*out_caps);
 452                         *out_caps = NULL;
 453                 }
 454
 455                 *out_caps = caps;
 456         } else {
 457                 gst_caps_unref(caps);
 458                 caps = NULL;
 459         }
 460
 461         g_signal_connect(pipeline[MEDIACODECSYNC_ELEMENT_APPSRC], "need-data", need_data_cb, handle);
 462         g_signal_connect(pipeline[MEDIACODECSYNC_ELEMENT_APPSRC], "enough-data", enough_data_cb, handle);
 463
 464         LOGD("done");
 465
 466         return MEDIACODEC_ERROR_NONE;
 467
 468 _CREATE_PIPELINE_FAILED:
 469         LOGE("[%s] failed", pipe_name);
 470
 471         _mediacodecsync_destroy_pipeline(pipeline);
 472
 473         return MEDIACODEC_ERROR_INTERNAL;
 474 }
 475
 476
 477 static int _mediacodecsync_prepare_pipeline(mediacodecsync_s *handle)
 478 {
 479         int ret = MEDIACODEC_ERROR_NONE;
 480         GstClock *clock = NULL;
 481         GstStateChangeReturn result = GST_STATE_CHANGE_SUCCESS;
 482
 483         LOGD("enter");
 484
 485         if (handle->audio_format) {
 486                 /* create audio pipeline */
 487                 ret = _mediacodecsync_create_pipeline(handle, handle->audio_format,
 488                         handle->audio_pipe, PIPELINE_NAME_AUDIO, "audio",
 489                         (GCallback)__mediacodecsync_need_audio_data_cb,
 490                         (GCallback)__mediacodecsync_enough_audio_data_cb,
 491                         NULL);
 492                 if (ret != MEDIACODEC_ERROR_NONE)
 493                         return ret;
 494
 495                 /* set state */
 496                 result = gst_element_set_state(handle->audio_pipe[MEDIACODECSYNC_PIPELINE], GST_STATE_PAUSED);
 497                 if (result == GST_STATE_CHANGE_FAILURE) {
 498                         LOGE("state change failed - VIDEO pipeline");
 499                         ret = MEDIACODEC_ERROR_INTERNAL;
 500                         goto _PREPARE_FAILED;
 501                 }
 502
 503                 LOGD("audio pipeline done");
 504         }
 505
 506         if (handle->video_format) {
 507                 /* create video pipeline */
 508                 ret = _mediacodecsync_create_pipeline(handle, handle->video_format,
 509                         handle->video_pipe, PIPELINE_NAME_VIDEO, "video",
 510                         (GCallback)__mediacodecsync_need_video_data_cb,
 511                         (GCallback)__mediacodecsync_enough_video_data_cb,
 512                         &handle->video_caps);
 513                 if (ret != MEDIACODEC_ERROR_NONE)
 514                         goto _PREPARE_FAILED;
 515
 516                 /* get video info */
 517                 if (!gst_video_info_from_caps(&handle->video_info, handle->video_caps)) {
 518                         LOGE("failed to get video info");
 519                         goto _PREPARE_FAILED;
 520                 }
 521
 522                 /* set state */
 523                 result = gst_element_set_state(handle->video_pipe[MEDIACODECSYNC_PIPELINE], GST_STATE_PAUSED);
 524                 if (result == GST_STATE_CHANGE_FAILURE) {
 525                         LOGE("state change failed - VIDEO pipeline");
 526                         ret = MEDIACODEC_ERROR_INTERNAL;
 527                         goto _PREPARE_FAILED;
 528                 }
 529
 530                 LOGD("video pipeline done");
 531
 532                 /* share clock to synchronize */
 533                 if (handle->audio_pipe[MEDIACODECSYNC_PIPELINE]) {
 534                         LOGD("set clock of audio pipeline to video pipeline");
 535
 536                         clock = gst_pipeline_get_clock(GST_PIPELINE(handle->audio_pipe[MEDIACODECSYNC_PIPELINE]));
 537
 538                         if (!gst_pipeline_set_clock(GST_PIPELINE(handle->video_pipe[MEDIACODECSYNC_PIPELINE]), clock))
 539                                 LOGW("failed to set clock to video pipeline");
 540
 541                         gst_object_unref(clock);
 542                         clock = NULL;
 543                 }
 544         }
 545
 546         LOGD("done");
 547
 548         return ret;
 549
 550 _PREPARE_FAILED:
 551         _mediacodecsync_destroy_pipeline(handle->audio_pipe);
 552         _mediacodecsync_destroy_pipeline(handle->video_pipe);
 553
 554         return ret;
 555 }
 556
 557
 558 static void _mediacodecsync_unprepare_pipeline(mediacodecsync_s *handle)
 559 {
 560         _mediacodecsync_destroy_pipeline(handle->audio_pipe);
 561         _mediacodecsync_destroy_pipeline(handle->video_pipe);
 562 }
 563
 564
 565
 566 int mediacodecsync_create(mediacodecsync_buffer_used_cb callback, void *userdata, mediacodecsync_h *mediacodecsync)
 567 {
 568         mediacodecsync_s *new_handle = NULL;
 569
 570         MEDIACODEC_NULL_ARG_CHECK(mediacodecsync);
 571         MEDIACODEC_NULL_ARG_CHECK(callback);
 572
 573         LOGD("enter");
 574
 575         new_handle = (mediacodecsync_s *)malloc(sizeof(mediacodecsync_s));
 576         if (new_handle == NULL) {
 577                 LOGE("handle allocation failed");
 578                 return MEDIACODEC_ERROR_OUT_OF_MEMORY;
 579         }
 580
 581         memset(new_handle, 0x0, sizeof(mediacodecsync_s));
 582
 583         new_handle->buffer_used_cb = callback;
 584         new_handle->buffer_used_cb_userdata = userdata;
 585         new_handle->state = MEDIACODECSYNC_STATE_CREATED;
 586         new_handle->allocator = gst_tizen_allocator_new();
 587         g_mutex_init(&new_handle->lock);
 588
 589         *mediacodecsync = (mediacodecsync_h)new_handle;
 590
 591         LOGD("new handle : %p", *mediacodecsync);
 592
 593         return MEDIACODEC_ERROR_NONE;
 594 }
 595
 596
 597 int mediacodecsync_destroy(mediacodecsync_h mediacodecsync)
 598 {
 599         mediacodecsync_s *handle = NULL;
 600
 601         MEDIACODEC_INSTANCE_CHECK(mediacodecsync);
 602
 603         handle = (mediacodecsync_s *)mediacodecsync;
 604
 605         LOGD("handle to destroy : %p", handle);
 606
 607         g_mutex_lock(&handle->lock);
 608
 609         if (handle->state != MEDIACODECSYNC_STATE_CREATED) {
 610                 LOGE("invalid state %d", handle->state);
 611                 g_mutex_unlock(&handle->lock);
 612                 return MEDIACODEC_ERROR_INVALID_STATE;
 613         }
 614
 615         if (handle->audio_format) {
 616                 media_format_unref(handle->audio_format);
 617                 handle->audio_format = NULL;
 618         }
 619
 620         if (handle->video_format) {
 621                 media_format_unref(handle->video_format);
 622                 handle->video_format = NULL;
 623         }
 624
 625         if (handle->video_caps) {
 626                 gst_caps_unref(handle->video_caps);
 627                 handle->video_caps = NULL;
 628         }
 629
 630         g_mutex_unlock(&handle->lock);
 631
 632         g_mutex_clear(&handle->lock);
 633
 634         free(handle);
 635
 636         LOGD("done");
 637
 638         return MEDIACODEC_ERROR_NONE;
 639 }
 640
 641
 642 int mediacodecsync_set_format(mediacodecsync_h mediacodecsync, media_format_h audio_format, media_format_h video_format)
 643 {
 644         int ret = MEDIACODEC_ERROR_NONE;
 645         mediacodecsync_s *handle = NULL;
 646
 647         MEDIACODEC_INSTANCE_CHECK(mediacodecsync);
 648
 649         if (!audio_format && !video_format) {
 650                 LOGE("all formats are NULL");
 651                 return MEDIACODEC_ERROR_INVALID_PARAMETER;
 652         }
 653
 654         handle = (mediacodecsync_s *)mediacodecsync;
 655
 656         g_mutex_lock(&handle->lock);
 657
 658         if (handle->state != MEDIACODECSYNC_STATE_CREATED) {
 659                 LOGE("invalid state %d", handle->state);
 660                 ret = MEDIACODEC_ERROR_INVALID_STATE;
 661                 goto _SET_FORMAT_DONE;
 662         }
 663
 664         if (audio_format) {
 665                 LOGD("set audio format : %p", audio_format);
 666
 667                 if (media_format_ref(audio_format) != MEDIACODEC_ERROR_NONE) {
 668                         LOGE("audio format ref failed");
 669                         ret = MEDIACODEC_ERROR_INTERNAL;
 670                         goto _SET_FORMAT_DONE;
 671                 }
 672
 673                 if (handle->audio_format)
 674                         media_format_unref(handle->audio_format);
 675
 676                 handle->audio_format = audio_format;
 677         }
 678
 679         if (video_format) {
 680                 LOGD("set video format : %p", audio_format);
 681
 682                 if (media_format_ref(video_format) != MEDIACODEC_ERROR_NONE) {
 683                         LOGE("video format ref failed");
 684                         ret = MEDIACODEC_ERROR_INTERNAL;
 685                         goto _SET_FORMAT_DONE;
 686                 }
 687
 688                 if (handle->video_format)
 689                         media_format_unref(handle->video_format);
 690
 691                 handle->video_format = video_format;
 692         }
 693
 694 _SET_FORMAT_DONE:
 695         g_mutex_unlock(&handle->lock);
 696
 697         return ret;
 698 }
 699
 700
 701 int mediacodecsync_prepare(mediacodecsync_h mediacodecsync)
 702 {
 703         int ret = MEDIACODEC_ERROR_NONE;
 704         mediacodecsync_s *handle = NULL;
 705
 706         MEDIACODEC_INSTANCE_CHECK(mediacodecsync);
 707
 708         handle = (mediacodecsync_s *)mediacodecsync;
 709
 710         g_mutex_lock(&handle->lock);
 711
 712         if (handle->state != MEDIACODECSYNC_STATE_CREATED) {
 713                 LOGE("invalid state %d", handle->state);
 714                 g_mutex_unlock(&handle->lock);
 715                 return MEDIACODEC_ERROR_INVALID_STATE;
 716         }
 717
 718         ret = _mediacodecsync_prepare_pipeline(handle);
 719         if (ret == MEDIACODEC_ERROR_NONE)
 720                 handle->state = MEDIACODECSYNC_STATE_READY;
 721
 722         g_mutex_unlock(&handle->lock);
 723
 724         return ret;
 725 }
 726
 727
 728 int mediacodecsync_unprepare(mediacodecsync_h mediacodecsync)
 729 {
 730         int ret = MEDIACODEC_ERROR_NONE;
 731         mediacodecsync_s *handle = NULL;
 732
 733         MEDIACODEC_INSTANCE_CHECK(mediacodecsync);
 734
 735         handle = (mediacodecsync_s *)mediacodecsync;
 736
 737         g_mutex_lock(&handle->lock);
 738
 739         if (handle->state != MEDIACODECSYNC_STATE_READY) {
 740                 LOGE("invalid state %d", handle->state);
 741                 g_mutex_unlock(&handle->lock);
 742                 return MEDIACODEC_ERROR_INVALID_STATE;
 743         }
 744
 745         _mediacodecsync_unprepare_pipeline(handle);
 746         handle->state = MEDIACODECSYNC_STATE_CREATED;
 747
 748         g_mutex_unlock(&handle->lock);
 749
 750         return ret;
 751 }
 752
 753
 754 int mediacodecsync_run(mediacodecsync_h mediacodecsync)
 755 {
 756         int ret = MEDIACODEC_ERROR_NONE;
 757         mediacodecsync_s *handle = NULL;
 758         GstStateChangeReturn result = GST_STATE_CHANGE_SUCCESS;
 759
 760         MEDIACODEC_INSTANCE_CHECK(mediacodecsync);
 761
 762         handle = (mediacodecsync_s *)mediacodecsync;
 763
 764         g_mutex_lock(&handle->lock);
 765
 766         if (handle->state != MEDIACODECSYNC_STATE_READY &&
 767                 handle->state != MEDIACODECSYNC_STATE_PAUSED) {
 768                 LOGE("invalid state %d", handle->state);
 769                 ret = MEDIACODEC_ERROR_INVALID_STATE;
 770                 goto _RUN_DONE;
 771         }
 772
 773         /* set state */
 774         if (handle->audio_pipe[MEDIACODECSYNC_PIPELINE]) {
 775                 result = gst_element_set_state(handle->audio_pipe[MEDIACODECSYNC_PIPELINE], GST_STATE_PLAYING);
 776                 if (result == GST_STATE_CHANGE_FAILURE) {
 777                         LOGE("state change failed - AUDIO pipeline");
 778                         ret = MEDIACODEC_ERROR_INTERNAL;
 779                         goto _RUN_DONE;
 780                 }
 781         }
 782
 783         if (handle->video_pipe[MEDIACODECSYNC_PIPELINE]) {
 784                 result = gst_element_set_state(handle->video_pipe[MEDIACODECSYNC_PIPELINE], GST_STATE_PLAYING);
 785                 if (result == GST_STATE_CHANGE_FAILURE) {
 786                         LOGE("state change failed - VIDEO pipeline");
 787                         ret = MEDIACODEC_ERROR_INTERNAL;
 788                         goto _RUN_DONE;
 789                 }
 790         }
 791
 792         LOGD("pipeline is now playing");
 793         handle->state = MEDIACODECSYNC_STATE_RUNNING;
 794
 795 _RUN_DONE:
 796         g_mutex_unlock(&handle->lock);
 797
 798         return ret;
 799 }
 800
 801
 802 int mediacodecsync_stop(mediacodecsync_h mediacodecsync)
 803 {
 804         int ret = MEDIACODEC_ERROR_NONE;
 805         mediacodecsync_s *handle = NULL;
 806         GstStateChangeReturn result = GST_STATE_CHANGE_SUCCESS;
 807
 808         MEDIACODEC_INSTANCE_CHECK(mediacodecsync);
 809
 810         handle = (mediacodecsync_s *)mediacodecsync;
 811
 812         g_mutex_lock(&handle->lock);
 813
 814         if (handle->state < MEDIACODECSYNC_STATE_RUNNING) {
 815                 LOGE("invalid state %d", handle->state);
 816                 g_mutex_unlock(&handle->lock);
 817                 return MEDIACODEC_ERROR_INVALID_STATE;
 818         }
 819
 820         /* set state */
 821         result = gst_element_set_state(handle->video_pipe[MEDIACODECSYNC_PIPELINE], GST_STATE_READY);
 822         if (result == GST_STATE_CHANGE_FAILURE) {
 823                 LOGE("state change failed - VIDEO pipeline");
 824                 ret = MEDIACODEC_ERROR_INTERNAL;
 825         } else {
 826                 LOGD("pipeline is ready");
 827                 handle->state = MEDIACODECSYNC_STATE_READY;
 828         }
 829
 830         g_mutex_unlock(&handle->lock);
 831
 832         return ret;
 833 }
 834
 835
 836 int mediacodecsync_pause(mediacodecsync_h mediacodecsync)
 837 {
 838         int ret = MEDIACODEC_ERROR_NONE;
 839         mediacodecsync_s *handle = NULL;
 840         GstStateChangeReturn result = GST_STATE_CHANGE_SUCCESS;
 841
 842         MEDIACODEC_INSTANCE_CHECK(mediacodecsync);
 843
 844         handle = (mediacodecsync_s *)mediacodecsync;
 845
 846         g_mutex_lock(&handle->lock);
 847
 848         if (handle->state != MEDIACODECSYNC_STATE_RUNNING) {
 849                 LOGE("invalid state %d", handle->state);
 850                 g_mutex_unlock(&handle->lock);
 851                 return MEDIACODEC_ERROR_INVALID_STATE;
 852         }
 853
 854         /* set state */
 855         result = gst_element_set_state(handle->video_pipe[MEDIACODECSYNC_PIPELINE], GST_STATE_PAUSED);
 856         if (result == GST_STATE_CHANGE_FAILURE) {
 857                 LOGE("state change failed - VIDEO pipeline");
 858                 ret = MEDIACODEC_ERROR_INTERNAL;
 859         } else {
 860                 LOGD("pipeline is paused");
 861                 handle->state = MEDIACODECSYNC_STATE_PAUSED;
 862         }
 863
 864         g_mutex_unlock(&handle->lock);
 865
 866         return ret;
 867 }
 868
 869
 870 static int _mediacodecsync_push_packet_to_pipeline(GstElement *appsrc, GstBuffer *buffer)
 871 {
 872         GstFlowReturn gst_ret = GST_FLOW_OK;
 873
 874         if (!appsrc || !buffer) {
 875                 LOGE("NULL params %p,%p", appsrc, buffer);
 876                 return MEDIACODEC_ERROR_INTERNAL;
 877         }
 878
 879         gst_ret = gst_app_src_push_buffer((GstAppSrc *)appsrc, buffer);
 880         if (gst_ret != GST_FLOW_OK) {
 881                 LOGE("appsrc push failed %d", gst_ret);
 882                 return MEDIACODEC_ERROR_INTERNAL;
 883         }
 884
 885         return MEDIACODEC_ERROR_NONE;
 886 }
 887
 888
 889 int mediacodecsync_push_packet(mediacodecsync_h mediacodecsync, media_packet_h packet, mediacodecsync_packet_type_e type)
 890 {
 891         int ret = MEDIACODEC_ERROR_NONE;
 892         mediacodecsync_s *handle = NULL;
 893         GstMCSBuffer *mcs_buffer = NULL;
 894         GstElement *appsrc = NULL;
 895
 896         MEDIACODEC_INSTANCE_CHECK(mediacodecsync);
 897
 898         handle = (mediacodecsync_s *)mediacodecsync;
 899
 900         g_mutex_lock(&handle->lock);
 901
 902         if (handle->state != MEDIACODECSYNC_STATE_RUNNING) {
 903                 LOGE("invalid state %d", handle->state);
 904                 ret = MEDIACODEC_ERROR_INVALID_STATE;
 905                 goto _PUSH_PACKET_DONE;
 906         }
 907
 908         mcs_buffer = _mediacodecsync_gst_buffer_new(handle, packet, type);
 909         if (!mcs_buffer) {
 910                 LOGE("mediacodecsync new buffer failed");
 911                 ret = MEDIACODEC_ERROR_INTERNAL;
 912                 goto _PUSH_PACKET_DONE;
 913         }
 914
 915         LOGI("type %u, new buffer %p [gst %p][packet %p]",
 916                 type, mcs_buffer, mcs_buffer->gst_buffer, packet);
 917
 918         switch (type) {
 919         case MEDIACODECSYNC_PACKET_TYPE_AUDIO:
 920                 appsrc = handle->audio_pipe[MEDIACODECSYNC_ELEMENT_APPSRC];
 921                 break;
 922         case MEDIACODECSYNC_PACKET_TYPE_VIDEO:
 923                 appsrc = handle->video_pipe[MEDIACODECSYNC_ELEMENT_APPSRC];
 924                 break;
 925         default:
 926                 LOGE("unhandled packet type %u, release created buffer", type);
 927                 gst_buffer_unref(mcs_buffer->gst_buffer);
 928                 ret = MEDIACODEC_ERROR_INTERNAL;
 929                 goto _PUSH_PACKET_DONE;
 930         }
 931
 932         ret = _mediacodecsync_push_packet_to_pipeline(appsrc, mcs_buffer->gst_buffer);
 933
 934 _PUSH_PACKET_DONE:
 935         g_mutex_unlock(&handle->lock);
 936
 937         return ret;
 938 }
 939
 940
 941 int mediacodecsync_get_state(mediacodecsync_h mediacodecsync, mediacodecsync_state_e *state)
 942 {
 943         mediacodecsync_s *handle = NULL;
 944
 945         MEDIACODEC_INSTANCE_CHECK(mediacodecsync);
 946         MEDIACODEC_NULL_ARG_CHECK(state);
 947
 948         handle = (mediacodecsync_s *)mediacodecsync;
 949
 950         *state = handle->state;
 951
 952         LOGI("state %u", *state);
 953
 954         return MEDIACODEC_ERROR_NONE;
 955 }