src/mm_player_capture.c

   1 /*
   2  * libmm-player
   3  *
   4  * Copyright(c) 2000 - 2011 Samsung Electronics Co., Ltd. All rights reserved.
   5  *
   6  * Contact: JongHyuk Choi <jhchoi.choi@samsung.com>, YeJin Cho <cho.yejin@samsung.com>,
   7  * Seungbae Shin <seungbae.shin@samsung.com>, YoungHwan An <younghwan_.an@samsung.com>
   8  *
   9  * Licensed under the Apache License, Version 2.0(the "License");
  10  * you may not use this file except in compliance with the License.
  11  * You may obtain a copy of the License at
  12  *
  13  * http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  * Unless required by applicable law or agreed to in writing, software
  16  * distributed under the License is distributed on an "AS IS" BASIS,
  17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  * See the License for the specific language governing permissions and
  19  * limitations under the License.
  20  *
  21  */
  22
  23 /*===========================================================================================
  24 |                                                                                                                                                                                       |
  25 |  INCLUDE FILES                                                                                                                                                        |
  26 |                                                                                                                                                                                       |
  27 ========================================================================================== */
  28 #include <dlog.h>
  29 #include "mm_player_utils.h"
  30 #include "mm_player_capture.h"
  31 #include "mm_player_priv.h"
  32
  33 #include <mm_util_imgp.h>
  34 #include <gst/video/video-info.h>
  35
  36 /*---------------------------------------------------------------------------
  37 |    LOCAL VARIABLE DEFINITIONS for internal                                                            |
  38 ---------------------------------------------------------------------------*/
  39
  40 /*---------------------------------------------------------------------------
  41 |    LOCAL FUNCTION PROTOTYPES:                                                                                         |
  42 ---------------------------------------------------------------------------*/
  43 static GstPadProbeReturn __mmplayer_video_capture_probe(GstPad *pad, GstPadProbeInfo *info, gpointer u_data);
  44 static int  __mmplayer_get_video_frame_from_buffer(mm_player_t* player, GstPad *pad, GstBuffer *buffer);
  45 static gpointer __mmplayer_capture_thread(gpointer data);
  46 static void __csc_tiled_to_linear_crop(unsigned char *yuv420_dest, unsigned char *nv12t_src, int yuv420_width, int yuv420_height, int left, int top, int right, int buttom);
  47 static int __tile_4x2_read(int x_size, int y_size, int x_pos, int y_pos);
  48 static int __mm_player_convert_colorspace(mm_player_t* player, unsigned char* src_data, mm_util_img_format src_fmt, unsigned int src_w, unsigned int src_h, mm_util_img_format dst_fmt);
  49
  50 /*===========================================================================================
  51 |                                                                                                                                                                                       |
  52 |  FUNCTION DEFINITIONS                                                                                                                                         |
  53 |                                                                                                                                                                                       |
  54 ========================================================================================== */
  55 int
  56 _mmplayer_initialize_video_capture(mm_player_t* player)
  57 {
  58         int ret = MM_ERROR_NONE;
  59         MMPLAYER_RETURN_VAL_IF_FAIL(player, MM_ERROR_PLAYER_NOT_INITIALIZED);
  60         /* create capture mutex */
  61         g_mutex_init(&player->capture_thread_mutex);
  62
  63         /* create capture cond */
  64         g_cond_init(&player->capture_thread_cond);
  65
  66
  67         player->capture_thread_exit = FALSE;
  68
  69         /* create video capture thread */
  70         player->capture_thread =
  71                         g_thread_try_new("capture_thread", __mmplayer_capture_thread, (gpointer)player, NULL);
  72
  73         if (!player->capture_thread) {
  74                 ret = MM_ERROR_PLAYER_RESOURCE_LIMIT;
  75                 goto ERROR;
  76         }
  77
  78         return ret;
  79
  80 ERROR:
  81         /* capture thread */
  82         g_mutex_clear(&player->capture_thread_mutex);
  83
  84         g_cond_clear(&player->capture_thread_cond);
  85
  86         return ret;
  87 }
  88
  89 int
  90 _mmplayer_release_video_capture(mm_player_t* player)
  91 {
  92         MMPLAYER_RETURN_VAL_IF_FAIL(player, MM_ERROR_PLAYER_NOT_INITIALIZED);
  93         /* release capture thread */
  94         MMPLAYER_CAPTURE_THREAD_LOCK(player);
  95         player->capture_thread_exit = TRUE;
  96         MMPLAYER_CAPTURE_THREAD_SIGNAL(player);
  97         MMPLAYER_CAPTURE_THREAD_UNLOCK(player);
  98
  99         LOGD("waitting for capture thread exit");
 100         g_thread_join(player->capture_thread);
 101         g_mutex_clear(&player->capture_thread_mutex);
 102         g_cond_clear(&player->capture_thread_cond);
 103         LOGD("capture thread released");
 104
 105         return MM_ERROR_NONE;
 106 }
 107
 108 int
 109 _mmplayer_do_video_capture(MMHandleType hplayer)
 110 {
 111         mm_player_t* player = (mm_player_t*) hplayer;
 112         int ret = MM_ERROR_NONE;
 113         GstPad *pad = NULL;
 114
 115         MMPLAYER_FENTER();
 116
 117         MMPLAYER_RETURN_VAL_IF_FAIL(player && player->pipeline, MM_ERROR_PLAYER_NOT_INITIALIZED);
 118
 119         /* capturing or not */
 120         if (player->video_capture_cb_probe_id || player->capture.data
 121                         || player->captured.data[0] || player->captured.data[1]) {
 122                 LOGW("capturing... we can't do any more");
 123                 return MM_ERROR_PLAYER_INVALID_STATE;
 124         }
 125         gint surface_type = 0;
 126         mm_attrs_get_int_by_name(player->attrs, "display_surface_type", &surface_type);
 127
 128         /* check if video pipeline is linked or not */
 129         if (!player->pipeline->videobin) {
 130                 if (surface_type == MM_DISPLAY_SURFACE_REMOTE && player->video_fakesink) {
 131                         LOGD("it is evas surface type.");
 132                 } else {
 133                         LOGW("not ready to capture");
 134                         return MM_ERROR_PLAYER_INVALID_STATE;
 135                 }
 136         }
 137
 138         if (player->pipeline->videobin)
 139                 pad = gst_element_get_static_pad(player->pipeline->videobin[MMPLAYER_V_SINK].gst, "sink");
 140         else // evas surface
 141                 pad = gst_element_get_static_pad(player->video_fakesink, "sink");
 142
 143         if (player->state != MM_PLAYER_STATE_PLAYING) {
 144                 if (player->state == MM_PLAYER_STATE_PAUSED) {
 145                         // get last buffer from video sink
 146                         GstSample *sample = NULL;
 147
 148                         gst_element_get_state(player->pipeline->mainbin[MMPLAYER_M_PIPE].gst, NULL, NULL, 5 * GST_SECOND); //5 seconds
 149
 150                         if (player->pipeline->videobin)
 151                                 g_object_get(player->pipeline->videobin[MMPLAYER_V_SINK].gst, "last-sample", &sample, NULL);
 152                         else // evas surface
 153                                 g_object_get(player->video_fakesink, "last-sample", &sample, NULL);
 154
 155                         if (sample) {
 156                                 GstBuffer *buf = NULL;
 157                                 buf = gst_sample_get_buffer(sample);
 158                                 if (buf) {
 159                                         if (__mmplayer_get_video_frame_from_buffer(player, pad, buf) != MM_ERROR_NONE)
 160                                                 ret = MM_ERROR_PLAYER_INTERNAL;
 161                                 } else {
 162                                         LOGW("failed to get video frame");
 163                                 }
 164                                 gst_sample_unref(sample);
 165                         }
 166                         return ret;
 167                 } else {
 168                         LOGW("invalid state(%d) to capture", player->state);
 169                         gst_object_unref(GST_OBJECT(pad));
 170                         pad = NULL;
 171                         return MM_ERROR_PLAYER_INVALID_STATE;
 172                 }
 173         }
 174
 175         /* register probe */
 176         player->video_capture_cb_probe_id = gst_pad_add_probe(pad, GST_PAD_PROBE_TYPE_BUFFER,
 177                 __mmplayer_video_capture_probe, player, NULL);
 178
 179         gst_object_unref(GST_OBJECT(pad));
 180         pad = NULL;
 181
 182         MMPLAYER_FLEAVE();
 183
 184         return ret;
 185 }
 186
 187 int
 188 __mmplayer_handle_orientation(mm_player_t* player, int orientation, int format)
 189 {
 190         unsigned char *src_buffer = NULL;
 191         int ret = MM_ERROR_NONE;
 192         unsigned char *dst_frame = NULL;
 193         unsigned int dst_width = 0;
 194         unsigned int dst_height = 0;
 195         unsigned int dst_size = 0;
 196         mm_util_img_rotate_type rot_enum = MM_UTIL_ROTATE_NUM;
 197
 198         player->capture.orientation = orientation;
 199
 200         if (orientation == 90 || orientation == 270) {
 201                 dst_width = player->captured.height[0];
 202                 dst_height = player->captured.width[0];
 203                 LOGD("interchange width & height");
 204         } else if (orientation == 180) {
 205                 dst_width = player->captured.width[0];
 206                 dst_height = player->captured.height[0];
 207         } else if (orientation == 0) {
 208                 LOGE("no need handle orientation : %d", orientation);
 209                 player->capture.width = player->captured.width[0];
 210                 player->capture.height = player->captured.height[0];
 211                 return MM_ERROR_NONE;
 212         } else
 213                 LOGE("wrong orientation value...");
 214
 215     /* height & width will be interchanged for 90 and 270 orientation */
 216         ret = mm_util_get_image_size(format, dst_width, dst_height, &dst_size);
 217         if (ret != MM_ERROR_NONE) {
 218                 LOGE("failed to get destination frame size");
 219                 return ret;
 220         }
 221
 222         LOGD("before rotation : dst_width = %d and dst_height = %d", dst_width, dst_height);
 223
 224         dst_frame = (unsigned char*) malloc(dst_size);
 225         if (!dst_frame) {
 226                 LOGE("failed to allocate memory");
 227                 return MM_ERROR_PLAYER_NO_FREE_SPACE;
 228         }
 229
 230         src_buffer = (unsigned char*)player->capture.data;
 231
 232         /* convert orientation degree into enum here */
 233         switch (orientation) {
 234         case 90:
 235                 rot_enum = MM_UTIL_ROTATE_90;
 236                 break;
 237         case 180:
 238                 rot_enum = MM_UTIL_ROTATE_180;
 239                 break;
 240         case 270:
 241                 rot_enum = MM_UTIL_ROTATE_270;
 242                 break;
 243         default:
 244                 LOGE("wrong rotate value");
 245                 break;
 246         }
 247
 248         LOGD("source buffer for rotation = %p and rotation = %d", src_buffer, rot_enum);
 249
 250         ret = mm_util_rotate_image(src_buffer,
 251                         player->captured.width[0], player->captured.height[0], format,
 252                         dst_frame, &dst_width, &dst_height, rot_enum);
 253         if (ret != MM_ERROR_NONE) {
 254                 LOGE("failed to do rotate image");
 255                 free(dst_frame);
 256                 return ret;
 257         }
 258
 259         LOGD("after rotation same stride: dst_width = %d and dst_height = %d", dst_width, dst_height);
 260
 261         g_free(src_buffer);
 262
 263         player->capture.data = dst_frame;
 264         player->capture.size = dst_size;
 265         player->capture.orientation = orientation;
 266         player->capture.width = dst_width;
 267         player->capture.height = dst_height;
 268
 269         player->captured.width[0] = player->captured.stride_width[0] = dst_width;
 270         player->captured.height[0] = player->captured.stride_height[0] = dst_height;
 271
 272         return ret;
 273 }
 274
 275
 276 static gpointer
 277 __mmplayer_capture_thread(gpointer data)
 278 {
 279         mm_player_t* player = (mm_player_t*) data;
 280         MMMessageParamType msg = {0, };
 281         unsigned char * src_buffer = NULL;
 282         unsigned char * linear_y_plane = NULL;
 283         unsigned char * linear_uv_plane = NULL;
 284         int orientation = 0;
 285         int ret = 0;
 286         int planes[MAX_BUFFER_PLANE] = {0, };
 287         unsigned char * p_buf = NULL;
 288         unsigned char * temp = NULL;
 289         int i, j;
 290
 291         MMPLAYER_RETURN_VAL_IF_FAIL(player, NULL);
 292
 293         MMPLAYER_CAPTURE_THREAD_LOCK(player);
 294         while (!player->capture_thread_exit) {
 295                 LOGD("capture thread started. waiting for signal");
 296                 MMPLAYER_CAPTURE_THREAD_WAIT(player);
 297
 298                 if (player->capture_thread_exit) {
 299                         LOGD("exiting capture thread");
 300                         goto EXIT;
 301                 }
 302                 LOGD("capture thread is received signal");
 303
 304                 /* NOTE: Don't use MMPLAYER_CMD_LOCK() here.
 305                  * Because deadlock can be happened if other player api is used in message callback.
 306                  */
 307                 if (player->video_cs == MM_PLAYER_COLORSPACE_NV12_TILED) {
 308                         /* Colorspace conversion : NV12T-> NV12-> RGB888 */
 309                         int ret = 0;
 310                         int linear_y_plane_size;
 311                         int linear_uv_plane_size;
 312                         int width = player->captured.width[0];
 313                         int height = player->captured.height[0];
 314
 315                         linear_y_plane_size = (width * height);
 316                         linear_uv_plane_size = (width * height / 2);
 317
 318                         linear_y_plane = (unsigned char*) g_try_malloc(linear_y_plane_size);
 319                         if (linear_y_plane == NULL) {
 320                                 msg.code = MM_ERROR_PLAYER_NO_FREE_SPACE;
 321                                 goto ERROR;
 322                         }
 323
 324                         linear_uv_plane = (unsigned char*) g_try_malloc(linear_uv_plane_size);
 325                         if (linear_uv_plane == NULL) {
 326                                 msg.code = MM_ERROR_PLAYER_NO_FREE_SPACE;
 327                                 goto ERROR;
 328                         }
 329                         /* NV12 tiled to linear */
 330                         __csc_tiled_to_linear_crop(linear_y_plane,
 331                                         player->captured.data[0], width, height, 0, 0, 0, 0);
 332                         __csc_tiled_to_linear_crop(linear_uv_plane,
 333                                         player->captured.data[1], width, height / 2, 0, 0, 0, 0);
 334
 335                         for (i = 0; i < player->captured.handle_num; i++) {
 336                                 MMPLAYER_FREEIF(player->captured.data[i]);
 337                         }
 338
 339                         src_buffer = (unsigned char*) g_try_malloc(linear_y_plane_size+linear_uv_plane_size);
 340
 341                         if (src_buffer == NULL) {
 342                                 msg.code = MM_ERROR_PLAYER_NO_FREE_SPACE;
 343                                 goto ERROR;
 344                         }
 345                         memset(src_buffer, 0x00, linear_y_plane_size+linear_uv_plane_size);
 346                         memcpy(src_buffer, linear_y_plane, linear_y_plane_size);
 347                         memcpy(src_buffer+linear_y_plane_size, linear_uv_plane, linear_uv_plane_size);
 348
 349                         /* NV12 linear to RGB888 */
 350                         ret = __mm_player_convert_colorspace(player, src_buffer, MM_UTIL_IMG_FMT_NV12,
 351                                 width, height, MM_UTIL_IMG_FMT_RGB888);
 352
 353                         if (ret != MM_ERROR_NONE) {
 354                                 LOGE("failed to convert nv12 linear");
 355                                 goto ERROR;
 356                         }
 357                         /* clean */
 358                         MMPLAYER_FREEIF(src_buffer);
 359                         MMPLAYER_FREEIF(linear_y_plane);
 360                         MMPLAYER_FREEIF(linear_uv_plane);
 361                 } else if (MM_PLAYER_COLORSPACE_NV12 == player->video_cs) {
 362                         #define MM_ALIGN(x, a)      (((x) +(a) - 1) & ~((a) - 1))
 363                         int ret = 0;
 364                         /* using original width otherwises, app can't know aligned to resize */
 365                         planes[0] =  player->captured.width[0] * player->captured.height[0];
 366                         planes[1] = player->captured.width[0] * player->captured.height[1];
 367                         int src_buffer_size = planes[0] + planes[1];
 368                         src_buffer = (unsigned char*) g_try_malloc(src_buffer_size);
 369                         p_buf = src_buffer;
 370
 371                         if (!src_buffer_size) {
 372                                 LOGE("invalid data size");
 373                                 goto ERROR;
 374                         }
 375
 376                         if (!src_buffer) {
 377                                 msg.code = MM_ERROR_PLAYER_NO_FREE_SPACE;
 378                                 goto ERROR;
 379                         }
 380
 381                         memset(p_buf, 0x00, src_buffer_size);
 382
 383                         temp = player->captured.data[0];
 384
 385                         /* set Y plane */
 386                         for (i = 0; i < player->captured.height[0]; i++) {
 387                                 memcpy(p_buf, temp, player->captured.width[0]);
 388                                 p_buf += player->captured.width[0];
 389                                 temp += player->captured.stride_width[0];
 390                         }
 391
 392                         temp = player->captured.data[1];
 393
 394                         /* set UV plane*/
 395                         for (j = 0; j < player->captured.height[1]; j++) {
 396                                 memcpy(p_buf, temp, player->captured.width[0]);
 397                                 p_buf += player->captured.width[0];
 398                                 temp += player->captured.stride_width[1];
 399                         }
 400
 401                         /* free captured buf */
 402                         for (i = 0; i < player->captured.handle_num; i++) {
 403                                 MMPLAYER_FREEIF(player->captured.data[i]);
 404                         }
 405
 406                         /* NV12 -> RGB888 */
 407                         ret = __mm_player_convert_colorspace(player, (unsigned char*)src_buffer, MM_UTIL_IMG_FMT_NV12,
 408                                 player->captured.width[0], player->captured.height[0], MM_UTIL_IMG_FMT_RGB888);
 409                         if (ret != MM_ERROR_NONE) {
 410                                 LOGE("failed to convert nv12 linear");
 411                                 goto ERROR;
 412                         }
 413
 414                         /* clean */
 415                         MMPLAYER_FREEIF(src_buffer);
 416                 } else if (MM_PLAYER_COLORSPACE_I420 == player->video_cs) {
 417                         planes[0] = player->captured.width[0] * player->captured.height[0];
 418                         planes[1] = planes[2] = (player->captured.width[0]>>1) * (player->captured.height[0]>>1);
 419
 420                         src_buffer = (unsigned char*) g_try_malloc(player->captured.width[0] * player->captured.height[0]*3/2);
 421                         p_buf = src_buffer;
 422                         /* set Y plane */
 423                         memset(p_buf, 0x00, planes[0]);
 424                         temp = player->captured.data[0];
 425
 426                         for (i = 0; i < player->captured.height[0]; i++) {
 427                                 memcpy(p_buf, temp, player->captured.width[0]);
 428                                 temp += player->captured.stride_width[0];
 429                                 p_buf += player->captured.width[0];
 430                         }
 431
 432                         /* set U plane */
 433                         memset(p_buf, 0x00, planes[1]);
 434                         temp = player->captured.data[1];
 435
 436                         for (i = 0; i < player->captured.height[1]; i++) {
 437                                 memcpy(p_buf, temp, player->captured.width[1]);
 438                                 temp += player->captured.stride_width[1];
 439                                 p_buf += player->captured.width[1];
 440                         }
 441
 442                         /* set V plane */
 443                         memset(p_buf, 0x00, planes[2]);
 444                         temp = player->captured.data[2];
 445
 446                         for (i = 0; i < player->captured.height[2]; i++) {
 447                                 memcpy(p_buf, temp, player->captured.width[2]);
 448                                 temp += player->captured.stride_width[2];
 449                                 p_buf += player->captured.width[2];
 450                         }
 451
 452                         /* I420 -> RGB888 */
 453                         ret = __mm_player_convert_colorspace(player, (unsigned char*)src_buffer, MM_UTIL_IMG_FMT_I420,
 454                                 player->captured.width[0], player->captured.height[0], MM_UTIL_IMG_FMT_RGB888);
 455                         if (ret != MM_ERROR_NONE) {
 456                                 LOGE("failed to convert I420 linear");
 457                                 goto ERROR;
 458                         }
 459
 460                         /* free captured buf */
 461                         MMPLAYER_FREEIF(src_buffer);
 462                         for (i = 0; i < player->captured.handle_num; i++) {
 463                                 MMPLAYER_FREEIF(player->captured.data[i]);
 464                         }
 465                 }
 466
 467                 ret = _mmplayer_get_video_rotate_angle((MMHandleType)player, &orientation);
 468                 if (ret != MM_ERROR_NONE) {
 469                         LOGE("failed to get rotation angle");
 470                         goto ERROR;
 471                 }
 472
 473                 LOGD("orientation value = %d", orientation);
 474
 475                 ret = __mmplayer_handle_orientation(player, orientation, MM_UTIL_IMG_FMT_RGB888);
 476                 if (ret != MM_ERROR_NONE) {
 477                         LOGE("failed to convert nv12 linear");
 478                         goto ERROR;
 479                 }
 480
 481                 player->capture.fmt = MM_PLAYER_COLORSPACE_RGB888;
 482                 msg.data = &player->capture;
 483                 msg.size = player->capture.size;
 484 //              msg.captured_frame.width = player->capture.width;
 485 //              msg.captured_frame.height = player->capture.height;
 486 //              msg.captured_frame.orientation = player->capture.orientation;
 487
 488                 if (player->cmd >= MMPLAYER_COMMAND_START) {
 489                         MMPLAYER_POST_MSG(player, MM_MESSAGE_VIDEO_CAPTURED, &msg);
 490                         LOGD("returned from capture message callback");
 491                 }
 492
 493                 //MMPLAYER_FREEIF(player->capture.data);
 494                 continue;
 495 ERROR:
 496                 MMPLAYER_FREEIF(src_buffer);
 497
 498                 for (i = 0; i < player->captured.handle_num; i++) {
 499                         MMPLAYER_FREEIF(player->captured.data[i]);
 500                 }
 501
 502                 if (player->video_cs == MM_PLAYER_COLORSPACE_NV12_TILED) {
 503                         /* clean */
 504                         MMPLAYER_FREEIF(linear_y_plane);
 505                         MMPLAYER_FREEIF(linear_uv_plane);
 506                 }
 507
 508                 msg.union_type = MM_MSG_UNION_CODE;
 509
 510                 MMPLAYER_POST_MSG(player, MM_MESSAGE_VIDEO_NOT_CAPTURED, &msg);
 511         }
 512
 513 EXIT:
 514         MMPLAYER_CAPTURE_THREAD_UNLOCK(player);
 515         return NULL;
 516 }
 517
 518 /**
 519   * The output is fixed as RGB888
 520   */
 521 static int
 522 __mmplayer_get_video_frame_from_buffer(mm_player_t* player, GstPad *pad, GstBuffer *buffer)
 523 {
 524         int ret = MM_ERROR_NONE;
 525         gint planes[MAX_BUFFER_PLANE] = {0, };
 526         gint i = 0;
 527         gint src_width = 0;
 528         gint src_height = 0;
 529         GstCaps *caps = NULL;
 530         GstStructure *structure = NULL;
 531         GstMapInfo mapinfo = GST_MAP_INFO_INIT;
 532         GstMemory *memory = NULL;
 533         mm_util_img_format src_fmt = MM_UTIL_IMG_FMT_YUV420;
 534         mm_util_img_format dst_fmt = MM_UTIL_IMG_FMT_RGB888; // fixed
 535
 536         MMPLAYER_FENTER();
 537
 538         MMPLAYER_RETURN_VAL_IF_FAIL(player, MM_ERROR_PLAYER_NOT_INITIALIZED);
 539         MMPLAYER_RETURN_VAL_IF_FAIL(buffer, MM_ERROR_INVALID_ARGUMENT);
 540
 541         /* get fourcc */
 542         caps = gst_pad_get_current_caps(pad);
 543
 544         MMPLAYER_RETURN_VAL_IF_FAIL(caps, MM_ERROR_INVALID_ARGUMENT);
 545         MMPLAYER_LOG_GST_CAPS_TYPE(caps);
 546
 547         structure = gst_caps_get_structure(caps, 0);
 548         if (!structure) {
 549                 LOGE("cannot get structure from caps.\n");
 550                 ret = MM_ERROR_PLAYER_INTERNAL;
 551                 goto ERROR;
 552         }
 553
 554         /* init capture image buffer */
 555         memset(&player->capture, 0x00, sizeof(MMPlayerVideoCapture));
 556
 557         gst_structure_get_int(structure, "width", &src_width);
 558         gst_structure_get_int(structure, "height", &src_height);
 559
 560         /* check rgb or yuv */
 561         if (gst_structure_has_name(structure, "video/x-raw")) {
 562                 /* NV12T */
 563                 const gchar *gst_format = gst_structure_get_string(structure, "format");
 564                 if (!g_strcmp0(gst_format, "ST12") || !g_strcmp0(gst_format, "SN12") || !g_strcmp0(gst_format, "S420")) {
 565                         guint n;
 566                         LOGI("captured format is %s\n", gst_format);
 567
 568                         MMVideoBuffer *proved = NULL;
 569                         if (!g_strcmp0(gst_format, "ST12"))
 570                                 player->video_cs = MM_PLAYER_COLORSPACE_NV12_TILED;
 571                         else if (!g_strcmp0(gst_format, "S420"))
 572                                 player->video_cs = MM_PLAYER_COLORSPACE_I420;
 573                         else if (!g_strcmp0(gst_format, "SN12"))
 574                                 player->video_cs = MM_PLAYER_COLORSPACE_NV12;
 575                         else
 576                                 player->video_cs = MM_PLAYER_COLORSPACE_MAX;
 577
 578                         /* get video frame info from proved buffer */
 579                         n = gst_buffer_n_memory(buffer);
 580                         memory = gst_buffer_peek_memory(buffer, n-1);
 581                         gst_memory_map(memory, &mapinfo, GST_MAP_READ);
 582                         proved = (MMVideoBuffer *)mapinfo.data;
 583
 584                         if (!proved || !proved->data[0] || !proved->data[1]) {
 585                                 LOGE("fail to gst_memory_map");
 586                                 ret = MM_ERROR_PLAYER_INTERNAL;
 587                                 goto ERROR;
 588                         }
 589
 590                         memcpy(&player->captured, proved, sizeof(MMVideoBuffer));
 591
 592                         planes[0] = proved->size[0];
 593                         planes[1] = proved->size[1];
 594                         planes[2] = proved->size[2];
 595
 596                         /* Y */
 597                         player->captured.data[0] = g_try_malloc(planes[0]);
 598                         if (!player->captured.data[0]) {
 599                                 gst_memory_unmap(memory, &mapinfo);
 600                                 LOGE("no free space\n");
 601                                 ret = MM_ERROR_PLAYER_NO_FREE_SPACE;
 602                                 goto ERROR;
 603                         }
 604
 605                         /* U */
 606                         if (proved->size[1]) {
 607                                 player->captured.data[1] = g_try_malloc(planes[1]);
 608
 609                                 if (!player->captured.data[1]) {
 610                                         LOGE("no free space\n");
 611                                         gst_memory_unmap(memory, &mapinfo);
 612                                         ret = MM_ERROR_PLAYER_NO_FREE_SPACE;
 613                                         goto ERROR;
 614                                 }
 615                         }
 616
 617                         /* V */
 618                         if (proved->size[2]) {
 619                                 player->captured.data[2] = g_try_malloc(planes[2]);
 620
 621                                 if (!player->captured.data[2]) {
 622                                         LOGE("no free space\n");
 623                                         gst_memory_unmap(memory, &mapinfo);
 624                                         ret = MM_ERROR_PLAYER_NO_FREE_SPACE;
 625                                         goto ERROR;
 626                                 }
 627                         }
 628
 629                         LOGD("Buffer type %d", proved->type);
 630                         if (proved->type == MM_VIDEO_BUFFER_TYPE_TBM_BO) {
 631                                 tbm_bo_ref(proved->handle.bo[0]);
 632                                 tbm_bo_ref(proved->handle.bo[1]);
 633                                 LOGD("plane[0] : %p, size %d", proved->data[0], planes[0]);
 634                                 if (proved->data[0])
 635                                         memcpy(player->captured.data[0], proved->data[0], planes[0]);
 636
 637                                 LOGD("plane[1] : %p, size %d", proved->data[1], planes[1]);
 638                                 if (proved->data[1])
 639                                         memcpy(player->captured.data[1], proved->data[1], planes[1]);
 640
 641                                 tbm_bo_unref(proved->handle.bo[0]);
 642                                 tbm_bo_unref(proved->handle.bo[1]);
 643
 644                                 if (player->video_cs == MM_PLAYER_COLORSPACE_I420) {
 645                                         tbm_bo_ref(proved->handle.bo[2]);
 646                                         LOGD("plane[2] : %p, size %d", proved->data[2], planes[2]);
 647
 648                                         if (proved->data[2])
 649                                                 memcpy(player->captured.data[2], proved->data[2], planes[2]);
 650
 651                                         tbm_bo_unref(proved->handle.bo[2]);
 652                                 }
 653                         } else {
 654                                 LOGE("Not support video buffer type %d", proved->type);
 655                                 gst_memory_unmap(memory, &mapinfo);
 656                                 ret = MM_ERROR_PLAYER_INTERNAL;
 657                                 goto ERROR;
 658                         }
 659
 660                         gst_memory_unmap(memory, &mapinfo);
 661                         goto DONE;
 662                 } else {
 663                         GstVideoInfo format_info;
 664                         gst_video_info_from_caps(&format_info, caps);
 665
 666                         switch (GST_VIDEO_INFO_FORMAT(&format_info)) {
 667                         case GST_VIDEO_FORMAT_I420:
 668                                 src_fmt = MM_UTIL_IMG_FMT_I420;
 669                                 break;
 670                         case GST_VIDEO_FORMAT_BGRA:
 671                                 src_fmt = MM_UTIL_IMG_FMT_BGRA8888;
 672                                 break;
 673                         case GST_VIDEO_FORMAT_BGRx:
 674                                 src_fmt = MM_UTIL_IMG_FMT_BGRX8888;
 675                                 break;
 676                         default:
 677                                 LOGE("unknown format to capture\n");
 678                                 ret = MM_ERROR_PLAYER_INTERNAL;
 679                                 goto ERROR;
 680                                 break;
 681                         }
 682                 }
 683         } else {
 684                 LOGE("unknown format to capture\n");
 685                 ret = MM_ERROR_PLAYER_INTERNAL;
 686                 goto ERROR;
 687         }
 688
 689         gst_buffer_map(buffer, &mapinfo, GST_MAP_READ);
 690         __mm_player_convert_colorspace(player, mapinfo.data, src_fmt, src_width, src_height, dst_fmt);
 691         gst_buffer_unmap(buffer, &mapinfo);
 692
 693 DONE:
 694         /* do convert colorspace */
 695         MMPLAYER_CAPTURE_THREAD_SIGNAL(player);
 696
 697 ERROR:
 698         if (caps) {
 699                 gst_caps_unref(caps);
 700                 caps = NULL;
 701         }
 702
 703         for (i = 0; i < player->captured.handle_num; i++) {
 704                 MMPLAYER_FREEIF(player->captured.data[i]);
 705         }
 706
 707         MMPLAYER_FLEAVE();
 708         return ret;
 709 }
 710
 711 static GstPadProbeReturn
 712 __mmplayer_video_capture_probe(GstPad *pad, GstPadProbeInfo *info, gpointer u_data)
 713 {
 714         mm_player_t* player = (mm_player_t*) u_data;
 715         GstBuffer *buffer = NULL;
 716         int ret = MM_ERROR_NONE;
 717
 718         MMPLAYER_RETURN_VAL_IF_FAIL(info->data, GST_PAD_PROBE_REMOVE);
 719         MMPLAYER_FENTER();
 720
 721         buffer = gst_pad_probe_info_get_buffer(info);
 722         ret = __mmplayer_get_video_frame_from_buffer(player, pad, buffer);
 723
 724         if (ret != MM_ERROR_NONE) {
 725                 LOGE("failed to get video frame");
 726                 return GST_PAD_PROBE_REMOVE;
 727         }
 728
 729         /* remove probe to be called at one time */
 730         if (player->video_capture_cb_probe_id) {
 731                 gst_pad_remove_probe(pad, player->video_capture_cb_probe_id);
 732                 player->video_capture_cb_probe_id = 0;
 733         }
 734
 735         MMPLAYER_FLEAVE();
 736
 737         return GST_PAD_PROBE_OK;
 738 }
 739
 740 static int
 741 __mm_player_convert_colorspace(mm_player_t* player, unsigned char* src_data, mm_util_img_format src_fmt, unsigned int src_w, unsigned int src_h, mm_util_img_format dst_fmt)
 742 {
 743         unsigned char *dst_data = NULL;
 744         unsigned int dst_size;
 745         int ret = MM_ERROR_NONE;
 746
 747         MMPLAYER_RETURN_VAL_IF_FAIL(player, MM_ERROR_PLAYER_INTERNAL);
 748         ret = mm_util_get_image_size(dst_fmt, src_w, src_h, &dst_size);
 749
 750         if (ret != MM_ERROR_NONE) {
 751                 LOGE("failed to get image size for capture, %d\n", ret);
 752                 return MM_ERROR_PLAYER_INTERNAL;
 753         }
 754
 755         SECURE_LOGD("width: %d, height: %d to capture, dest size: %d\n", src_w, src_h, dst_size);
 756
 757         dst_data = (unsigned char*)g_malloc0(dst_size);
 758
 759         if (!dst_data) {
 760                 LOGE("no free space to capture\n");
 761                 g_free(dst_data);
 762                 return MM_ERROR_PLAYER_NO_FREE_SPACE;
 763         }
 764
 765         ret = mm_util_convert_colorspace(src_data, src_w, src_h, src_fmt, dst_data, dst_fmt);
 766
 767         if (ret != MM_ERROR_NONE) {
 768                 LOGE("failed to convert for capture, %d\n", ret);
 769                 g_free(dst_data);
 770                 return MM_ERROR_PLAYER_INTERNAL;
 771         }
 772
 773         player->capture.size = dst_size;
 774         player->capture.data = dst_data;
 775
 776         return MM_ERROR_NONE;
 777 }
 778
 779 /*
 780  * Get tiled address of position(x,y)
 781  *
 782  * @param x_size
 783  *   width of tiled[in]
 784  *
 785  * @param y_size
 786  *   height of tiled[in]
 787  *
 788  * @param x_pos
 789  *   x position of tield[in]
 790  *
 791  * @param src_size
 792  *   y position of tield[in]
 793  *
 794  * @return
 795  *   address of tiled data
 796  */
 797 static int
 798 __tile_4x2_read(int x_size, int y_size, int x_pos, int y_pos)
 799 {
 800         int pixel_x_m1, pixel_y_m1;
 801         int roundup_x;
 802         int linear_addr0, linear_addr1, bank_addr ;
 803         int x_addr;
 804         int trans_addr;
 805
 806         pixel_x_m1 = x_size -1;
 807         pixel_y_m1 = y_size -1;
 808
 809         roundup_x = ((pixel_x_m1 >> 7) + 1);
 810
 811         x_addr = x_pos >> 2;
 812
 813         if ((y_size <= y_pos+32) && (y_pos < y_size) &&
 814                 (((pixel_y_m1 >> 5) & 0x1) == 0) && (((y_pos >> 5) & 0x1) == 0)) {
 815                 linear_addr0 = (((y_pos & 0x1f) <<4) | (x_addr & 0xf));
 816                 linear_addr1 = (((y_pos >> 6) & 0xff) * roundup_x +((x_addr >> 6) & 0x3f));
 817
 818                 if (((x_addr >> 5) & 0x1) == ((y_pos >> 5) & 0x1))
 819                         bank_addr = ((x_addr >> 4) & 0x1);
 820                 else
 821                         bank_addr = 0x2 | ((x_addr >> 4) & 0x1);
 822         } else {
 823                 linear_addr0 = (((y_pos & 0x1f) << 4) | (x_addr & 0xf));
 824                 linear_addr1 = (((y_pos >> 6) & 0xff) * roundup_x +((x_addr >> 5) & 0x7f));
 825
 826                 if (((x_addr >> 5) & 0x1) == ((y_pos >> 5) & 0x1))
 827                         bank_addr = ((x_addr >> 4) & 0x1);
 828                 else
 829                         bank_addr = 0x2 | ((x_addr >> 4) & 0x1);
 830         }
 831
 832         linear_addr0 = linear_addr0 << 2;
 833         trans_addr = (linear_addr1 <<13) | (bank_addr << 11) | linear_addr0;
 834
 835         return trans_addr;
 836 }
 837
 838 /*
 839  * Converts tiled data to linear
 840  * Crops left, top, right, buttom
 841  * 1. Y of NV12T to Y of YUV420P
 842  * 2. Y of NV12T to Y of YUV420S
 843  * 3. UV of NV12T to UV of YUV420S
 844  *
 845  * @param yuv420_dest
 846  *   Y or UV plane address of YUV420[out]
 847  *
 848  * @param nv12t_src
 849  *   Y or UV plane address of NV12T[in]
 850  *
 851  * @param yuv420_width
 852  *   Width of YUV420[in]
 853  *
 854  * @param yuv420_height
 855  *   Y: Height of YUV420, UV: Height/2 of YUV420[in]
 856  *
 857  * @param left
 858  *   Crop size of left
 859  *
 860  * @param top
 861  *   Crop size of top
 862  *
 863  * @param right
 864  *   Crop size of right
 865  *
 866  * @param buttom
 867  *   Crop size of buttom
 868  */
 869 static void
 870 __csc_tiled_to_linear_crop(unsigned char *yuv420_dest, unsigned char *nv12t_src, int yuv420_width, int yuv420_height,
 871                                                         int left, int top, int right, int buttom)
 872 {
 873         int i, j;
 874         int tiled_offset = 0, tiled_offset1 = 0;
 875         int linear_offset = 0;
 876         int temp1 = 0, temp2 = 0, temp3 = 0, temp4 = 0;
 877
 878         temp3 = yuv420_width-right;
 879         temp1 = temp3-left;
 880         /* real width is greater than or equal 256 */
 881         if (temp1 >= 256) {
 882                 for (i = top; i < yuv420_height-buttom; i = i+1) {
 883                         j = left;
 884                         temp3 = (j>>8)<<8;
 885                         temp3 = temp3>>6;
 886                         temp4 = i>>5;
 887                         if (temp4 & 0x1) {
 888                                 /* odd fomula: 2+x+(x>>2)<<2+x_block_num*(y-1) */
 889                                 tiled_offset = temp4-1;
 890                                 temp1 = ((yuv420_width+127)>>7)<<7;
 891                                 tiled_offset = tiled_offset*(temp1>>6);
 892                                 tiled_offset = tiled_offset+temp3;
 893                                 tiled_offset = tiled_offset+2;
 894                                 temp1 = (temp3>>2)<<2;
 895                                 tiled_offset = tiled_offset+temp1;
 896                                 tiled_offset = tiled_offset<<11;
 897                                 tiled_offset1 = tiled_offset+2048*2;
 898                                 temp4 = 8;
 899                         } else {
 900                                 temp2 = ((yuv420_height+31)>>5)<<5;
 901                                 if ((i+32) < temp2) {
 902                                         /* even1 fomula: x+((x+2)>>2)<<2+x_block_num*y */
 903                                         temp1 = temp3+2;
 904                                         temp1 = (temp1>>2)<<2;
 905                                         tiled_offset = temp3+temp1;
 906                                         temp1 = ((yuv420_width+127)>>7)<<7;
 907                                         tiled_offset = tiled_offset+temp4*(temp1>>6);
 908                                         tiled_offset = tiled_offset<<11;
 909                                         tiled_offset1 = tiled_offset+2048*6;
 910                                         temp4 = 8;
 911                                 } else {
 912                                         /* even2 fomula: x+x_block_num*y */
 913                                         temp1 = ((yuv420_width+127)>>7)<<7;
 914                                         tiled_offset = temp4*(temp1>>6);
 915                                         tiled_offset = tiled_offset+temp3;
 916                                         tiled_offset = tiled_offset<<11;
 917                                         tiled_offset1 = tiled_offset+2048*2;
 918                                         temp4 = 4;
 919                                 }
 920                         }
 921
 922                         temp1 = i&0x1F;
 923                         tiled_offset = tiled_offset+64*(temp1);
 924                         tiled_offset1 = tiled_offset1+64*(temp1);
 925                         temp2 = yuv420_width-left-right;
 926                         linear_offset = temp2*(i-top);
 927                         temp3 = ((j+256)>>8)<<8;
 928                         temp3 = temp3-j;
 929                         temp1 = left&0x3F;
 930                         if (temp3 > 192) {
 931                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset+temp1, 64-temp1);
 932                                 memcpy(yuv420_dest+linear_offset+64-temp1, nv12t_src+tiled_offset+2048, 64);
 933                                 memcpy(yuv420_dest+linear_offset+128-temp1, nv12t_src+tiled_offset1, 64);
 934                                 memcpy(yuv420_dest+linear_offset+192-temp1, nv12t_src+tiled_offset1+2048, 64);
 935                                 linear_offset = linear_offset+256-temp1;
 936                         } else if (temp3 > 128) {
 937                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset+2048+temp1, 64-temp1);
 938                                 memcpy(yuv420_dest+linear_offset+64-temp1, nv12t_src+tiled_offset1, 64);
 939                                 memcpy(yuv420_dest+linear_offset+128-temp1, nv12t_src+tiled_offset1+2048, 64);
 940                                 linear_offset = linear_offset+192-temp1;
 941                         } else if (temp3 > 64) {
 942                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset1+temp1, 64-temp1);
 943                                 memcpy(yuv420_dest+linear_offset+64-temp1, nv12t_src+tiled_offset1+2048, 64);
 944                                 linear_offset = linear_offset+128-temp1;
 945                         } else if (temp3 > 0) {
 946                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset1+2048+temp1, 64-temp1);
 947                                 linear_offset = linear_offset+64-temp1;
 948                         }
 949
 950                         tiled_offset = tiled_offset+temp4*2048;
 951                         j = (left>>8)<<8;
 952                         j = j + 256;
 953                         temp2 = yuv420_width-right-256;
 954                         for (; j <= temp2; j = j+256) {
 955                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
 956                                 tiled_offset1 = tiled_offset1+temp4*2048;
 957                                 memcpy(yuv420_dest+linear_offset+64, nv12t_src+tiled_offset+2048, 64);
 958                                 memcpy(yuv420_dest+linear_offset+128, nv12t_src+tiled_offset1, 64);
 959                                 tiled_offset = tiled_offset+temp4*2048;
 960                                 memcpy(yuv420_dest+linear_offset+192, nv12t_src+tiled_offset1+2048, 64);
 961                                 linear_offset = linear_offset+256;
 962                         }
 963
 964                         tiled_offset1 = tiled_offset1+temp4*2048;
 965                         temp2 = yuv420_width-right-j;
 966                         if (temp2 > 192) {
 967                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
 968                                 memcpy(yuv420_dest+linear_offset+64, nv12t_src+tiled_offset+2048, 64);
 969                                 memcpy(yuv420_dest+linear_offset+128, nv12t_src+tiled_offset1, 64);
 970                                 memcpy(yuv420_dest+linear_offset+192, nv12t_src+tiled_offset1+2048, temp2-192);
 971                         } else if (temp2 > 128) {
 972                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
 973                                 memcpy(yuv420_dest+linear_offset+64, nv12t_src+tiled_offset+2048, 64);
 974                                 memcpy(yuv420_dest+linear_offset+128, nv12t_src+tiled_offset1, temp2-128);
 975                         } else if (temp2 > 64) {
 976                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
 977                                 memcpy(yuv420_dest+linear_offset+64, nv12t_src+tiled_offset+2048, temp2-64);
 978                         } else
 979                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, temp2);
 980                 }
 981         } else if (temp1 >= 64) {
 982                 for (i = top; i < (yuv420_height-buttom); i = i+1) {
 983                         j = left;
 984                         tiled_offset = __tile_4x2_read(yuv420_width, yuv420_height, j, i);
 985                         temp2 = ((j+64)>>6)<<6;
 986                         temp2 = temp2-j;
 987                         linear_offset = temp1*(i-top);
 988                         temp4 = j&0x3;
 989                         tiled_offset = tiled_offset+temp4;
 990                         memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, temp2);
 991                         linear_offset = linear_offset+temp2;
 992                         j = j+temp2;
 993                         if ((j+64) <= temp3) {
 994                                 tiled_offset = __tile_4x2_read(yuv420_width, yuv420_height, j, i);
 995                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
 996                                 linear_offset = linear_offset+64;
 997                                 j = j+64;
 998                         }
 999                         if ((j+64) <= temp3) {
1000                                 tiled_offset = __tile_4x2_read(yuv420_width, yuv420_height, j, i);
1001                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
1002                                 linear_offset = linear_offset+64;
1003                                 j = j+64;
1004                         }
1005                         if (j < temp3) {
1006                                 tiled_offset = __tile_4x2_read(yuv420_width, yuv420_height, j, i);
1007                                 temp2 = temp3-j;
1008                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, temp2);
1009                         }
1010                 }
1011         } else {
1012                 for (i = top; i < (yuv420_height-buttom); i = i+1) {
1013                         linear_offset = temp1*(i-top);
1014                         for (j = left; j < (yuv420_width-right); j = j+2) {
1015                                 tiled_offset = __tile_4x2_read(yuv420_width, yuv420_height, j, i);
1016                                 temp4 = j&0x3;
1017                                 tiled_offset = tiled_offset+temp4;
1018                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 2);
1019                                 linear_offset = linear_offset+2;
1020                         }
1021                 }
1022         }
1023 }