src/mm_player_capture.c

   1 /*
   2  * libmm-player
   3  *
   4  * Copyright(c) 2000 - 2011 Samsung Electronics Co., Ltd. All rights reserved.
   5  *
   6  * Contact: JongHyuk Choi <jhchoi.choi@samsung.com>, YeJin Cho <cho.yejin@samsung.com>,
   7  * Seungbae Shin <seungbae.shin@samsung.com>, YoungHwan An <younghwan_.an@samsung.com>
   8  *
   9  * Licensed under the Apache License, Version 2.0(the "License");
  10  * you may not use this file except in compliance with the License.
  11  * You may obtain a copy of the License at
  12  *
  13  * http://www.apache.org/licenses/LICENSE-2.0
  14  *
  15  * Unless required by applicable law or agreed to in writing, software
  16  * distributed under the License is distributed on an "AS IS" BASIS,
  17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  18  * See the License for the specific language governing permissions and
  19  * limitations under the License.
  20  *
  21  */
  22
  23 /*===========================================================================================
  24 |                                                                                                                                                                                       |
  25 |  INCLUDE FILES                                                                                                                                                        |
  26 |                                                                                                                                                                                       |
  27 ========================================================================================== */
  28 #include <dlog.h>
  29 #include "mm_player_utils.h"
  30 #include "mm_player_capture.h"
  31 #include "mm_player_priv.h"
  32
  33 #include <mm_util_imgp.h>
  34 #include <gst/video/video-info.h>
  35
  36 //#define       CAPTURE_OUTPUT_DUMP     1
  37 /*---------------------------------------------------------------------------
  38 |    LOCAL VARIABLE DEFINITIONS for internal                                                            |
  39 ---------------------------------------------------------------------------*/
  40
  41 /*---------------------------------------------------------------------------
  42 |    LOCAL FUNCTION PROTOTYPES:                                                                                         |
  43 ---------------------------------------------------------------------------*/
  44 static GstPadProbeReturn __mmplayer_video_capture_probe(GstPad *pad, GstPadProbeInfo *info, gpointer u_data);
  45 static int  __mmplayer_get_video_frame_from_buffer(mm_player_t* player, GstPad *pad, GstBuffer *buffer);
  46 static gpointer __mmplayer_capture_thread(gpointer data);
  47 static void __csc_tiled_to_linear_crop(unsigned char *yuv420_dest, unsigned char *nv12t_src, int yuv420_width, int yuv420_height, int left, int top, int right, int buttom);
  48 static int __tile_4x2_read(int x_size, int y_size, int x_pos, int y_pos);
  49 static int __mm_player_convert_colorspace(mm_player_t* player, unsigned char* src_data, mm_util_color_format_e src_fmt, unsigned int src_w, unsigned int src_h, mm_util_color_format_e dst_fmt);
  50 #ifdef CAPTURE_OUTPUT_DUMP
  51 static void capture_output_dump(mm_player_t* player);
  52 #endif
  53
  54 /*===========================================================================================
  55 |                                                                                                                                                                                       |
  56 |  FUNCTION DEFINITIONS                                                                                                                                         |
  57 |                                                                                                                                                                                       |
  58 ========================================================================================== */
  59 int
  60 _mmplayer_initialize_video_capture(mm_player_t* player)
  61 {
  62         int ret = MM_ERROR_NONE;
  63         MMPLAYER_RETURN_VAL_IF_FAIL(player, MM_ERROR_PLAYER_NOT_INITIALIZED);
  64         /* create capture mutex */
  65         g_mutex_init(&player->capture_thread_mutex);
  66
  67         /* create capture cond */
  68         g_cond_init(&player->capture_thread_cond);
  69
  70
  71         player->capture_thread_exit = FALSE;
  72
  73         /* create video capture thread */
  74         player->capture_thread =
  75                         g_thread_try_new("capture_thread", __mmplayer_capture_thread, (gpointer)player, NULL);
  76
  77         if (!player->capture_thread) {
  78                 ret = MM_ERROR_PLAYER_RESOURCE_LIMIT;
  79                 goto ERROR;
  80         }
  81
  82         return ret;
  83
  84 ERROR:
  85         /* capture thread */
  86         g_mutex_clear(&player->capture_thread_mutex);
  87
  88         g_cond_clear(&player->capture_thread_cond);
  89
  90         return ret;
  91 }
  92
  93 int
  94 _mmplayer_release_video_capture(mm_player_t* player)
  95 {
  96         MMPLAYER_RETURN_VAL_IF_FAIL(player, MM_ERROR_PLAYER_NOT_INITIALIZED);
  97         /* release capture thread */
  98         MMPLAYER_CAPTURE_THREAD_LOCK(player);
  99         player->capture_thread_exit = TRUE;
 100         MMPLAYER_CAPTURE_THREAD_SIGNAL(player);
 101         MMPLAYER_CAPTURE_THREAD_UNLOCK(player);
 102
 103         LOGD("waitting for capture thread exit");
 104         g_thread_join(player->capture_thread);
 105         g_mutex_clear(&player->capture_thread_mutex);
 106         g_cond_clear(&player->capture_thread_cond);
 107         LOGD("capture thread released");
 108
 109         return MM_ERROR_NONE;
 110 }
 111
 112 int
 113 _mmplayer_do_video_capture(MMHandleType hplayer)
 114 {
 115         mm_player_t* player = (mm_player_t*) hplayer;
 116         int ret = MM_ERROR_NONE;
 117         GstPad *pad = NULL;
 118
 119         MMPLAYER_FENTER();
 120
 121         MMPLAYER_RETURN_VAL_IF_FAIL(player && player->pipeline, MM_ERROR_PLAYER_NOT_INITIALIZED);
 122
 123         /* capturing or not */
 124         if (player->video_capture_cb_probe_id || player->capture.data
 125                         || player->captured.data[0] || player->captured.data[1]) {
 126                 LOGW("capturing... we can't do any more");
 127                 return MM_ERROR_PLAYER_INVALID_STATE;
 128         }
 129         gint surface_type = 0;
 130         mm_attrs_get_int_by_name(player->attrs, "display_surface_type", &surface_type);
 131
 132         /* check if video pipeline is linked or not */
 133         if (!player->pipeline->videobin) {
 134                 if (surface_type == MM_DISPLAY_SURFACE_REMOTE && player->video_fakesink) {
 135                         LOGD("it is evas surface type.");
 136                 } else {
 137                         LOGW("not ready to capture");
 138                         return MM_ERROR_PLAYER_INVALID_STATE;
 139                 }
 140         }
 141
 142         if (player->pipeline->videobin)
 143                 pad = gst_element_get_static_pad(player->pipeline->videobin[MMPLAYER_V_SINK].gst, "sink");
 144         else // evas surface
 145                 pad = gst_element_get_static_pad(player->video_fakesink, "sink");
 146
 147         if (player->state != MM_PLAYER_STATE_PLAYING) {
 148                 if (player->state == MM_PLAYER_STATE_PAUSED) {
 149                         // get last buffer from video sink
 150                         GstSample *sample = NULL;
 151
 152                         gst_element_get_state(player->pipeline->mainbin[MMPLAYER_M_PIPE].gst, NULL, NULL, 5 * GST_SECOND); //5 seconds
 153
 154                         if (player->pipeline->videobin)
 155                                 g_object_get(player->pipeline->videobin[MMPLAYER_V_SINK].gst, "last-sample", &sample, NULL);
 156                         else // evas surface
 157                                 g_object_get(player->video_fakesink, "last-sample", &sample, NULL);
 158
 159                         if (sample) {
 160                                 GstBuffer *buf = NULL;
 161                                 buf = gst_sample_get_buffer(sample);
 162                                 if (buf) {
 163                                         if (__mmplayer_get_video_frame_from_buffer(player, pad, buf) != MM_ERROR_NONE)
 164                                                 ret = MM_ERROR_PLAYER_INTERNAL;
 165                                 } else {
 166                                         LOGW("failed to get video frame");
 167                                 }
 168                                 gst_sample_unref(sample);
 169                         }
 170                         return ret;
 171                 } else {
 172                         LOGW("invalid state(%d) to capture", player->state);
 173                         gst_object_unref(GST_OBJECT(pad));
 174                         pad = NULL;
 175                         return MM_ERROR_PLAYER_INVALID_STATE;
 176                 }
 177         }
 178
 179         /* register probe */
 180         player->video_capture_cb_probe_id = gst_pad_add_probe(pad, GST_PAD_PROBE_TYPE_BUFFER,
 181                 __mmplayer_video_capture_probe, player, NULL);
 182
 183         gst_object_unref(GST_OBJECT(pad));
 184         pad = NULL;
 185
 186         MMPLAYER_FLEAVE();
 187
 188         return ret;
 189 }
 190
 191 int
 192 __mmplayer_handle_orientation(mm_player_t* player, int orientation, int format)
 193 {
 194         unsigned char *src_buffer = NULL;
 195         int ret = MM_ERROR_NONE;
 196         unsigned char *dst_frame = NULL;
 197         unsigned int dst_width = 0;
 198         unsigned int dst_height = 0;
 199         size_t dst_size = 0;
 200         mm_util_img_rotate_type rot_enum = MM_UTIL_ROTATE_NUM;
 201
 202         player->capture.orientation = orientation;
 203
 204         if (orientation == 90 || orientation == 270) {
 205                 dst_width = player->captured.height[0];
 206                 dst_height = player->captured.width[0];
 207                 LOGD("interchange width & height");
 208         } else if (orientation == 180) {
 209                 dst_width = player->captured.width[0];
 210                 dst_height = player->captured.height[0];
 211         } else if (orientation == 0) {
 212                 LOGE("no need handle orientation : %d", orientation);
 213                 player->capture.width = player->captured.width[0];
 214                 player->capture.height = player->captured.height[0];
 215                 return MM_ERROR_NONE;
 216         } else
 217                 LOGE("wrong orientation value...");
 218
 219         /* height & width will be interchanged for 90 and 270 orientation */
 220         LOGD("before rotation : dst_width = %d and dst_height = %d", dst_width, dst_height);
 221         src_buffer = (unsigned char*)player->capture.data;
 222
 223         /* convert orientation degree into enum here */
 224         switch (orientation) {
 225         case 90:
 226                 rot_enum = MM_UTIL_ROTATE_90;
 227                 break;
 228         case 180:
 229                 rot_enum = MM_UTIL_ROTATE_180;
 230                 break;
 231         case 270:
 232                 rot_enum = MM_UTIL_ROTATE_270;
 233                 break;
 234         default:
 235                 LOGE("wrong rotate value");
 236                 break;
 237         }
 238
 239         LOGD("source buffer for rotation = %p and rotation = %d", src_buffer, rot_enum);
 240
 241         ret = mm_util_rotate_image(src_buffer,
 242                         player->captured.width[0], player->captured.height[0], format, rot_enum,
 243                         &dst_frame, &dst_width, &dst_height, &dst_size);
 244         if (ret != MM_ERROR_NONE || !dst_frame) {
 245                 LOGE("failed to do rotate image");
 246                 free(dst_frame);
 247                 return ret;
 248         }
 249
 250         LOGD("after rotation same stride: dst_width = %d and dst_height = %d, dst_size = %zu", dst_width, dst_height, dst_size);
 251
 252         g_free(src_buffer);
 253
 254         player->capture.data = dst_frame;
 255         player->capture.size = (int)dst_size;
 256         player->capture.orientation = orientation;
 257         player->capture.width = dst_width;
 258         player->capture.height = dst_height;
 259
 260         player->captured.width[0] = player->captured.stride_width[0] = dst_width;
 261         player->captured.height[0] = player->captured.stride_height[0] = dst_height;
 262
 263         return ret;
 264 }
 265
 266
 267 static gpointer
 268 __mmplayer_capture_thread(gpointer data)
 269 {
 270         mm_player_t* player = (mm_player_t*) data;
 271         MMMessageParamType msg = {0, };
 272         unsigned char * src_buffer = NULL;
 273         unsigned char * linear_y_plane = NULL;
 274         unsigned char * linear_uv_plane = NULL;
 275         int orientation = 0;
 276         int user_angle = 0;
 277         int ret = 0;
 278         int planes[MAX_BUFFER_PLANE] = {0, };
 279         unsigned char * p_buf = NULL;
 280         unsigned char * temp = NULL;
 281         int i, j;
 282
 283         MMPLAYER_RETURN_VAL_IF_FAIL(player, NULL);
 284
 285         MMPLAYER_CAPTURE_THREAD_LOCK(player);
 286         while (!player->capture_thread_exit) {
 287                 LOGD("capture thread started. waiting for signal");
 288                 MMPLAYER_CAPTURE_THREAD_WAIT(player);
 289
 290                 if (player->capture_thread_exit) {
 291                         LOGD("exiting capture thread");
 292                         goto EXIT;
 293                 }
 294                 LOGD("capture thread is received signal");
 295
 296                 /* NOTE: Don't use MMPLAYER_CMD_LOCK() here.
 297                  * Because deadlock can be happened if other player api is used in message callback.
 298                  */
 299                 if (player->video_cs == MM_PLAYER_COLORSPACE_NV12_TILED) {
 300                         /* Colorspace conversion : NV12T-> NV12-> RGB888 */
 301                         int ret = 0;
 302                         int linear_y_plane_size;
 303                         int linear_uv_plane_size;
 304                         int width = player->captured.width[0];
 305                         int height = player->captured.height[0];
 306
 307                         linear_y_plane_size = (width * height);
 308                         linear_uv_plane_size = (width * height / 2);
 309
 310                         linear_y_plane = (unsigned char*) g_try_malloc(linear_y_plane_size);
 311                         if (linear_y_plane == NULL) {
 312                                 msg.code = MM_ERROR_PLAYER_NO_FREE_SPACE;
 313                                 goto ERROR;
 314                         }
 315
 316                         linear_uv_plane = (unsigned char*) g_try_malloc(linear_uv_plane_size);
 317                         if (linear_uv_plane == NULL) {
 318                                 msg.code = MM_ERROR_PLAYER_NO_FREE_SPACE;
 319                                 goto ERROR;
 320                         }
 321                         /* NV12 tiled to linear */
 322                         __csc_tiled_to_linear_crop(linear_y_plane,
 323                                         player->captured.data[0], width, height, 0, 0, 0, 0);
 324                         __csc_tiled_to_linear_crop(linear_uv_plane,
 325                                         player->captured.data[1], width, height / 2, 0, 0, 0, 0);
 326
 327                         for (i = 0; i < player->captured.handle_num; i++) {
 328                                 MMPLAYER_FREEIF(player->captured.data[i]);
 329                         }
 330
 331                         src_buffer = (unsigned char*) g_try_malloc(linear_y_plane_size+linear_uv_plane_size);
 332
 333                         if (src_buffer == NULL) {
 334                                 msg.code = MM_ERROR_PLAYER_NO_FREE_SPACE;
 335                                 goto ERROR;
 336                         }
 337                         memset(src_buffer, 0x00, linear_y_plane_size+linear_uv_plane_size);
 338                         memcpy(src_buffer, linear_y_plane, linear_y_plane_size);
 339                         memcpy(src_buffer+linear_y_plane_size, linear_uv_plane, linear_uv_plane_size);
 340
 341                         /* NV12 linear to RGB888 */
 342                         ret = __mm_player_convert_colorspace(player, src_buffer, MM_UTIL_COLOR_NV12,
 343                                 width, height, MM_UTIL_COLOR_RGB24);
 344
 345                         if (ret != MM_ERROR_NONE) {
 346                                 LOGE("failed to convert nv12 linear");
 347                                 goto ERROR;
 348                         }
 349                         /* clean */
 350                         MMPLAYER_FREEIF(src_buffer);
 351                         MMPLAYER_FREEIF(linear_y_plane);
 352                         MMPLAYER_FREEIF(linear_uv_plane);
 353                 } else if (MM_PLAYER_COLORSPACE_NV12 == player->video_cs) {
 354                         #define MM_ALIGN(x, a)      (((x) +(a) - 1) & ~((a) - 1))
 355                         int ret = 0;
 356                         /* using original width otherwises, app can't know aligned to resize */
 357                         planes[0] =  player->captured.stride_width[0] * player->captured.stride_height[0];
 358                         planes[1] = player->captured.stride_width[1] * player->captured.stride_height[1];
 359                         int src_buffer_size = planes[0] + planes[1];
 360                         src_buffer = (unsigned char*) g_try_malloc(src_buffer_size);
 361                         p_buf = src_buffer;
 362
 363                         if (!src_buffer_size) {
 364                                 LOGE("invalid data size");
 365                                 goto ERROR;
 366                         }
 367
 368                         if (!src_buffer) {
 369                                 msg.code = MM_ERROR_PLAYER_NO_FREE_SPACE;
 370                                 goto ERROR;
 371                         }
 372
 373                         memset(src_buffer, 0x00, src_buffer_size);
 374
 375                         temp = player->captured.data[0];
 376
 377                         /* set Y plane */
 378                         for (i = 0; i < player->captured.height[0]; i++) {
 379                                 memcpy(p_buf, temp, player->captured.width[0]);
 380                                 p_buf += player->captured.width[0];
 381                                 temp += player->captured.stride_width[0];
 382                         }
 383
 384                         temp = player->captured.data[1];
 385
 386                         /* set UV plane*/
 387                         for (j = 0; j < player->captured.height[1]; j++) {
 388                                 memcpy(p_buf, temp, player->captured.width[1]);
 389                                 p_buf += player->captured.width[1];
 390                                 temp += player->captured.stride_width[1];
 391                         }
 392
 393                         /* NV12 -> RGB888 */
 394                         ret = __mm_player_convert_colorspace(player, (unsigned char*)src_buffer, MM_UTIL_COLOR_NV12,
 395                                 player->captured.width[0], player->captured.height[0], MM_UTIL_COLOR_RGB24);
 396                         if (ret != MM_ERROR_NONE) {
 397                                 LOGE("failed to convert nv12 linear");
 398                                 goto ERROR;
 399                         }
 400 #ifdef CAPTURE_OUTPUT_DUMP
 401                         capture_output_dump(player);
 402 #endif
 403                         /* clean */
 404                         MMPLAYER_FREEIF(src_buffer);
 405                         /* free captured buf */
 406                         for (i = 0; i < player->captured.handle_num; i++) {
 407                                 MMPLAYER_FREEIF(player->captured.data[i]);
 408                         }
 409                 } else if (MM_PLAYER_COLORSPACE_I420 == player->video_cs) {
 410                         planes[0] = player->captured.stride_width[0] * player->captured.stride_height[0];
 411                         planes[1] = planes[2] = (player->captured.stride_width[0]>>1) * (player->captured.stride_height[0]>>1);
 412
 413                         src_buffer = (unsigned char*) g_try_malloc(player->captured.stride_width[0] * player->captured.stride_height[0]*3/2);
 414                         p_buf = src_buffer;
 415                         /* set Y plane */
 416                         memset(p_buf, 0x00, planes[0]);
 417                         temp = player->captured.data[0];
 418
 419                         for (i = 0; i < player->captured.height[0]; i++) {
 420                                 memcpy(p_buf, temp, player->captured.width[0]);
 421                                 temp += player->captured.stride_width[0];
 422                                 p_buf += player->captured.width[0];
 423                         }
 424
 425                         /* set U plane */
 426                         memset(p_buf, 0x00, planes[1]);
 427                         temp = player->captured.data[1];
 428
 429                         for (i = 0; i < player->captured.height[1]; i++) {
 430                                 memcpy(p_buf, temp, player->captured.width[1]);
 431                                 temp += player->captured.stride_width[1];
 432                                 p_buf += player->captured.width[1];
 433                         }
 434
 435                         /* set V plane */
 436                         memset(p_buf, 0x00, planes[2]);
 437                         temp = player->captured.data[2];
 438
 439                         for (i = 0; i < player->captured.height[2]; i++) {
 440                                 memcpy(p_buf, temp, player->captured.width[2]);
 441                                 temp += player->captured.stride_width[2];
 442                                 p_buf += player->captured.width[2];
 443                         }
 444
 445                         /* I420 -> RGB888 */
 446                         ret = __mm_player_convert_colorspace(player, (unsigned char*)src_buffer, MM_UTIL_COLOR_I420,
 447                                 player->captured.width[0], player->captured.height[0], MM_UTIL_COLOR_RGB24);
 448                         if (ret != MM_ERROR_NONE) {
 449                                 LOGE("failed to convert I420 linear");
 450                                 goto ERROR;
 451                         }
 452 #ifdef CAPTURE_OUTPUT_DUMP
 453                         capture_output_dump(player);
 454 #endif
 455                         /* clean */
 456                         MMPLAYER_FREEIF(src_buffer);
 457                         /* free captured buf */
 458                         for (i = 0; i < player->captured.handle_num; i++) {
 459                                 MMPLAYER_FREEIF(player->captured.data[i]);
 460                         }
 461                 }
 462
 463                 ret = __mmplayer_get_video_angle((MMHandleType)player, &user_angle, &orientation);
 464                 if (ret != MM_ERROR_NONE) {
 465                         LOGE("failed to get rotation angle");
 466                         goto ERROR;
 467                 }
 468
 469                 LOGD("orientation value = %d user_angle = %d", orientation, user_angle);
 470
 471                 ret = __mmplayer_handle_orientation(player, orientation, MM_UTIL_COLOR_RGB24);
 472                 if (ret != MM_ERROR_NONE) {
 473                         LOGE("failed to convert nv12 linear");
 474                         goto ERROR;
 475                 }
 476
 477                 player->capture.fmt = MM_PLAYER_COLORSPACE_RGB888;
 478                 msg.data = &player->capture;
 479                 msg.size = player->capture.size;
 480 //              msg.captured_frame.width = player->capture.width;
 481 //              msg.captured_frame.height = player->capture.height;
 482 //              msg.captured_frame.orientation = player->capture.orientation;
 483
 484                 if (player->cmd >= MMPLAYER_COMMAND_START) {
 485                         MMPLAYER_POST_MSG(player, MM_MESSAGE_VIDEO_CAPTURED, &msg);
 486                         LOGD("returned from capture message callback");
 487                 }
 488
 489                 //MMPLAYER_FREEIF(player->capture.data);
 490                 continue;
 491 ERROR:
 492                 MMPLAYER_FREEIF(src_buffer);
 493
 494                 for (i = 0; i < player->captured.handle_num; i++) {
 495                         MMPLAYER_FREEIF(player->captured.data[i]);
 496                 }
 497
 498                 if (player->video_cs == MM_PLAYER_COLORSPACE_NV12_TILED) {
 499                         /* clean */
 500                         MMPLAYER_FREEIF(linear_y_plane);
 501                         MMPLAYER_FREEIF(linear_uv_plane);
 502                 }
 503
 504                 msg.union_type = MM_MSG_UNION_CODE;
 505
 506                 MMPLAYER_POST_MSG(player, MM_MESSAGE_VIDEO_NOT_CAPTURED, &msg);
 507         }
 508
 509 EXIT:
 510         MMPLAYER_CAPTURE_THREAD_UNLOCK(player);
 511         return NULL;
 512 }
 513
 514 /**
 515   * The output is fixed as RGB888
 516   */
 517 static int
 518 __mmplayer_get_video_frame_from_buffer(mm_player_t* player, GstPad *pad, GstBuffer *buffer)
 519 {
 520         int ret = MM_ERROR_NONE;
 521         gint planes[MAX_BUFFER_PLANE] = {0, };
 522         gint i = 0;
 523         gint src_width = 0;
 524         gint src_height = 0;
 525         GstCaps *caps = NULL;
 526         GstStructure *structure = NULL;
 527         GstMapInfo mapinfo = GST_MAP_INFO_INIT;
 528         GstMemory *memory = NULL;
 529         mm_util_color_format_e src_fmt = MM_UTIL_COLOR_YUV420;
 530         mm_util_color_format_e dst_fmt = MM_UTIL_COLOR_RGB24; // fixed
 531
 532         MMPLAYER_FENTER();
 533
 534         MMPLAYER_RETURN_VAL_IF_FAIL(player, MM_ERROR_PLAYER_NOT_INITIALIZED);
 535         MMPLAYER_RETURN_VAL_IF_FAIL(buffer, MM_ERROR_INVALID_ARGUMENT);
 536
 537         /* get fourcc */
 538         caps = gst_pad_get_current_caps(pad);
 539
 540         MMPLAYER_RETURN_VAL_IF_FAIL(caps, MM_ERROR_INVALID_ARGUMENT);
 541         MMPLAYER_LOG_GST_CAPS_TYPE(caps);
 542
 543         structure = gst_caps_get_structure(caps, 0);
 544         if (!structure) {
 545                 LOGE("cannot get structure from caps.\n");
 546                 ret = MM_ERROR_PLAYER_INTERNAL;
 547                 goto ERROR;
 548         }
 549
 550         /* init capture image buffer */
 551         memset(&player->capture, 0x00, sizeof(MMPlayerVideoCapture));
 552
 553         gst_structure_get_int(structure, "width", &src_width);
 554         gst_structure_get_int(structure, "height", &src_height);
 555
 556         /* check rgb or yuv */
 557         if (gst_structure_has_name(structure, "video/x-raw")) {
 558                 /* NV12T */
 559                 const gchar *gst_format = gst_structure_get_string(structure, "format");
 560
 561                 if (!g_strcmp0(gst_format, "ST12"))
 562                         player->video_cs = MM_PLAYER_COLORSPACE_NV12_TILED;
 563                 else if (!g_strcmp0(gst_format, "S420"))
 564                         player->video_cs = MM_PLAYER_COLORSPACE_I420;
 565                 else if (!g_strcmp0(gst_format, "SN12"))
 566                         player->video_cs = MM_PLAYER_COLORSPACE_NV12;
 567                 else if (!g_strcmp0(gst_format, "BGRx"))
 568                         player->video_cs = MM_PLAYER_COLORSPACE_BGRx;
 569                 else
 570                         player->video_cs = MM_PLAYER_COLORSPACE_MAX;
 571
 572                 LOGI("captured format is %s\n", gst_format);
 573
 574                 if (!g_strcmp0(gst_format, "ST12") || !g_strcmp0(gst_format, "SN12")
 575                         || !g_strcmp0(gst_format, "S420")) {
 576                         guint n;
 577                         MMVideoBuffer *proved = NULL;
 578
 579                         /* get video frame info from proved buffer */
 580                         n = gst_buffer_n_memory(buffer);
 581                         memory = gst_buffer_peek_memory(buffer, n-1);
 582                         gst_memory_map(memory, &mapinfo, GST_MAP_READ);
 583                         proved = (MMVideoBuffer *)mapinfo.data;
 584
 585                         if (!proved || !proved->data[0] || !proved->data[1]) {
 586                                 LOGE("fail to gst_memory_map");
 587                                 ret = MM_ERROR_PLAYER_INTERNAL;
 588                                 goto ERROR;
 589                         }
 590
 591                         memcpy(&player->captured, proved, sizeof(MMVideoBuffer));
 592
 593                         planes[0] = proved->size[0];
 594                         planes[1] = proved->size[1];
 595                         planes[2] = proved->size[2];
 596
 597                         /* Y */
 598                         player->captured.data[0] = g_try_malloc(planes[0]);
 599                         if (!player->captured.data[0]) {
 600                                 gst_memory_unmap(memory, &mapinfo);
 601                                 LOGE("no free space\n");
 602                                 ret = MM_ERROR_PLAYER_NO_FREE_SPACE;
 603                                 goto ERROR;
 604                         }
 605
 606                         /* U */
 607                         if (proved->size[1]) {
 608                                 player->captured.data[1] = g_try_malloc(planes[1]);
 609
 610                                 if (!player->captured.data[1]) {
 611                                         LOGE("no free space\n");
 612                                         gst_memory_unmap(memory, &mapinfo);
 613                                         ret = MM_ERROR_PLAYER_NO_FREE_SPACE;
 614                                         goto ERROR;
 615                                 }
 616                         }
 617
 618                         /* V */
 619                         if (proved->size[2]) {
 620                                 player->captured.data[2] = g_try_malloc(planes[2]);
 621
 622                                 if (!player->captured.data[2]) {
 623                                         LOGE("no free space\n");
 624                                         gst_memory_unmap(memory, &mapinfo);
 625                                         ret = MM_ERROR_PLAYER_NO_FREE_SPACE;
 626                                         goto ERROR;
 627                                 }
 628                         }
 629
 630                         LOGD("Buffer type %d", proved->type);
 631                         if (proved->type == MM_VIDEO_BUFFER_TYPE_TBM_BO) {
 632
 633                                 for (i = 0; i < player->captured.handle_num; i++) {
 634                                         tbm_bo_ref(proved->handle.bo[i]);
 635                                 }
 636                                 LOGD("plane[0] : %p, size %d", proved->data[0], planes[0]);
 637
 638                                 if (proved->data[0])
 639                                         memcpy(player->captured.data[0], proved->data[0], planes[0]);
 640
 641                                 LOGD("plane[1] : %p, size %d", proved->data[1], planes[1]);
 642
 643                                 if (proved->data[1])
 644                                         memcpy(player->captured.data[1], proved->data[1], planes[1]);
 645
 646                                 if (player->video_cs == MM_PLAYER_COLORSPACE_I420) {
 647                                         LOGD("plane[2] : %p, size %d", proved->data[2], planes[2]);
 648
 649                                         if (proved->data[2])
 650                                                 memcpy(player->captured.data[2], proved->data[2], planes[2]);
 651                                 }
 652
 653                                 for (i = 0; i < player->captured.handle_num; i++) {
 654                                         tbm_bo_unref(proved->handle.bo[i]);
 655                                 }
 656                         } else {
 657                                 LOGE("Not support video buffer type %d", proved->type);
 658                                 gst_memory_unmap(memory, &mapinfo);
 659                                 ret = MM_ERROR_PLAYER_INTERNAL;
 660                                 goto ERROR;
 661                         }
 662
 663                         gst_memory_unmap(memory, &mapinfo);
 664                         goto DONE;
 665                 } else {
 666                         GstVideoInfo format_info;
 667                         gst_video_info_from_caps(&format_info, caps);
 668
 669                         player->captured.width[0] = src_width;
 670                         player->captured.height[0] = src_height;
 671
 672                         switch (GST_VIDEO_INFO_FORMAT(&format_info)) {
 673                         case GST_VIDEO_FORMAT_I420:
 674                                 src_fmt = MM_UTIL_COLOR_I420;
 675                                 player->captured.width[1] = player->captured.width[2] = src_width>>1;
 676                                 player->captured.height[1] = player->captured.width[2] = src_height>>1;
 677                                 break;
 678                         case GST_VIDEO_FORMAT_BGRA:
 679                                 src_fmt = MM_UTIL_COLOR_BGRA;
 680                                 break;
 681                         case GST_VIDEO_FORMAT_BGRx:
 682                                 src_fmt = MM_UTIL_COLOR_BGRX;
 683                                 break;
 684                         default:
 685                                 LOGE("unknown format to capture\n");
 686                                 ret = MM_ERROR_PLAYER_INTERNAL;
 687                                 goto ERROR;
 688                                 break;
 689                         }
 690                 }
 691         } else {
 692                 LOGE("unknown format to capture\n");
 693                 ret = MM_ERROR_PLAYER_INTERNAL;
 694                 goto ERROR;
 695         }
 696
 697         gst_buffer_map(buffer, &mapinfo, GST_MAP_READ);
 698         __mm_player_convert_colorspace(player, mapinfo.data, src_fmt, src_width, src_height, dst_fmt);
 699         gst_buffer_unmap(buffer, &mapinfo);
 700
 701 DONE:
 702         /* do convert colorspace */
 703         MMPLAYER_CAPTURE_THREAD_SIGNAL(player);
 704
 705         if (caps) {
 706                 gst_caps_unref(caps);
 707                 caps = NULL;
 708         }
 709
 710         return ret;
 711
 712 ERROR:
 713
 714         if (caps) {
 715                 gst_caps_unref(caps);
 716                 caps = NULL;
 717         }
 718
 719         for (i = 0; i < player->captured.handle_num; i++) {
 720                 MMPLAYER_FREEIF(player->captured.data[i]);
 721         }
 722
 723         MMPLAYER_FLEAVE();
 724         return ret;
 725 }
 726
 727 static GstPadProbeReturn
 728 __mmplayer_video_capture_probe(GstPad *pad, GstPadProbeInfo *info, gpointer u_data)
 729 {
 730         mm_player_t* player = (mm_player_t*) u_data;
 731         GstBuffer *buffer = NULL;
 732         int ret = MM_ERROR_NONE;
 733
 734         MMPLAYER_RETURN_VAL_IF_FAIL(info->data, GST_PAD_PROBE_REMOVE);
 735         MMPLAYER_FENTER();
 736
 737         buffer = gst_pad_probe_info_get_buffer(info);
 738         ret = __mmplayer_get_video_frame_from_buffer(player, pad, buffer);
 739
 740         if (ret != MM_ERROR_NONE) {
 741                 LOGE("failed to get video frame");
 742                 return GST_PAD_PROBE_REMOVE;
 743         }
 744
 745         /* remove probe to be called at one time */
 746         if (player->video_capture_cb_probe_id) {
 747                 gst_pad_remove_probe(pad, player->video_capture_cb_probe_id);
 748                 player->video_capture_cb_probe_id = 0;
 749         }
 750
 751         MMPLAYER_FLEAVE();
 752
 753         return GST_PAD_PROBE_OK;
 754 }
 755
 756 static int
 757 __mm_player_convert_colorspace(mm_player_t* player, unsigned char* src_data, mm_util_color_format_e src_fmt, unsigned int src_w, unsigned int src_h, mm_util_color_format_e dst_fmt)
 758 {
 759         unsigned char *dst_data = NULL;
 760         unsigned int dst_width = 0;
 761         unsigned int dst_height = 0;
 762         size_t dst_size = 0;
 763         int ret = MM_ERROR_NONE;
 764
 765         MMPLAYER_RETURN_VAL_IF_FAIL(player, MM_ERROR_PLAYER_INTERNAL);
 766         SECURE_LOGD("src size info. width: %d, height: %d \n", src_w, src_h);
 767
 768         ret = mm_util_convert_colorspace(src_data, src_w, src_h, src_fmt, dst_fmt, &dst_data, &dst_width, &dst_height, &dst_size);
 769         if (ret != MM_ERROR_NONE || !dst_data) {
 770                 LOGE("failed to convert for capture, %d\n", ret);
 771                 g_free(dst_data);
 772                 return MM_ERROR_PLAYER_INTERNAL;
 773         }
 774         SECURE_LOGD("dst size info. width: %d, height: %d, size: %zu\n", dst_width, dst_height, dst_size);
 775
 776         player->capture.size = (int)dst_size;
 777         player->capture.data = dst_data;
 778
 779         return MM_ERROR_NONE;
 780 }
 781
 782 /*
 783  * Get tiled address of position(x,y)
 784  *
 785  * @param x_size
 786  *   width of tiled[in]
 787  *
 788  * @param y_size
 789  *   height of tiled[in]
 790  *
 791  * @param x_pos
 792  *   x position of tield[in]
 793  *
 794  * @param src_size
 795  *   y position of tield[in]
 796  *
 797  * @return
 798  *   address of tiled data
 799  */
 800 static int
 801 __tile_4x2_read(int x_size, int y_size, int x_pos, int y_pos)
 802 {
 803         int pixel_x_m1, pixel_y_m1;
 804         int roundup_x;
 805         int linear_addr0, linear_addr1, bank_addr ;
 806         int x_addr;
 807         int trans_addr;
 808
 809         pixel_x_m1 = x_size -1;
 810         pixel_y_m1 = y_size -1;
 811
 812         roundup_x = ((pixel_x_m1 >> 7) + 1);
 813
 814         x_addr = x_pos >> 2;
 815
 816         if ((y_size <= y_pos+32) && (y_pos < y_size) &&
 817                 (((pixel_y_m1 >> 5) & 0x1) == 0) && (((y_pos >> 5) & 0x1) == 0)) {
 818                 linear_addr0 = (((y_pos & 0x1f) <<4) | (x_addr & 0xf));
 819                 linear_addr1 = (((y_pos >> 6) & 0xff) * roundup_x +((x_addr >> 6) & 0x3f));
 820
 821                 if (((x_addr >> 5) & 0x1) == ((y_pos >> 5) & 0x1))
 822                         bank_addr = ((x_addr >> 4) & 0x1);
 823                 else
 824                         bank_addr = 0x2 | ((x_addr >> 4) & 0x1);
 825         } else {
 826                 linear_addr0 = (((y_pos & 0x1f) << 4) | (x_addr & 0xf));
 827                 linear_addr1 = (((y_pos >> 6) & 0xff) * roundup_x +((x_addr >> 5) & 0x7f));
 828
 829                 if (((x_addr >> 5) & 0x1) == ((y_pos >> 5) & 0x1))
 830                         bank_addr = ((x_addr >> 4) & 0x1);
 831                 else
 832                         bank_addr = 0x2 | ((x_addr >> 4) & 0x1);
 833         }
 834
 835         linear_addr0 = linear_addr0 << 2;
 836         trans_addr = (linear_addr1 <<13) | (bank_addr << 11) | linear_addr0;
 837
 838         return trans_addr;
 839 }
 840
 841 /*
 842  * Converts tiled data to linear
 843  * Crops left, top, right, buttom
 844  * 1. Y of NV12T to Y of YUV420P
 845  * 2. Y of NV12T to Y of YUV420S
 846  * 3. UV of NV12T to UV of YUV420S
 847  *
 848  * @param yuv420_dest
 849  *   Y or UV plane address of YUV420[out]
 850  *
 851  * @param nv12t_src
 852  *   Y or UV plane address of NV12T[in]
 853  *
 854  * @param yuv420_width
 855  *   Width of YUV420[in]
 856  *
 857  * @param yuv420_height
 858  *   Y: Height of YUV420, UV: Height/2 of YUV420[in]
 859  *
 860  * @param left
 861  *   Crop size of left
 862  *
 863  * @param top
 864  *   Crop size of top
 865  *
 866  * @param right
 867  *   Crop size of right
 868  *
 869  * @param buttom
 870  *   Crop size of buttom
 871  */
 872 static void
 873 __csc_tiled_to_linear_crop(unsigned char *yuv420_dest, unsigned char *nv12t_src, int yuv420_width, int yuv420_height,
 874                                                         int left, int top, int right, int buttom)
 875 {
 876         int i, j;
 877         int tiled_offset = 0, tiled_offset1 = 0;
 878         int linear_offset = 0;
 879         int temp1 = 0, temp2 = 0, temp3 = 0, temp4 = 0;
 880
 881         temp3 = yuv420_width-right;
 882         temp1 = temp3-left;
 883         /* real width is greater than or equal 256 */
 884         if (temp1 >= 256) {
 885                 for (i = top; i < yuv420_height-buttom; i = i+1) {
 886                         j = left;
 887                         temp3 = (j>>8)<<8;
 888                         temp3 = temp3>>6;
 889                         temp4 = i>>5;
 890                         if (temp4 & 0x1) {
 891                                 /* odd fomula: 2+x+(x>>2)<<2+x_block_num*(y-1) */
 892                                 tiled_offset = temp4-1;
 893                                 temp1 = ((yuv420_width+127)>>7)<<7;
 894                                 tiled_offset = tiled_offset*(temp1>>6);
 895                                 tiled_offset = tiled_offset+temp3;
 896                                 tiled_offset = tiled_offset+2;
 897                                 temp1 = (temp3>>2)<<2;
 898                                 tiled_offset = tiled_offset+temp1;
 899                                 tiled_offset = tiled_offset<<11;
 900                                 tiled_offset1 = tiled_offset+2048*2;
 901                                 temp4 = 8;
 902                         } else {
 903                                 temp2 = ((yuv420_height+31)>>5)<<5;
 904                                 if ((i+32) < temp2) {
 905                                         /* even1 fomula: x+((x+2)>>2)<<2+x_block_num*y */
 906                                         temp1 = temp3+2;
 907                                         temp1 = (temp1>>2)<<2;
 908                                         tiled_offset = temp3+temp1;
 909                                         temp1 = ((yuv420_width+127)>>7)<<7;
 910                                         tiled_offset = tiled_offset+temp4*(temp1>>6);
 911                                         tiled_offset = tiled_offset<<11;
 912                                         tiled_offset1 = tiled_offset+2048*6;
 913                                         temp4 = 8;
 914                                 } else {
 915                                         /* even2 fomula: x+x_block_num*y */
 916                                         temp1 = ((yuv420_width+127)>>7)<<7;
 917                                         tiled_offset = temp4*(temp1>>6);
 918                                         tiled_offset = tiled_offset+temp3;
 919                                         tiled_offset = tiled_offset<<11;
 920                                         tiled_offset1 = tiled_offset+2048*2;
 921                                         temp4 = 4;
 922                                 }
 923                         }
 924
 925                         temp1 = i&0x1F;
 926                         tiled_offset = tiled_offset+64*(temp1);
 927                         tiled_offset1 = tiled_offset1+64*(temp1);
 928                         temp2 = yuv420_width-left-right;
 929                         linear_offset = temp2*(i-top);
 930                         temp3 = ((j+256)>>8)<<8;
 931                         temp3 = temp3-j;
 932                         temp1 = left&0x3F;
 933                         if (temp3 > 192) {
 934                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset+temp1, 64-temp1);
 935                                 memcpy(yuv420_dest+linear_offset+64-temp1, nv12t_src+tiled_offset+2048, 64);
 936                                 memcpy(yuv420_dest+linear_offset+128-temp1, nv12t_src+tiled_offset1, 64);
 937                                 memcpy(yuv420_dest+linear_offset+192-temp1, nv12t_src+tiled_offset1+2048, 64);
 938                                 linear_offset = linear_offset+256-temp1;
 939                         } else if (temp3 > 128) {
 940                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset+2048+temp1, 64-temp1);
 941                                 memcpy(yuv420_dest+linear_offset+64-temp1, nv12t_src+tiled_offset1, 64);
 942                                 memcpy(yuv420_dest+linear_offset+128-temp1, nv12t_src+tiled_offset1+2048, 64);
 943                                 linear_offset = linear_offset+192-temp1;
 944                         } else if (temp3 > 64) {
 945                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset1+temp1, 64-temp1);
 946                                 memcpy(yuv420_dest+linear_offset+64-temp1, nv12t_src+tiled_offset1+2048, 64);
 947                                 linear_offset = linear_offset+128-temp1;
 948                         } else if (temp3 > 0) {
 949                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset1+2048+temp1, 64-temp1);
 950                                 linear_offset = linear_offset+64-temp1;
 951                         }
 952
 953                         tiled_offset = tiled_offset+temp4*2048;
 954                         j = (left>>8)<<8;
 955                         j = j + 256;
 956                         temp2 = yuv420_width-right-256;
 957                         for (; j <= temp2; j = j+256) {
 958                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
 959                                 tiled_offset1 = tiled_offset1+temp4*2048;
 960                                 memcpy(yuv420_dest+linear_offset+64, nv12t_src+tiled_offset+2048, 64);
 961                                 memcpy(yuv420_dest+linear_offset+128, nv12t_src+tiled_offset1, 64);
 962                                 tiled_offset = tiled_offset+temp4*2048;
 963                                 memcpy(yuv420_dest+linear_offset+192, nv12t_src+tiled_offset1+2048, 64);
 964                                 linear_offset = linear_offset+256;
 965                         }
 966
 967                         tiled_offset1 = tiled_offset1+temp4*2048;
 968                         temp2 = yuv420_width-right-j;
 969                         if (temp2 > 192) {
 970                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
 971                                 memcpy(yuv420_dest+linear_offset+64, nv12t_src+tiled_offset+2048, 64);
 972                                 memcpy(yuv420_dest+linear_offset+128, nv12t_src+tiled_offset1, 64);
 973                                 memcpy(yuv420_dest+linear_offset+192, nv12t_src+tiled_offset1+2048, temp2-192);
 974                         } else if (temp2 > 128) {
 975                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
 976                                 memcpy(yuv420_dest+linear_offset+64, nv12t_src+tiled_offset+2048, 64);
 977                                 memcpy(yuv420_dest+linear_offset+128, nv12t_src+tiled_offset1, temp2-128);
 978                         } else if (temp2 > 64) {
 979                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
 980                                 memcpy(yuv420_dest+linear_offset+64, nv12t_src+tiled_offset+2048, temp2-64);
 981                         } else
 982                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, temp2);
 983                 }
 984         } else if (temp1 >= 64) {
 985                 for (i = top; i < (yuv420_height-buttom); i = i+1) {
 986                         j = left;
 987                         tiled_offset = __tile_4x2_read(yuv420_width, yuv420_height, j, i);
 988                         temp2 = ((j+64)>>6)<<6;
 989                         temp2 = temp2-j;
 990                         linear_offset = temp1*(i-top);
 991                         temp4 = j&0x3;
 992                         tiled_offset = tiled_offset+temp4;
 993                         memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, temp2);
 994                         linear_offset = linear_offset+temp2;
 995                         j = j+temp2;
 996                         if ((j+64) <= temp3) {
 997                                 tiled_offset = __tile_4x2_read(yuv420_width, yuv420_height, j, i);
 998                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
 999                                 linear_offset = linear_offset+64;
1000                                 j = j+64;
1001                         }
1002                         if ((j+64) <= temp3) {
1003                                 tiled_offset = __tile_4x2_read(yuv420_width, yuv420_height, j, i);
1004                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 64);
1005                                 linear_offset = linear_offset+64;
1006                                 j = j+64;
1007                         }
1008                         if (j < temp3) {
1009                                 tiled_offset = __tile_4x2_read(yuv420_width, yuv420_height, j, i);
1010                                 temp2 = temp3-j;
1011                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, temp2);
1012                         }
1013                 }
1014         } else {
1015                 for (i = top; i < (yuv420_height-buttom); i = i+1) {
1016                         linear_offset = temp1*(i-top);
1017                         for (j = left; j < (yuv420_width-right); j = j+2) {
1018                                 tiled_offset = __tile_4x2_read(yuv420_width, yuv420_height, j, i);
1019                                 temp4 = j&0x3;
1020                                 tiled_offset = tiled_offset+temp4;
1021                                 memcpy(yuv420_dest+linear_offset, nv12t_src+tiled_offset, 2);
1022                                 linear_offset = linear_offset+2;
1023                         }
1024                 }
1025         }
1026 }
1027
1028 #ifdef CAPTURE_OUTPUT_DUMP /* for capture output dump */
1029 static void capture_output_dump(mm_player_t* player)
1030 {
1031         unsigned char *temp = NULL;
1032         char file[100] = { 0, };
1033         FILE *fp = NULL;
1034         int ret = 0;
1035         int i = 0, j = 0;
1036
1037         LOGE("capture output dump start. size = %d", player->capture.size);
1038         sprintf(file, "/tmp/dec_output_dump_%dx%d.yuv", player->captured.width[0], player->captured.height[0]);
1039         fp = fopen(file, "ab");
1040
1041         for (i = 0; i < player->captured.plane_num; i++) {
1042                 temp = player->captured.data[i];
1043
1044                 for (j = 0; j < player->captured.height[i]; j++) {
1045                         ret = fwrite(temp, player->captured.width[i], 1, fp);
1046                         temp += player->captured.width[i];
1047                 }
1048         }
1049         LOGE("capture yuv dumped!! ret = %d", ret);
1050         fclose(fp);
1051
1052         temp = (unsigned char *)player->capture.data;
1053         sprintf(file, "/tmp/dec_output_dump_%dx%d.rgb", player->captured.width[0], player->captured.height[0]);
1054         fp = fopen(file, "ab");
1055         ret = fwrite(temp, player->capture.size, 1, fp);
1056         fclose(fp);
1057         LOGE("capture rgb dumped!! ret = %d", ret);
1058 }
1059 #endif