Add memory usage reduction for creating image thumbnail 50/288150/10
authorjiyong.min <jiyong.min@samsung.com>
Sun, 12 Feb 2023 22:52:11 +0000 (07:52 +0900)
committerjiyong.min <jiyong.min@samsung.com>
Mon, 13 Feb 2023 06:59:56 +0000 (15:59 +0900)
 - Please use this option for low memory limit due to memory policy.
  For large-sized images, memory usage may decrease to less than 100MB,
  but thumbnail quality may be slightly lowered in some cases.
 - A-Gif has been changed from mmutil_magick to mmutil_gif.
 - Jpeg has been added to use the downscale option.

Change-Id: Id0cff4690e4ebea10649859ee2ff678d67d850db

CMakeLists.txt
packaging/libmedia-thumbnail.spec
src/media-thumbnail.c

index 4d6a30490b944ca0ec6a955fa991fa3dbaa42802..68ab48d1f8184ba3e899b77053f8d9da9ebcac9e 100644 (file)
@@ -33,8 +33,13 @@ MESSAGE("Build type: ${CMAKE_BUILD_TYPE}")
 
 INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/src/include ${CMAKE_SOURCE_DIR}/src/include/util ${CMAKE_SOURCE_DIR}/src/include/util ${CMAKE_SOURCE_DIR}/src/include/ipc ${CMAKE_SOURCE_DIR}/server/include ${CMAKE_SOURCE_DIR}/md5)
 
+IF(WITH_DA_PROFILE)
+INCLUDE(FindPkgConfig)
+       pkg_check_modules(pkgs REQUIRED glib-2.0 gthread-2.0 dlog sqlite3 mm-fileinfo aul libmedia-utils libtzplatform-config mmutil-common mmutil-gif mmutil-jpeg mmutil-magick libexif)
+ELSE(WITH_DA_PROFILE)
 INCLUDE(FindPkgConfig)
        pkg_check_modules(pkgs REQUIRED glib-2.0 gthread-2.0 dlog sqlite3 mm-fileinfo aul libmedia-utils libtzplatform-config mmutil-common mmutil-magick)
+ENDIF(WITH_DA_PROFILE)
 
 FOREACH(flag ${pkgs_CFLAGS})
        SET(EXTRA_CFLAGS "${EXTRA_CFLAGS} ${flag}")
@@ -57,7 +62,8 @@ ADD_DEFINITIONS("-DPACKAGE_NAME=\"${PKGNAME}\"")
 ADD_DEFINITIONS("-DPREFIX=\"${PREFIX}\"")
 ADD_DEFINITIONS("-D_GNU_SOURCE")
 IF(WITH_DA_PROFILE)
-       ADD_DEFINITIONS("-DWITH_DA_PROFILE")
+       # Enable reduced memory solution
+       ADD_DEFINITIONS("-DUSE_MEMORY_USAGE_REDUCTION")
 ENDIF(WITH_DA_PROFILE)
 
 SET(CMAKE_EXE_LINKER_FLAGS "-Wl,--as-needed -Wl,--hash-style=both")
index 7bc4dfed3520b092d367ee8e1e9b0ebe2f8f5687..089cdd02a83f2c858d348aa2e199859ceb78f027 100644 (file)
@@ -1,6 +1,6 @@
 Name:       libmedia-thumbnail
 Summary:    Media thumbnail service library for multimedia applications
-Version: 0.3.4
+Version: 0.3.5
 Release:    0
 Group:      Multimedia/Libraries
 License:    Apache-2.0 and PD
@@ -20,7 +20,12 @@ BuildRequires: pkgconfig(sqlite3)
 BuildRequires: pkgconfig(mmutil-common)
 BuildRequires: pkgconfig(mmutil-magick)
 %if 0%{?gtests:1}
-BuildRequires:  pkgconfig(gmock)
+BuildRequires: pkgconfig(gmock)
+%endif
+%if 0%{?_with_da_profile:1}
+BuildRequires: pkgconfig(mmutil-gif)
+BuildRequires: pkgconfig(mmutil-jpeg)
+BuildRequires: pkgconfig(libexif)
 %endif
 
 %description
index b8ecdee397edf003f29b6b85b85823fbfcc0ac62..a99482c55adb76ac9a2df062a39b262a0cafd7e2 100755 (executable)
 #include "media-thumb-util.h"
 #include "media-thumb-internal.h"
 #include "media-thumb-ipc.h"
-
-#if defined(WITH_DA_PROFILE)
-/* The maximum of resolution to be able to get thumbnail is 3000 x 2000, except for only jpeg */
-#define THUMB_MAX_ALLOWED_MEM_FOR_THUMB 6000000
+#if defined(USE_MEMORY_USAGE_REDUCTION)
+#include <mm_util_gif.h>
+#include <mm_util_jpeg.h>
+#include <libexif/exif-data.h>
 #endif
+
 #define THUMB_MAX_ALLOWED_RESOLUTION 2000
 
 int thumbnail_request_from_db_async(unsigned int request_id, const char *origin_path, ThumbFunc func, void *user_data, uid_t uid)
@@ -504,7 +505,7 @@ int create_video_thumbnail_to_buffer(const char *path,
        return err;
 }
 
-static int __get_image_info(const char *path, unsigned int *width, unsigned int *height)
+static int __get_image_info(const char *path, unsigned int *width, unsigned int *height, mm_util_img_codec_type *type)
 {
        int err = MS_MEDIA_ERR_NONE;
        mm_util_img_codec_type image_type = 0;
@@ -519,19 +520,61 @@ static int __get_image_info(const char *path, unsigned int *width, unsigned int
        thumb_retvm_if(err != MS_MEDIA_ERR_NONE, MS_MEDIA_ERR_INTERNAL, "Getting image info is failed err: %d", err);
        thumb_retvm_if(image_type == IMG_CODEC_UNKNOWN_TYPE, MS_MEDIA_ERR_UNSUPPORTED_CONTENT, "Unsupported image codec");
 
-#if defined(WITH_DA_PROFILE)
-       if (image_w * image_h >= THUMB_MAX_ALLOWED_MEM_FOR_THUMB) {
-               thumb_warn("This original image is too big. w[%d] h[%d] size limit[%d]", image_w, image_h, THUMB_MAX_ALLOWED_MEM_FOR_THUMB);
-               return MS_MEDIA_ERR_UNSUPPORTED_CONTENT;
-       }
-#endif
 
+       *type = image_type;
        *width = image_w;
        *height = image_h;
 
        return MS_MEDIA_ERR_NONE;
 }
 
+#if defined(USE_MEMORY_USAGE_REDUCTION)
+static mm_util_rotate_type_e __get_image_orientation(const char *path)
+{
+       ExifData *ed = NULL;
+       ExifEntry *entry = NULL;
+       short orientation = 0;
+       mm_util_rotate_type_e rotation = MM_UTIL_ROTATE_0;
+
+       ed = exif_data_new_from_file(path);
+       thumb_retvm_if(!ed, MM_UTIL_ROTATE_0, "no exif data");
+
+       entry = exif_data_get_entry(ed, EXIF_TAG_ORIENTATION);
+       if (!entry) {
+               thumb_info("no orientation entry");
+               exif_data_unref(ed);
+               return MM_UTIL_ROTATE_0;
+       }
+
+       orientation = exif_get_short(entry->data, exif_data_get_byte_order(ed));
+
+       /* exif orientation
+        * 0 - not available, 1- normal, 2 - hflip, 3 - rot_180,
+        * 4 - vflip, 5 - tranpose, 6 - rot_90, 7 - transverse, 8 - rot_270
+        */
+       switch (orientation) {
+       case 3:
+               rotation = MM_UTIL_ROTATE_180;
+               break;
+       case 6:
+               rotation = MM_UTIL_ROTATE_90;
+               break;
+       case 8:
+               rotation = MM_UTIL_ROTATE_270;
+               break;
+       default:
+               thumb_warn("not supported orientation[%d], rotation will be 0", orientation);
+               rotation = MM_UTIL_ROTATE_0;
+               break;
+       };
+
+       exif_data_unref(ed);
+       thumb_info("orientation: %d, rotation: %d", orientation, rotation);
+
+       return rotation;
+}
+#endif
+
 int create_image_thumbnail_to_file(const char *path, unsigned int width, unsigned int height, const char *thumb_path, bool auto_rotate)
 {
        int err = MS_MEDIA_ERR_NONE;
@@ -539,17 +582,70 @@ int create_image_thumbnail_to_file(const char *path, unsigned int width, unsigne
        unsigned int image_h = 0;
        unsigned int thumb_w = width;
        unsigned int thumb_h = height;
+       mm_util_img_codec_type image_type = 0;
 
        err = __check_parameter_validity_for_file(path, width, height, thumb_path);
        thumb_retvm_if(err != MS_MEDIA_ERR_NONE, err, "Invalid parameter");
 
        //Get image info
-       err = __get_image_info(path, &image_w, &image_h);
+       err = __get_image_info(path, &image_w, &image_h, &image_type);
        thumb_retvm_if(err != MM_UTIL_ERROR_NONE, err, "fail to __get_image_info [%d]", err);
 
        //Extract thumbnail
        __media_thumb_get_proper_thumb_size(image_w, image_h, &thumb_w, &thumb_h);
 
+#if defined(USE_MEMORY_USAGE_REDUCTION)
+       if (image_type == IMG_CODEC_GIF) {
+               mm_util_image_h decode_image = NULL;
+
+               err = mm_util_decode_from_gif_file(path, &decode_image);
+               thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_decode_from_gif_file failed : %d", err);
+
+               err = mm_util_resize_B_P(decode_image, thumb_w, thumb_h, thumb_path);
+               mm_image_destroy_image(decode_image);
+               thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_resize_B_P failed : %d", err);
+
+               return MS_MEDIA_ERR_NONE;
+       } else if (image_type == IMG_CODEC_JPEG) {
+               mm_util_image_h decode_image = NULL;
+               mm_util_image_h resize_image = NULL;
+               mm_util_jpeg_decode_downscale downscale = MM_UTIL_JPEG_DECODE_DOWNSCALE_1_1;
+               unsigned int thumb_size = thumb_w * thumb_h;
+               unsigned int image_size = image_w * image_h;
+               mm_util_rotate_type_e rotation = __get_image_orientation(path);
+
+               // The downscale divide each width & height, so we should use squares 4(2*2), 16(4*4), 64(8*8).
+               if ((image_size >= thumb_size * 4) && (image_size < thumb_size * 16)) {
+                       downscale = MM_UTIL_JPEG_DECODE_DOWNSCALE_1_2;
+               } else if ((image_size >= thumb_size * 16) && (image_size < thumb_size * 64)) {
+                       downscale = MM_UTIL_JPEG_DECODE_DOWNSCALE_1_4;
+               } else if (image_size >= thumb_size * 64) {
+                       downscale = MM_UTIL_JPEG_DECODE_DOWNSCALE_1_8;
+               }
+
+               thumb_info("downscale: %d", downscale);
+
+               err = mm_util_decode_from_jpeg_file(path, MM_UTIL_COLOR_RGB24, downscale, &decode_image);
+               thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_decode_from_jpeg_file failed : %d", err);
+
+               if (auto_rotate && (rotation != MM_UTIL_ROTATE_0)) {
+                       err = mm_util_resize_B_B(decode_image, thumb_w, thumb_h, &resize_image);
+                       mm_image_destroy_image(decode_image);
+                       thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_resize_B_B failed : %d", err);
+
+                       err = mm_util_rotate_B_P(resize_image, rotation, thumb_path);
+                       mm_image_destroy_image(resize_image);
+                       thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_rotate_B_P failed : %d", err);
+               } else {
+                       err = mm_util_resize_B_P(decode_image, thumb_w, thumb_h, thumb_path);
+                       mm_image_destroy_image(decode_image);
+                       thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_resize_B_P failed : %d", err);
+               }
+
+               return MS_MEDIA_ERR_NONE;
+       }
+#endif
+
        if (auto_rotate)
                err = mm_util_resize_and_rotate_P_P(path, thumb_w, thumb_h, thumb_path);
        else
@@ -567,19 +663,58 @@ int create_image_thumbnail_to_buffer(const char *path, unsigned int width, unsig
        unsigned int thumb_w = width;
        unsigned int thumb_h = height;
        mm_util_image_h img = NULL;
+       mm_util_img_codec_type image_type = 0;
 
        err = __check_parameter_validity_for_buffer(path, width, height, thumb_buffer, thumb_size, thumb_width, thumb_height);
        thumb_retvm_if(err != MS_MEDIA_ERR_NONE, err, "Invalid parameter");
 
        //Get image info
-       err = __get_image_info(path, &image_w, &image_h);
+       err = __get_image_info(path, &image_w, &image_h, &image_type);
        thumb_retvm_if(err != MM_UTIL_ERROR_NONE, err, "fail to __get_image_info [%d]", err);
 
        //Extract thumbnail
        __media_thumb_get_proper_thumb_size(image_w, image_h, &thumb_w, &thumb_h);
 
-       err = mm_util_resize_P_B(path, thumb_w, thumb_h, MM_UTIL_COLOR_BGRA, &img);
-       thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_resize_P_B failed : %d", err);
+#if defined(USE_MEMORY_USAGE_REDUCTION)
+       if (image_type == IMG_CODEC_GIF) {
+               mm_util_image_h decode_image = NULL;
+
+               err = mm_util_decode_from_gif_file(path, &decode_image);
+               thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_decode_from_gif_file failed : %d", err);
+
+               err = mm_util_resize_B_B(decode_image, thumb_w, thumb_h, &img);
+               mm_image_destroy_image(decode_image);
+               thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_resize_B_B failed : %d", err);
+       } else if (image_type == IMG_CODEC_JPEG) {
+               mm_util_image_h decode_image = NULL;
+               mm_util_jpeg_decode_downscale downscale = MM_UTIL_JPEG_DECODE_DOWNSCALE_1_1;
+               unsigned int thumb_size = thumb_w * thumb_h;
+               unsigned int image_size = image_w * image_h;
+
+               // The downscale divide each width & height, so we should use squares 4(2*2), 16(4*4), 64(8*8).
+               if ((image_size >= thumb_size * 4) && (image_size < thumb_size * 16)) {
+                       downscale = MM_UTIL_JPEG_DECODE_DOWNSCALE_1_2;
+               } else if ((image_size >= thumb_size * 16) && (image_size < thumb_size * 64)) {
+                       downscale = MM_UTIL_JPEG_DECODE_DOWNSCALE_1_4;
+               } else if (image_size >= thumb_size * 64) {
+                       downscale = MM_UTIL_JPEG_DECODE_DOWNSCALE_1_8;
+               }
+
+               thumb_info("downscale: %d", downscale);
+               err = mm_util_decode_from_jpeg_file(path, MM_UTIL_COLOR_RGB24, downscale, &decode_image);
+               thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_decode_from_jpeg_file failed : %d", err);
+
+               err = mm_util_resize_B_B(decode_image, thumb_w, thumb_h, &img);
+               mm_image_destroy_image(decode_image);
+               thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_resize_B_B failed : %d", err);
+
+               return MS_MEDIA_ERR_NONE;
+       } else
+#endif
+       {
+               err = mm_util_resize_P_B(path, thumb_w, thumb_h, MM_UTIL_COLOR_BGRA, &img);
+               thumb_retvm_if(err != MM_UTIL_ERROR_NONE, MS_MEDIA_ERR_INTERNAL, "mm_util_resize_P_B failed : %d", err);
+       }
 
        err = mm_image_get_image(img, thumb_width, thumb_height, NULL, thumb_buffer, thumb_size);