Make logger resize storage when system is low on memory

Logger registers for the wait on a vconf signal.

Change-Id: I38d80a33a8af0093eea3c59766c423c9148f17c3

diff --git a/Makefile.am b/Makefile.am
index 5c2499f..9d1a02b 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -13,6 +13,8 @@ AM_CFLAGS = -I$(srcdir)/include \
        -Wall \
        -Werror \
        $(CAPI_BASE_COMMON_CFLAGS) \
+       $(VCONF_CFLAGS) \
+       $(GIO_CFLAGS) \
        -D_GNU_SOURCE
 
 AM_LDFLAGS = -Wl,--as-needed,-z,noexecstack
@@ -135,6 +137,8 @@ dlog_logger_LDFLAGS = \
        $(AM_LDFLAGS) \
        -pie
 
+dlog_logger_LDADD = libdlog.la $(GIO_LIBS) $(VCONF_LIBS)
+
 dlog_logger_SOURCES = \
        external/sd-daemon/sd-daemon.c \
        external/fastlz/fastlz.c \
@@ -732,6 +736,7 @@ src_tests_qos_distributions_LDFLAGS = $(AM_LDFLAGS)
 src_tests_logger_SOURCES = src/tests/logger.c $(dlog_logger_SOURCES)
 src_tests_logger_CFLAGS = $(check_CFLAGS) -pthread
 src_tests_logger_LDFLAGS = $(AM_LDFLAGS) -Wl,--wrap=getgrnam_r,--wrap=getpwnam_r,--wrap=getegid,--wrap=geteuid,--wrap=setgid,--wrap=setuid,--wrap=socket,--wrap=unlink,--wrap=bind,--wrap=close,--wrap=chmod,--wrap=listen,--wrap=sysconf,--wrap=sd_listen_fds,--wrap=sd_is_socket_unix,--wrap=symlink,--wrap=calloc,--wrap=open,--wrap=open64,--wrap=fcntl,--wrap=fcntl64,--wrap=log_storage_reader_get_ready_bytes,--wrap=free,--wrap=log_storage_add_new_entry,--wrap=epoll_ctl
+src_tests_logger_LDADD = libdlog.la $(GIO_LIBS) $(VCONF_LIBS)
 
 src_tests_logutil_pos_SOURCES = src/tests/logutil_pos.c \
        src/libdlogutil/sort_vector.c \

diff --git a/configure.ac b/configure.ac
index 099b636..374596f 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -36,6 +36,8 @@ AC_CHECK_FUNCS([memset])
 
 PKG_PROG_PKG_CONFIG
 PKG_CHECK_MODULES([CAPI_BASE_COMMON], [capi-base-common])
+PKG_CHECK_MODULES([VCONF], [vconf])
+PKG_CHECK_MODULES([GIO], [gio-2.0])
 
 AC_ARG_ENABLE([server-user],
         AS_HELP_STRING([--enable-server-user=USER], [user dlog_server should run with]),

diff --git a/include/log_file.h b/include/log_file.h
index 7f680f2..39b98aa 100644 (file)
--- a/include/log_file.h
+++ b/include/log_file.h
@@ -28,6 +28,7 @@
 
 /* Logfile rotation */
 #define DEFAULT_ROTATE_SIZE_KB 1024
+#define DEFAULT_ROTATE_SIZE_KB_LOW_MEM DEFAULT_ROTATE_SIZE_KB
 #define DEFAULT_ROTATE_NUM_FILES 3
 
 #define BtoKiB(x)      ((x) >> 10)
@@ -39,6 +40,7 @@ struct log_file {
        int fd;
        int should_close;
        size_t rotate_size_kbytes;
+       size_t rotate_size_kbytes_low_mem;
        size_t max_rotated;
        struct log_format format;
        bool isatty;

diff --git a/include/util_parser.h b/include/util_parser.h
index d143d56..4703d74 100644 (file)
--- a/include/util_parser.h
+++ b/include/util_parser.h
@@ -36,6 +36,7 @@ struct parse_result {
                        bool colors_force; /**< Whether colors should always be enabled */
                        log_print_format format; /**< Printing format */
                        size_t rotate_size_kbytes; /**< Rotation file size */
+                       size_t rotate_size_kbytes_low_mem; /**< Rotation file size used when system is low on memory (applies only to compressed buffers) */
                        size_t max_rotated; /**< Maximum amount of rotation files */
                        const char *file_path; /**< Output file path from input arguments */
                        size_t write_buffer_size; /**< Size of the write buffer */

diff --git a/packaging/dlog.spec b/packaging/dlog.spec
index 7da51d5..07d17de 100644 (file)
--- a/packaging/dlog.spec
+++ b/packaging/dlog.spec
@@ -23,6 +23,8 @@ BuildRequires: libtool
 BuildRequires: pkgconfig(capi-base-common)
 BuildRequires: pkgconfig(libtzplatform-config)
 BuildRequires: linux-tizen-modules-headers
+BuildRequires: pkgconfig(vconf)
+BuildRequires: pkgconfig(gio-2.0)
 %if 0%{?gcov:1}
 BuildRequires: lcov
 BuildRequires: zip
@@ -67,6 +69,8 @@ Summary:    Logger service
 License:    Apache-2.0
 Group:      Development/Libraries
 Requires:   %{name} = %{version}-%{release}
+Requires:   pkgconfig(vconf)
+Requires:   pkgconf(gio-2.0)
 %description logger
 
 %package -n dlogutil

diff --git a/src/logger/dlogutil_line.c b/src/logger/dlogutil_line.c
index 13a8068..82b1122 100644 (file)
--- a/src/logger/dlogutil_line.c
+++ b/src/logger/dlogutil_line.c
@@ -83,6 +83,7 @@ static int get_dlogutil_params_from_argc_argv(int argc, char **argv, log_print_f
        params->compression = pr.compression ? strdup(pr.compression) : NULL;
        params->mem_algo = pr.mem_algo ? strdup(pr.mem_algo) : NULL;
        params->file.rotate_size_kbytes = pr.rotate_size_kbytes;
+       params->file.rotate_size_kbytes_low_mem = pr.rotate_size_kbytes_low_mem;
        params->file.max_rotated = pr.max_rotated;
        params->file.format.format = pr.format;
        params->enabled_buffers = pr.enabled_buffers;

diff --git a/src/logger/log_compressed_storage.c b/src/logger/log_compressed_storage.c
index d02f95c..1ff3312 100644 (file)
--- a/src/logger/log_compressed_storage.c
+++ b/src/logger/log_compressed_storage.c
@@ -38,7 +38,9 @@ typedef struct log_compressed_storage_entry {
 
 struct log_compressed_storage {
        char *name; // name for identification purposes
-       unsigned capacity;      // maximum size of stored entries
+       unsigned capacity;      // maximum size of stored entries (dynamic)
+       unsigned capacity_low_mem; // maximum size of stored entries when system is low on memory
+       unsigned capacity_high_mem; // maximum size of stored entries when system is at high memory
        uint64_t counter_begin; // "pointer" to the first stored entry in the entry stream, see description above
        uint64_t counter_end;   // "pointer" to the end of the stored entry stream
        log_compressed_storage_entry *entries; // the entries are stored here
@@ -95,7 +97,7 @@ static struct compression_algo *get_algo_by_name(const char *algo_name)
        return NULL;
 }
 
-log_compressed_storage *log_compressed_storage_create(unsigned capacity, const char *name, const char *algo_name)
+log_compressed_storage *log_compressed_storage_create(unsigned capacity, unsigned capacity_low_mem, const char *name, const char *algo_name)
 {
        struct compression_algo *algo = get_algo_by_name(algo_name);
        if (!algo)
@@ -121,6 +123,8 @@ log_compressed_storage *log_compressed_storage_create(unsigned capacity, const c
 
        storage->algo = algo;
        storage->capacity = capacity;
+       storage->capacity_high_mem = capacity;
+       storage->capacity_low_mem = capacity_low_mem;
        storage->counter_begin = 0;
        storage->counter_end = 0;
        storage->entries = NULL;
@@ -423,6 +427,18 @@ bool log_compressed_storage_reader_is_finished(const log_compressed_storage_read
        return reader->storage == NULL;
 }
 
+void log_compressed_storage_resize_low_mem(log_compressed_storage *storage)
+{
+       unsigned int capacity = storage->capacity_low_mem;
+       log_compressed_storage_resize(storage, capacity);
+}
+
+void log_compressed_storage_restore_high_mem(log_compressed_storage *storage)
+{
+       unsigned int capacity = storage->capacity_high_mem;
+       log_compressed_storage_resize(storage, capacity);
+}
+
 void log_compressed_storage_resize(log_compressed_storage *storage, unsigned int capacity)
 {
        assert(capacity != 0);

diff --git a/src/logger/log_compressed_storage.h b/src/logger/log_compressed_storage.h
index b8159b6..bf27df0 100644 (file)
--- a/src/logger/log_compressed_storage.h
+++ b/src/logger/log_compressed_storage.h
@@ -56,7 +56,7 @@ typedef struct log_compressed_storage log_compressed_storage;
  * @param[in] algo_name Name of the compression algo to be used.
  * @return The created storage instance or NULL in case of lack of memory.
  */
-log_compressed_storage *log_compressed_storage_create(unsigned capacity, const char *name, const char *algo_name);
+log_compressed_storage *log_compressed_storage_create(unsigned capacity, unsigned capacity_low_mem, const char *name, const char *algo_name);
 
 /**
  * @brief Frees the storage and all the resources associated with it.
@@ -191,6 +191,24 @@ const char *log_compressed_storage_get_name(const log_compressed_storage *storag
 struct compression_algo *get_algo_from_storage(const struct log_compressed_storage *storage);
 
 /**
+ * @brief This function resizes storage to its "low memory" capacity.
+ * @param[in] storage The storage that will be resized.
+ * @note This is to be called when system is low on memory.
+ *       "low memory" capacity of a storage is provided as a cmd option.
+ *       When new capacity is smaller than the current one, this function
+ *       will be removing earliest logs until they stop surpassing new max capacity.
+ */
+void log_compressed_storage_resize_low_mem(log_compressed_storage *storage);
+
+/**
+ * @brief This function resizes storage to its "high memory" capacity.
+ * @param[in] storage The storage that will be resized.
+ * @note This is to be called when system is no longer low on memory.
+ *       This restores the storage to the initial size.
+ */
+void log_compressed_storage_restore_high_mem(log_compressed_storage *storage);
+
+/**
  * @brief This function resizes storage to new, specified capacity.
  * @param[in] storage The storage that will be resized.
  * @param[in] capacity New capacity to which the storage will be resized to.

diff --git a/src/logger/logger.c b/src/logger/logger.c
index d8f844c..8f72cfd 100644 (file)
--- a/src/logger/logger.c
+++ b/src/logger/logger.c
@@ -23,6 +23,7 @@
 #include "subreader_file.h"
 #include "subreader_metrics.h"
 #include "subreader_memory.h"
+#include "log_compressed_storage.h"
 
 #include <buffer_traits.h>
 #include <metrics.h>
@@ -30,6 +31,9 @@
 #include <dynamic_config.h>
 #include <qos_defaults.h>
 
+#include <vconf.h>
+#include <glib.h>
+
 /**
  * @addtogroup DLOG_IMPLEMENTATION
  * @{
@@ -172,7 +176,12 @@ int create_fifo_fds(struct logger *server, int pipe_fd[2], int flags, bool dump)
 
 static int create_memory_subreader_for_common(struct dlogutil_line_params *params, struct reader_common *reader, struct logger *server)
 {
-       struct log_compressed_storage *storage = log_compressed_storage_create(params->file.rotate_size_kbytes * 1024, params->compression, params->mem_algo);
+       struct log_compressed_storage *storage =
+               log_compressed_storage_create(params->file.rotate_size_kbytes * 1024,
+                                      params->file.rotate_size_kbytes_low_mem * 1024,
+                                      params->compression,
+                                      params->mem_algo);
+
        if (!storage)
                return -ENOMEM;
 
@@ -556,6 +565,8 @@ int logger_create(struct logger_config_data *data, struct logger *l, struct sign
 
 static bool do_logger_one_iteration(struct logger *server, bool *use_lazy_polling)
 {
+       g_main_context_iteration(g_main_context_default(), FALSE);
+
        if (*use_lazy_polling)
                sleep_while_handling_socket(server, &server->epoll_socket, g_backend.lazy_polling_sleep);
 
@@ -954,6 +965,37 @@ static int parse_args(int argc, char **argv, struct buf_params *b)
        return 0;
 }
 
+static void resize_compressed_storage_cb(void *compressed_storage, void *_action)
+{
+       void (*action)(struct log_compressed_storage *) = _action;
+       struct log_compressed_storage *storage = (struct log_compressed_storage *)compressed_storage;
+
+       action(storage);
+}
+
+static void low_sys_mem_callback(keynode_t *key, void *cb_data)
+{
+       int status;
+       if (vconf_get_int(VCONFKEY_SYSMAN_LOW_MEMORY, &status) != 0)
+               return;
+
+       struct logger *server = (struct logger *)cb_data;
+       switch (status) {
+
+       case VCONFKEY_SYSMAN_LOW_MEMORY_HARD_WARNING:
+               list_foreach(server->compressed_memories, log_compressed_storage_resize_low_mem, resize_compressed_storage_cb);
+               break;
+
+       case VCONFKEY_SYSMAN_LOW_MEMORY_NORMAL:
+               list_foreach(server->compressed_memories, log_compressed_storage_restore_high_mem, resize_compressed_storage_cb);
+               break;
+
+       case VCONFKEY_SYSMAN_LOW_MEMORY_SOFT_WARNING: // not sent by resourced; AFAICT nothing sends it officially
+       default:
+               break;
+       }
+}
+
 /**
  * @brief Finalize initialisation
  * @details Do misc stuff needed at the end of the initialisation
@@ -967,7 +1009,10 @@ static int finalize_init(struct logger_config_data *data, struct logger *server)
        if (r < 0)
                return r;
 
-       //create files after resetting self privileges
+       // Register for vconf "low memory" signal
+       vconf_notify_key_changed(VCONFKEY_SYSMAN_LOW_MEMORY, low_sys_mem_callback, (void *)server);
+
+       // create files after resetting self privileges
        list_foreach(data->logfile_configs, &(struct parse_logfile_config_data) {
                .server = server,
                .data = data,

diff --git a/src/shared/log_file.c b/src/shared/log_file.c
index d07fa79..41f2051 100644 (file)
--- a/src/shared/log_file.c
+++ b/src/shared/log_file.c
@@ -46,6 +46,7 @@ void logfile_init(struct log_file *l_file)
 
        l_file->fd = -1;
        l_file->rotate_size_kbytes = DEFAULT_ROTATE_SIZE_KB;
+       l_file->rotate_size_kbytes_low_mem = DEFAULT_ROTATE_SIZE_KB_LOW_MEM;
        l_file->max_rotated = DEFAULT_ROTATE_NUM_FILES;
        l_file->format.format = FORMAT_BRIEF;
        l_file->isatty = false;

diff --git a/src/shared/util_parser.c b/src/shared/util_parser.c
index 1c13560..1d01862 100644 (file)
--- a/src/shared/util_parser.c
+++ b/src/shared/util_parser.c
@@ -21,6 +21,7 @@ struct parse_result parse_options(int argc, char **argv, log_print_format defaul
        bool colors_force = false;
        log_print_format format = default_format;
        size_t rotate_size_kbytes = DEFAULT_ROTATE_SIZE_KB;
+       size_t rotate_size_kbytes_low_mem = DEFAULT_ROTATE_SIZE_KB_LOW_MEM;
        size_t max_rotated = DEFAULT_ROTATE_NUM_FILES;
        const char *file_path = NULL;
        char *compression = NULL;
@@ -44,14 +45,15 @@ struct parse_result parse_options(int argc, char **argv, log_print_format defaul
 
        while (1) {
                static const struct option long_options[] = {
-                       {"tid"     , required_argument, NULL,   0},
-                       {"pid"     , required_argument, NULL,   1},
-                       {"version" ,       no_argument, NULL,   2},
-                       {"color"   , required_argument, NULL,   3},
-                       {"sort-by" , required_argument, NULL,   4},
-                       {"monitor" ,       no_argument, NULL,   5},
-                       {"mem_algo", required_argument, NULL,   6},
-                       {"help"    ,       no_argument, NULL, 'h'},
+                       {"tid"           , required_argument, NULL,   0},
+                       {"pid"           , required_argument, NULL,   1},
+                       {"version"       ,       no_argument, NULL,   2},
+                       {"color"         , required_argument, NULL,   3},
+                       {"sort-by"       , required_argument, NULL,   4},
+                       {"monitor"       ,       no_argument, NULL,   5},
+                       {"mem_algo"      , required_argument, NULL,   6},
+                       {"low-mem-resize", required_argument, NULL,   7},
+                       {"help"          ,       no_argument, NULL, 'h'},
                        {0}
                };
                int option = getopt_long(argc, argv, "cdm:t:gsf:r:n:v:b:u:e:h", long_options, NULL);
@@ -121,6 +123,10 @@ struct parse_result parse_options(int argc, char **argv, log_print_format defaul
                case 6:
                        mem_algo = optarg;
                        break;
+               case 7:
+                       if (sscanf(optarg, "%zu", &rotate_size_kbytes_low_mem) != 1)
+                               return (struct parse_result) { .status = PARSE_OPTION_NUMERICAL_ARGUMENT, .which_option = "--low-mem-resize", };
+                       break;
                case 'd':
                        mode = DLOGUTIL_MODE_DUMP;
                        break;
@@ -212,6 +218,7 @@ struct parse_result parse_options(int argc, char **argv, log_print_format defaul
                .colors_force = colors_force,
                .format = format,
                .rotate_size_kbytes = rotate_size_kbytes,
+               .rotate_size_kbytes_low_mem = rotate_size_kbytes_low_mem,
                .max_rotated = max_rotated,
                .file_path = file_path,
                .write_buffer_size = write_buffer_size,

diff --git a/src/tests/test_logger_compressed_storage.c b/src/tests/test_logger_compressed_storage.c
index b7db4c4..c04f7e1 100644 (file)
--- a/src/tests/test_logger_compressed_storage.c
+++ b/src/tests/test_logger_compressed_storage.c
@@ -24,6 +24,8 @@
 #include <assert.h>
 #include <limits.h>
 
+#define CAPACITY_LOW_MEM_DEFAULT 10 // this is not used in theese tests, but a value has to be provided
+
 static bool fail_malloc = false;
 
 void *__real_malloc(size_t size);
@@ -85,7 +87,7 @@ char output_buffer[16384];
 const char *g_algo;
 
 // we usually don't care about the name
-#define MAKE_STORAGE(x) log_compressed_storage_create(x, "foo", g_algo)
+#define MAKE_STORAGE(x) log_compressed_storage_create(x, CAPACITY_LOW_MEM_DEFAULT,"foo", g_algo)
 
 void test_tiny_storage()
 {
@@ -485,14 +487,14 @@ void test_dumper_finalize()
 
 void test_naming()
 {
-       log_compressed_storage *s1 = log_compressed_storage_create(123, "przemek", g_algo);
+       log_compressed_storage *s1 = log_compressed_storage_create(123, CAPACITY_LOW_MEM_DEFAULT, "przemek", g_algo);
        assert(s1);
        assert(!strcmp(log_compressed_storage_get_name(s1), "przemek"));
        log_compressed_storage_free(s1);
 
        // Make sure it makes a copy and not just a shallow pointer
        char emprah[7] = "sugmar";
-       s1 = log_compressed_storage_create(123, emprah, g_algo);
+       s1 = log_compressed_storage_create(123, CAPACITY_LOW_MEM_DEFAULT, emprah, g_algo);
        emprah[1] = 'i';
        assert(!strcmp(log_compressed_storage_get_name(s1), "sugmar"));
        log_compressed_storage_free(s1);
@@ -500,7 +502,7 @@ void test_naming()
 
 void test_storage_resize_extend_no_logs()
 {
-       log_compressed_storage *storage = log_compressed_storage_create(123, "storydzus", g_algo);
+       log_compressed_storage *storage = log_compressed_storage_create(123, CAPACITY_LOW_MEM_DEFAULT, "storydzus", g_algo);
        assert(log_compressed_storage_get_capacity(storage) == 123);
 
        log_compressed_storage_resize(storage, 456);
@@ -512,7 +514,7 @@ void test_storage_resize_extend_no_logs()
 void test_storage_resize_extend_then_shrink()
 {
        unsigned int storage_capacity = 1;
-       log_compressed_storage *storage = log_compressed_storage_create(storage_capacity, "storydzus", g_algo);
+       log_compressed_storage *storage = log_compressed_storage_create(storage_capacity, CAPACITY_LOW_MEM_DEFAULT, "storydzus", g_algo);
        assert(log_compressed_storage_get_capacity(storage) == storage_capacity);
 
        /* try adding a log above the storage's capacity */
@@ -551,7 +553,7 @@ void test_storage_resize_partial_shrink()
 
        size_t log_num_to_add = 5;
        unsigned int storage_capacity = log_num_to_add * log_len;
-       log_compressed_storage *storage = log_compressed_storage_create(storage_capacity, "storydzus", g_algo);
+       log_compressed_storage *storage = log_compressed_storage_create(storage_capacity, CAPACITY_LOW_MEM_DEFAULT, "storydzus", g_algo);
        assert(log_compressed_storage_get_capacity(storage) == storage_capacity);
 
        log_compressed_storage_reader *reader = log_compressed_storage_new_reader(storage, false, false, callback_dropped_log_increment, NULL);
@@ -587,7 +589,7 @@ void test_storage_resize_extend_with_logs()
 
        size_t log_num_to_add = 5;
        unsigned int storage_capacity = log_num_to_add * log_len;
-       log_compressed_storage *storage = log_compressed_storage_create(storage_capacity, "storydzus", g_algo);
+       log_compressed_storage *storage = log_compressed_storage_create(storage_capacity, CAPACITY_LOW_MEM_DEFAULT, "storydzus", g_algo);
        assert(log_compressed_storage_get_capacity(storage) == storage_capacity);
 
        log_compressed_storage_reader *reader = log_compressed_storage_new_reader(storage, false, false, callback_dropped_log_increment, NULL);
@@ -615,7 +617,7 @@ void test_storage_resize_extend_with_logs()
 void test_storage_resize_large_buffer()
 {
        unsigned int storage_capacity = 333333;
-       log_compressed_storage *storage = log_compressed_storage_create(storage_capacity, "storydzus", g_algo);
+       log_compressed_storage *storage = log_compressed_storage_create(storage_capacity, CAPACITY_LOW_MEM_DEFAULT, "storydzus", g_algo);
        assert(log_compressed_storage_get_capacity(storage) == storage_capacity);
 
        /* some extra characters to meet fastlz min buffer length limit to compress */
@@ -636,7 +638,7 @@ void test_storage_resize_large_buffer()
 
 void test_storage_resize_shrink_with_no_logs()
 {
-       log_compressed_storage *storage = log_compressed_storage_create(2, "storydzus", g_algo);
+       log_compressed_storage *storage = log_compressed_storage_create(2, CAPACITY_LOW_MEM_DEFAULT, "storydzus", g_algo);
        assert(log_compressed_storage_get_capacity(storage) == 2);
 
        log_compressed_storage_resize(storage, 1);