Check the value at idle time for debugging the cpu usage issue

Change-Id: I0f7a411585e127952eb635a5127448452a413e40

diff --git a/core/include/muse_core.h b/core/include/muse_core.h
index b4ecc24..542128b 100644 (file)
--- a/core/include/muse_core.h
+++ b/core/include/muse_core.h
@@ -116,6 +116,7 @@ void muse_core_destroy_fd_table(void);
 void muse_core_log_process_thread_info(int pid);
 void muse_core_log_process_opened_fds(int pid);
 void muse_core_log_process_cpu_memory(int pid);
+int muse_core_get_process_cpu_usage(int pid);
 void muse_core_log_file_list(const char *path);
 void muse_core_log_cmd_info(char *cmd);
 void muse_core_remove_symlink(const char *path);

diff --git a/core/src/muse_core.c b/core/src/muse_core.c
index f330c21..1b91d5a 100644 (file)
--- a/core/src/muse_core.c
+++ b/core/src/muse_core.c
@@ -793,6 +793,34 @@ void muse_core_log_process_cpu_memory(int pid)
        muse_core_log_cmd_info(cmd);
 }
 
+int muse_core_get_process_cpu_usage(int pid)
+{
+       FILE *fp = NULL;
+       int cpu = 0;
+       int idx = 0;
+       char cmd[MUSE_MSG_LEN_MAX];
+       char buf[MUSE_MSG_LEN_MAX];
+
+       snprintf(cmd, sizeof(cmd), "/bin/ps -Lo pcpu,pmem,tid,comm -p %d", pid);
+
+       fp = popen(cmd, "r");
+
+       if (fp) {
+               while (fgets(buf, MUSE_MSG_LEN_MAX, fp)) {
+                       LOGW("%s", buf);
+                       if (idx++ == 1) {
+                               cpu = atoi(g_strstrip(buf));
+                               break;
+                       }
+               }
+
+               if (pclose(fp) == -1)
+                       LOGE("Fail to pclose");
+       }
+
+       return cpu;
+}
+
 void muse_core_log_file_list(const char *path)
 {
        char cmd[MUSE_MSG_LEN_MAX];

diff --git a/packaging/mused.spec b/packaging/mused.spec
index 2bdfa4f..d3ad8aa 100644 (file)
--- a/packaging/mused.spec
+++ b/packaging/mused.spec
@@ -1,6 +1,6 @@
 Name:       mused
 Summary:    A multimedia daemon
-Version:    0.3.113
+Version:    0.3.114
 Release:    0
 Group:      System/Libraries
 License:    Apache-2.0

diff --git a/server/include/muse_server_config.h b/server/include/muse_server_config.h
index 6488cba..8ce974f 100644 (file)
--- a/server/include/muse_server_config.h
+++ b/server/include/muse_server_config.h
@@ -35,6 +35,7 @@ extern "C" {
 #define MUSE_WATCHDOG_MIN_TIMEOUT                              "muse:min_timeout"
 #define MUSE_WATCHDOG_MAX_TIMEOUT                              "muse:max_timeout"
 #define MUSE_MEMORY_THRESHOLD                                  "muse:memory_threshold"
+#define MUSE_CPU_THRESHOLD                                             "muse:cpu_threshold"
 #define MUSE_WATCH_EXTERNAL_STORAGE                            "muse:enable_watch_external_storage"
 #define MUSE_ON_DEMAND_LAUNCH                                  "muse:on_demand"
 #define MUSE_ON_DEMAND_MAX_IDLE_TIME                   "muse:on_demand_max_idle_time"
@@ -55,6 +56,7 @@ extern "C" {
 #define DEFAULT_WATCHDOG_MAX_TIMEOUT                   60
 #define DEFAULT_DISPATCH_WATCHDOG_TIMEOUT              10
 #define DEFAULT_MEMORY_LEAK_THRESHOLD                  50000 /* 50MB */
+#define DEFAULT_CPU_THRESHOLD                                  100 /* 100% */
 #define DEFAULT_WATCHDOG_DISABLED_API_INDEX            -1 /* Watchdog is applied to all dispatcher functions */
 #define DEFAULT_MAX_INSTANCE                                   -1 /* Unlimited */
 #define DEFAULT_CAUTION_INSTANCE                               -1
@@ -80,6 +82,7 @@ typedef struct ms_config {
        int min_timeout;
        int max_timeout;
        int memory_threshold;
+       int cpu_threshold;
        gboolean is_watch_external_storage_enabled;
        char *gst_param_str[MUSE_PARAM_MAX];
        int gst_param_cnt;
@@ -108,6 +111,7 @@ int ms_config_get_log_period(void);
 int ms_config_get_min_timeout(void);
 int ms_config_get_max_timeout(void);
 int ms_config_get_memory_threshold(void);
+int ms_config_get_cpu_threshold(void);
 gboolean ms_config_is_on_demand(void);
 int ms_config_get_max_idle_time(void);
 int ms_config_get_idle_state_wait_time(void);

diff --git a/server/include/muse_server_connection.h b/server/include/muse_server_connection.h
index 134d24a..e972b53 100644 (file)
--- a/server/include/muse_server_connection.h
+++ b/server/include/muse_server_connection.h
@@ -43,13 +43,14 @@ typedef struct ms_connection {
        int instance_count[MUSE_MODULE_MAX];
        GQueue *instance_queue;
        GMutex lock;
+       GCond cond;
 } ms_connection_t;
 
 void ms_connection_init(ms_connection_t *connection);
 void ms_connection_deinit(ms_connection_t *connection);
 int ms_connection_register(muse_module_h m);
 int ms_connection_unregister(muse_module_h m);
-gboolean ms_connection_get_state(ms_connection_state_e *state);
+gboolean ms_connection_get_state(ms_connection_state_e *state, int *out_fd);
 void ms_connection_lock(ms_connection_t *connection);
 void ms_connection_unlock(ms_connection_t *connection);
 

diff --git a/server/include/muse_server_private.h b/server/include/muse_server_private.h
index eed0a4b..b755c84 100644 (file)
--- a/server/include/muse_server_private.h
+++ b/server/include/muse_server_private.h
@@ -70,6 +70,7 @@ typedef struct _muse_server {
        int stop;
        int retval;
        int pid;
+       int cpu_threshold;
        gint running;
        tbm_bufmgr bufmgr;
        GMainLoop *main_loop;
@@ -105,7 +106,7 @@ muse_server_h ms_get_instance(void);
 gboolean ms_check_module_idx(int idx);
 ms_module_t *ms_get_module_instance(int idx);
 int ms_deinit(void);
-void ms_check_memory(int pid);
+void ms_check_cpu_memory(int pid);
 void ms_new(void);
 int ms_open_lockfile(void);
 void ms_run(void);

diff --git a/server/src/muse_server_config.c b/server/src/muse_server_config.c
index 666ce5b..f8def81 100644 (file)
--- a/server/src/muse_server_config.c
+++ b/server/src/muse_server_config.c
@@ -97,6 +97,8 @@ static int _ms_config_parser(ms_config_t *conf)
 
        conf->memory_threshold = _ms_config_get_int(conf->muse_dict, MUSE_MEMORY_THRESHOLD, DEFAULT_MEMORY_LEAK_THRESHOLD);
 
+       conf->cpu_threshold = _ms_config_get_int(conf->muse_dict, MUSE_CPU_THRESHOLD, DEFAULT_CPU_THRESHOLD);
+
        conf->is_watch_external_storage_enabled = (gboolean)_ms_config_get_int(conf->muse_dict, MUSE_WATCH_EXTERNAL_STORAGE, FALSE);
 
        conf->is_on_demand = (gboolean)_ms_config_get_int(conf->muse_dict, MUSE_ON_DEMAND_LAUNCH, FALSE);
@@ -359,6 +361,14 @@ int ms_config_get_memory_threshold(void)
        return conf->memory_threshold;
 }
 
+int ms_config_get_cpu_threshold(void)
+{
+       ms_config_t *conf = _ms_config_get_instance();
+       muse_return_val_if_fail(conf, MUSE_ERR);
+
+       return conf->cpu_threshold;
+}
+
 gboolean ms_config_is_on_demand(void)
 {
        ms_config_t *conf = _ms_config_get_instance();

diff --git a/server/src/muse_server_connection.c b/server/src/muse_server_connection.c
index 20dede2..8f995cf 100644 (file)
--- a/server/src/muse_server_connection.c
+++ b/server/src/muse_server_connection.c
@@ -144,6 +144,9 @@ int ms_connection_unregister(muse_module_h m)
 
        _ms_connection_module_instance_info(m, connection, queue, API_DESTROY);
 
+       LOGI("[%d] g_cond_signal", fd);
+       g_cond_signal(&connection->cond);
+
        ms_connection_unlock(connection);
 
        LOGD("Leave");
@@ -151,7 +154,7 @@ int ms_connection_unregister(muse_module_h m)
        return MM_ERROR_NONE;
 }
 
-gboolean ms_connection_get_state(ms_connection_state_e *state)
+gboolean ms_connection_get_state(ms_connection_state_e *state, int *out_fd)
 {
        int idx, fd, ep_fd, fd_count, errsv;
        struct epoll_event *p_event;
@@ -160,6 +163,7 @@ gboolean ms_connection_get_state(ms_connection_state_e *state)
        ms_connection_t *connection = NULL;
 
        muse_return_val_if_fail(ms_get_instance(), MM_ERROR_UNKNOWN);
+       muse_return_val_if_fail(out_fd, MM_ERROR_INVALID_ARGUMENT);
 
        connection = ms_get_instance()->connection;
 
@@ -211,6 +215,7 @@ gboolean ms_connection_get_state(ms_connection_state_e *state)
                        }
                        *state = MUSE_CONNECTION_STATE_DISCONNECTED;
                }
+               *out_fd = fd;
        }
 
        return TRUE;

diff --git a/server/src/muse_server_ipc.c b/server/src/muse_server_ipc.c
index 679976a..4d8ae88 100644 (file)
--- a/server/src/muse_server_ipc.c
+++ b/server/src/muse_server_ipc.c
@@ -99,10 +99,10 @@ static void _ms_ipc_module_cleanup(muse_module_h m)
 
        _ms_ipc_module_data_ch_cleanup(m);
 
-       ms_connection_unregister(m);
-
        _ms_ipc_module_msg_ch_cleanup(m);
 
+       ms_connection_unregister(m);
+
        g_mutex_clear(&m->dispatch_lock);
 
        memset(m, 0, sizeof(muse_module_t));

diff --git a/server/src/muse_server_private.c b/server/src/muse_server_private.c
index c4cf770..0c8631e 100644 (file)
--- a/server/src/muse_server_private.c
+++ b/server/src/muse_server_private.c
@@ -38,6 +38,7 @@
 #define MUSE_LWIPC_WAIT_TIME                   1000
 #endif
 
+
 static const char *channel_name[MUSE_CHANNEL_MAX] = {
        "msg",
        "data"
@@ -445,16 +446,17 @@ static void _ms_check_idle_state(void)
 static void _ms_check_connection_event(void)
 {
        ms_connection_state_e state = MUSE_CONNECTION_STATE_INVALID;
+       int fd = -1;
 
        muse_return_if_fail(muse_server);
 
-       if (ms_connection_get_state(&state)) {
+       if (ms_connection_get_state(&state, &fd)) {
                gettimeofday(&muse_server->tv_s, NULL);
                if (state == MUSE_CONNECTION_STATE_CONNECTED) {
                        /* will be updated about connection at the next patch */
                } else if (state == MUSE_CONNECTION_STATE_DISCONNECTED) {
                        LOGD("Diagnostic thread checks the memory of idle");
-                       ms_check_memory(muse_server->pid);
+                       ms_check_cpu_memory(fd);
                }
        }
 }
@@ -545,6 +547,8 @@ static void _ms_init(void)
 
        muse_core_create_fd_table();
 
+       muse_server->cpu_threshold = ms_config_get_cpu_threshold();
+
        muse_server->main_loop = g_main_loop_new(NULL, FALSE);
        muse_return_if_fail(muse_server->main_loop);
 
@@ -1053,9 +1057,9 @@ int ms_deinit(void)
        return retval;
 }
 
-void ms_check_memory(int pid)
+void ms_check_cpu_memory(int fd)
 {
-       int used_pss, memory_threshold;
+       int used_pss, memory_threshold, cpu_usage;
        char err_msg[MUSE_MSG_LEN_MAX] = {'\0',};
        ms_connection_t *connection = NULL;
 
@@ -1064,33 +1068,38 @@ void ms_check_memory(int pid)
        connection = muse_server->connection;
        muse_return_if_fail(connection);
 
-       _ms_lock_state();
-
        ms_connection_lock(connection);
 
+       while (muse_core_fd_is_valid(fd)) {
+               g_cond_wait(&connection->cond, &connection->lock);
+               LOGI("[%d] is_valid : %d", fd, muse_core_fd_is_valid(fd));
+       }
+
        if (g_queue_is_empty(connection->instance_queue) && ms_is_server_ready()) {
-               used_pss = ms_system_get_memory_usage(pid);
+               used_pss = ms_system_get_memory_usage(muse_server->pid);
+               cpu_usage = muse_core_get_process_cpu_usage(muse_server->pid);
 
-               LOGW("[%d] Proportional set size %d (KByte)", pid, used_pss);
+               LOGW("[%d] Proportional set size %d (KByte) (CPU %d)", muse_server->pid, used_pss, cpu_usage);
                memory_threshold = ms_config_get_memory_threshold();
-               if (used_pss > memory_threshold) {
+
+               if (used_pss >= memory_threshold || cpu_usage >= muse_server->cpu_threshold) {
                        ms_set_state(MUSE_SERVER_STATE_IDLE);
-                       ms_log_process_info(pid);
+                       ms_log_process_info(muse_server->pid);
 
-                       snprintf(err_msg, sizeof(err_msg), "[Memory Leak] %d > %d (KByte)", used_pss, memory_threshold);
+                       snprintf(err_msg, sizeof(err_msg), "[Memory Leak] %d >= %d (KByte) [CPU] %d >= %d (percent)",
+                               used_pss, memory_threshold, cpu_usage, muse_server->cpu_threshold);
 
                        LOGE("%s", err_msg);
                        ms_connection_unlock(connection);
-                       _ms_unlock_state();
                        ms_respawn(SIGTERM);
                }
 
                muse_core_remove_all_fd_table();
+       } else {
+               LOGI("skip cpu memory check due to instance queue length : %d", g_queue_get_length(connection->instance_queue));
        }
 
        ms_connection_unlock(connection);
-
-       _ms_unlock_state();
 }
 
 int ms_kill_thread(int signo)