From 2a42323d40bf012b1839908f62f79632486053f6 Mon Sep 17 00:00:00 2001 From: YoungHun Kim Date: Fri, 22 May 2020 17:02:52 +0900 Subject: [PATCH] Check the value at idle time for debugging the cpu usage issue Change-Id: I0f7a411585e127952eb635a5127448452a413e40 --- core/include/muse_core.h | 1 + core/src/muse_core.c | 28 +++++++++++++++++++++++++ packaging/mused.spec | 2 +- server/include/muse_server_config.h | 4 ++++ server/include/muse_server_connection.h | 3 ++- server/include/muse_server_private.h | 3 ++- server/src/muse_server_config.c | 10 +++++++++ server/src/muse_server_connection.c | 7 ++++++- server/src/muse_server_ipc.c | 4 ++-- server/src/muse_server_private.c | 37 ++++++++++++++++++++------------- 10 files changed, 79 insertions(+), 20 deletions(-) diff --git a/core/include/muse_core.h b/core/include/muse_core.h index b4ecc24..542128b 100644 --- a/core/include/muse_core.h +++ b/core/include/muse_core.h @@ -116,6 +116,7 @@ void muse_core_destroy_fd_table(void); void muse_core_log_process_thread_info(int pid); void muse_core_log_process_opened_fds(int pid); void muse_core_log_process_cpu_memory(int pid); +int muse_core_get_process_cpu_usage(int pid); void muse_core_log_file_list(const char *path); void muse_core_log_cmd_info(char *cmd); void muse_core_remove_symlink(const char *path); diff --git a/core/src/muse_core.c b/core/src/muse_core.c index f330c21..1b91d5a 100644 --- a/core/src/muse_core.c +++ b/core/src/muse_core.c @@ -793,6 +793,34 @@ void muse_core_log_process_cpu_memory(int pid) muse_core_log_cmd_info(cmd); } +int muse_core_get_process_cpu_usage(int pid) +{ + FILE *fp = NULL; + int cpu = 0; + int idx = 0; + char cmd[MUSE_MSG_LEN_MAX]; + char buf[MUSE_MSG_LEN_MAX]; + + snprintf(cmd, sizeof(cmd), "/bin/ps -Lo pcpu,pmem,tid,comm -p %d", pid); + + fp = popen(cmd, "r"); + + if (fp) { + while (fgets(buf, MUSE_MSG_LEN_MAX, fp)) { + LOGW("%s", buf); + if (idx++ == 1) { + cpu = atoi(g_strstrip(buf)); + break; + } + } + + if (pclose(fp) == -1) + LOGE("Fail to pclose"); + } + + return cpu; +} + void muse_core_log_file_list(const char *path) { char cmd[MUSE_MSG_LEN_MAX]; diff --git a/packaging/mused.spec b/packaging/mused.spec index 2bdfa4f..d3ad8aa 100644 --- a/packaging/mused.spec +++ b/packaging/mused.spec @@ -1,6 +1,6 @@ Name: mused Summary: A multimedia daemon -Version: 0.3.113 +Version: 0.3.114 Release: 0 Group: System/Libraries License: Apache-2.0 diff --git a/server/include/muse_server_config.h b/server/include/muse_server_config.h index 6488cba..8ce974f 100644 --- a/server/include/muse_server_config.h +++ b/server/include/muse_server_config.h @@ -35,6 +35,7 @@ extern "C" { #define MUSE_WATCHDOG_MIN_TIMEOUT "muse:min_timeout" #define MUSE_WATCHDOG_MAX_TIMEOUT "muse:max_timeout" #define MUSE_MEMORY_THRESHOLD "muse:memory_threshold" +#define MUSE_CPU_THRESHOLD "muse:cpu_threshold" #define MUSE_WATCH_EXTERNAL_STORAGE "muse:enable_watch_external_storage" #define MUSE_ON_DEMAND_LAUNCH "muse:on_demand" #define MUSE_ON_DEMAND_MAX_IDLE_TIME "muse:on_demand_max_idle_time" @@ -55,6 +56,7 @@ extern "C" { #define DEFAULT_WATCHDOG_MAX_TIMEOUT 60 #define DEFAULT_DISPATCH_WATCHDOG_TIMEOUT 10 #define DEFAULT_MEMORY_LEAK_THRESHOLD 50000 /* 50MB */ +#define DEFAULT_CPU_THRESHOLD 100 /* 100% */ #define DEFAULT_WATCHDOG_DISABLED_API_INDEX -1 /* Watchdog is applied to all dispatcher functions */ #define DEFAULT_MAX_INSTANCE -1 /* Unlimited */ #define DEFAULT_CAUTION_INSTANCE -1 @@ -80,6 +82,7 @@ typedef struct ms_config { int min_timeout; int max_timeout; int memory_threshold; + int cpu_threshold; gboolean is_watch_external_storage_enabled; char *gst_param_str[MUSE_PARAM_MAX]; int gst_param_cnt; @@ -108,6 +111,7 @@ int ms_config_get_log_period(void); int ms_config_get_min_timeout(void); int ms_config_get_max_timeout(void); int ms_config_get_memory_threshold(void); +int ms_config_get_cpu_threshold(void); gboolean ms_config_is_on_demand(void); int ms_config_get_max_idle_time(void); int ms_config_get_idle_state_wait_time(void); diff --git a/server/include/muse_server_connection.h b/server/include/muse_server_connection.h index 134d24a..e972b53 100644 --- a/server/include/muse_server_connection.h +++ b/server/include/muse_server_connection.h @@ -43,13 +43,14 @@ typedef struct ms_connection { int instance_count[MUSE_MODULE_MAX]; GQueue *instance_queue; GMutex lock; + GCond cond; } ms_connection_t; void ms_connection_init(ms_connection_t *connection); void ms_connection_deinit(ms_connection_t *connection); int ms_connection_register(muse_module_h m); int ms_connection_unregister(muse_module_h m); -gboolean ms_connection_get_state(ms_connection_state_e *state); +gboolean ms_connection_get_state(ms_connection_state_e *state, int *out_fd); void ms_connection_lock(ms_connection_t *connection); void ms_connection_unlock(ms_connection_t *connection); diff --git a/server/include/muse_server_private.h b/server/include/muse_server_private.h index eed0a4b..b755c84 100644 --- a/server/include/muse_server_private.h +++ b/server/include/muse_server_private.h @@ -70,6 +70,7 @@ typedef struct _muse_server { int stop; int retval; int pid; + int cpu_threshold; gint running; tbm_bufmgr bufmgr; GMainLoop *main_loop; @@ -105,7 +106,7 @@ muse_server_h ms_get_instance(void); gboolean ms_check_module_idx(int idx); ms_module_t *ms_get_module_instance(int idx); int ms_deinit(void); -void ms_check_memory(int pid); +void ms_check_cpu_memory(int pid); void ms_new(void); int ms_open_lockfile(void); void ms_run(void); diff --git a/server/src/muse_server_config.c b/server/src/muse_server_config.c index 666ce5b..f8def81 100644 --- a/server/src/muse_server_config.c +++ b/server/src/muse_server_config.c @@ -97,6 +97,8 @@ static int _ms_config_parser(ms_config_t *conf) conf->memory_threshold = _ms_config_get_int(conf->muse_dict, MUSE_MEMORY_THRESHOLD, DEFAULT_MEMORY_LEAK_THRESHOLD); + conf->cpu_threshold = _ms_config_get_int(conf->muse_dict, MUSE_CPU_THRESHOLD, DEFAULT_CPU_THRESHOLD); + conf->is_watch_external_storage_enabled = (gboolean)_ms_config_get_int(conf->muse_dict, MUSE_WATCH_EXTERNAL_STORAGE, FALSE); conf->is_on_demand = (gboolean)_ms_config_get_int(conf->muse_dict, MUSE_ON_DEMAND_LAUNCH, FALSE); @@ -359,6 +361,14 @@ int ms_config_get_memory_threshold(void) return conf->memory_threshold; } +int ms_config_get_cpu_threshold(void) +{ + ms_config_t *conf = _ms_config_get_instance(); + muse_return_val_if_fail(conf, MUSE_ERR); + + return conf->cpu_threshold; +} + gboolean ms_config_is_on_demand(void) { ms_config_t *conf = _ms_config_get_instance(); diff --git a/server/src/muse_server_connection.c b/server/src/muse_server_connection.c index 20dede2..8f995cf 100644 --- a/server/src/muse_server_connection.c +++ b/server/src/muse_server_connection.c @@ -144,6 +144,9 @@ int ms_connection_unregister(muse_module_h m) _ms_connection_module_instance_info(m, connection, queue, API_DESTROY); + LOGI("[%d] g_cond_signal", fd); + g_cond_signal(&connection->cond); + ms_connection_unlock(connection); LOGD("Leave"); @@ -151,7 +154,7 @@ int ms_connection_unregister(muse_module_h m) return MM_ERROR_NONE; } -gboolean ms_connection_get_state(ms_connection_state_e *state) +gboolean ms_connection_get_state(ms_connection_state_e *state, int *out_fd) { int idx, fd, ep_fd, fd_count, errsv; struct epoll_event *p_event; @@ -160,6 +163,7 @@ gboolean ms_connection_get_state(ms_connection_state_e *state) ms_connection_t *connection = NULL; muse_return_val_if_fail(ms_get_instance(), MM_ERROR_UNKNOWN); + muse_return_val_if_fail(out_fd, MM_ERROR_INVALID_ARGUMENT); connection = ms_get_instance()->connection; @@ -211,6 +215,7 @@ gboolean ms_connection_get_state(ms_connection_state_e *state) } *state = MUSE_CONNECTION_STATE_DISCONNECTED; } + *out_fd = fd; } return TRUE; diff --git a/server/src/muse_server_ipc.c b/server/src/muse_server_ipc.c index 679976a..4d8ae88 100644 --- a/server/src/muse_server_ipc.c +++ b/server/src/muse_server_ipc.c @@ -99,10 +99,10 @@ static void _ms_ipc_module_cleanup(muse_module_h m) _ms_ipc_module_data_ch_cleanup(m); - ms_connection_unregister(m); - _ms_ipc_module_msg_ch_cleanup(m); + ms_connection_unregister(m); + g_mutex_clear(&m->dispatch_lock); memset(m, 0, sizeof(muse_module_t)); diff --git a/server/src/muse_server_private.c b/server/src/muse_server_private.c index c4cf770..0c8631e 100644 --- a/server/src/muse_server_private.c +++ b/server/src/muse_server_private.c @@ -38,6 +38,7 @@ #define MUSE_LWIPC_WAIT_TIME 1000 #endif + static const char *channel_name[MUSE_CHANNEL_MAX] = { "msg", "data" @@ -445,16 +446,17 @@ static void _ms_check_idle_state(void) static void _ms_check_connection_event(void) { ms_connection_state_e state = MUSE_CONNECTION_STATE_INVALID; + int fd = -1; muse_return_if_fail(muse_server); - if (ms_connection_get_state(&state)) { + if (ms_connection_get_state(&state, &fd)) { gettimeofday(&muse_server->tv_s, NULL); if (state == MUSE_CONNECTION_STATE_CONNECTED) { /* will be updated about connection at the next patch */ } else if (state == MUSE_CONNECTION_STATE_DISCONNECTED) { LOGD("Diagnostic thread checks the memory of idle"); - ms_check_memory(muse_server->pid); + ms_check_cpu_memory(fd); } } } @@ -545,6 +547,8 @@ static void _ms_init(void) muse_core_create_fd_table(); + muse_server->cpu_threshold = ms_config_get_cpu_threshold(); + muse_server->main_loop = g_main_loop_new(NULL, FALSE); muse_return_if_fail(muse_server->main_loop); @@ -1053,9 +1057,9 @@ int ms_deinit(void) return retval; } -void ms_check_memory(int pid) +void ms_check_cpu_memory(int fd) { - int used_pss, memory_threshold; + int used_pss, memory_threshold, cpu_usage; char err_msg[MUSE_MSG_LEN_MAX] = {'\0',}; ms_connection_t *connection = NULL; @@ -1064,33 +1068,38 @@ void ms_check_memory(int pid) connection = muse_server->connection; muse_return_if_fail(connection); - _ms_lock_state(); - ms_connection_lock(connection); + while (muse_core_fd_is_valid(fd)) { + g_cond_wait(&connection->cond, &connection->lock); + LOGI("[%d] is_valid : %d", fd, muse_core_fd_is_valid(fd)); + } + if (g_queue_is_empty(connection->instance_queue) && ms_is_server_ready()) { - used_pss = ms_system_get_memory_usage(pid); + used_pss = ms_system_get_memory_usage(muse_server->pid); + cpu_usage = muse_core_get_process_cpu_usage(muse_server->pid); - LOGW("[%d] Proportional set size %d (KByte)", pid, used_pss); + LOGW("[%d] Proportional set size %d (KByte) (CPU %d)", muse_server->pid, used_pss, cpu_usage); memory_threshold = ms_config_get_memory_threshold(); - if (used_pss > memory_threshold) { + + if (used_pss >= memory_threshold || cpu_usage >= muse_server->cpu_threshold) { ms_set_state(MUSE_SERVER_STATE_IDLE); - ms_log_process_info(pid); + ms_log_process_info(muse_server->pid); - snprintf(err_msg, sizeof(err_msg), "[Memory Leak] %d > %d (KByte)", used_pss, memory_threshold); + snprintf(err_msg, sizeof(err_msg), "[Memory Leak] %d >= %d (KByte) [CPU] %d >= %d (percent)", + used_pss, memory_threshold, cpu_usage, muse_server->cpu_threshold); LOGE("%s", err_msg); ms_connection_unlock(connection); - _ms_unlock_state(); ms_respawn(SIGTERM); } muse_core_remove_all_fd_table(); + } else { + LOGI("skip cpu memory check due to instance queue length : %d", g_queue_get_length(connection->instance_queue)); } ms_connection_unlock(connection); - - _ms_unlock_state(); } int ms_kill_thread(int signo) -- 2.7.4