Store stats information into separate files
authorDaniel Wagner <daniel.wagner@bmw-carit.de>
Tue, 21 Sep 2010 11:33:42 +0000 (13:33 +0200)
committerSamuel Ortiz <sameo@linux.intel.com>
Wed, 22 Sep 2010 09:20:57 +0000 (11:20 +0200)
Currently the statistic information is stored into the profile file.
This results in rewriting the whole file if connann_stats_save is
called. This results in too many disk I/Os for low power devices.

Furthermore, only the current value is stored. There is no way to
find out how many bytes have been transfered in the last month.

First, each service statistic is stored into a separate files under
/var/lib/connman having a *.data extension. This file contains fixed
sized records of stats counters and will be mmap into memory. It is
used like a ring buffer.

If the buffer is full or after a certain period (e.g. a month), the
raw data will be summerized into the *.info file.
Currently the summary is not implented yet. This will be implemented
in the next round of patches.

Makefile.am
configure.ac
src/connman.h
src/main.c
src/service.c
src/stats.c [new file with mode: 0644]

index 36cd664..70dafd6 100644 (file)
@@ -67,7 +67,7 @@ src_connmand_SOURCES = $(gdbus_sources) $(gdhcp_sources) $(gresolv_sources) \
                        src/wifi.c src/storage.c src/dbus.c src/config.c \
                        src/technology.c src/counter.c src/location.c \
                        src/session.c src/tethering.c src/ondemand.c \
-                       src/wpad.c
+                       src/wpad.c src/stats.c
 
 if UDEV
 src_connmand_SOURCES += src/udev.c
index 24c61ff..6dc3546 100644 (file)
@@ -264,6 +264,17 @@ fi
 AM_CONDITIONAL(NTPD, test "${enable_ntpd}" != "no")
 AM_CONDITIONAL(NTPD_BUILTIN, test "${enable_ntpd}" = "builtin")
 
+AC_ARG_WITH(stats-max-file-size, AC_HELP_STRING([--with-stats-max-file-size=SIZE],
+                                [Maximal size of a statistics round robin file]),
+                                [stats_max_file_size=${withval}])
+
+if (test -z "${stats_max_file_size}"); then
+   # default size is 512 kByte
+   stats_max_file_size="512 * 8 * 128"
+fi
+
+AC_DEFINE_UNQUOTED([STATS_MAX_FILE_SIZE], (${stats_max_file_size}), [Maximal size of a statistics round robin file])
+
 PKG_CHECK_MODULES(GLIB, glib-2.0 >= 2.16, dummy=yes,
                                AC_MSG_ERROR(GLib >= 2.16 is required))
 AC_SUBST(GLIB_CFLAGS)
index 21bf9e9..15ab36d 100644 (file)
@@ -454,6 +454,7 @@ void __connman_service_create_ipconfig(struct connman_service *service,
                                                                int index);
 struct connman_ipconfig *__connman_service_get_ipconfig(
                                struct connman_service *service);
+const char *__connman_service_get_ident(struct connman_service *service);
 const char *__connman_service_get_path(struct connman_service *service);
 unsigned int __connman_service_get_order(struct connman_service *service);
 struct connman_network *__connman_service_get_network(struct connman_service *service);
@@ -561,3 +562,26 @@ void __connman_session_cleanup(void);
 
 int __connman_ondemand_init(void);
 void __connman_ondemand_cleanup(void);
+
+struct connman_stats_data {
+       unsigned int rx_packets;
+       unsigned int tx_packets;
+       unsigned int rx_bytes;
+       unsigned int tx_bytes;
+       unsigned int rx_errors;
+       unsigned int tx_errors;
+       unsigned int rx_dropped;
+       unsigned int tx_dropped;
+       unsigned int time;
+};
+
+int __connman_stats_init(void);
+void __connman_stats_cleanup(void);
+int __connman_stats_service_register(struct connman_service *service);
+void __connman_stats_service_unregister(struct connman_service *service);
+int  __connman_stats_update(struct connman_service *service,
+                               connman_bool_t roaming,
+                               struct connman_stats_data *data);
+int __connman_stats_get(struct connman_service *service,
+                               connman_bool_t roaming,
+                               struct connman_stats_data *data);
index aa63225..28e72ba 100644 (file)
@@ -225,6 +225,7 @@ int main(int argc, char *argv[])
        __connman_manager_init(option_compat);
        __connman_profile_init();
        __connman_config_init();
+       __connman_stats_init();
 
        __connman_resolver_init();
        __connman_ipconfig_init();
@@ -262,6 +263,7 @@ int main(int argc, char *argv[])
        __connman_ipconfig_cleanup();
        __connman_resolver_cleanup();
 
+       __connman_stats_cleanup();
        __connman_config_cleanup();
        __connman_profile_cleanup();
        __connman_manager_cleanup();
index 42aac9f..2db12b9 100644 (file)
@@ -37,18 +37,6 @@ static GSequence *service_list = NULL;
 static GHashTable *service_hash = NULL;
 static GSList *counter_list = NULL;
 
-struct connman_stats_data {
-       unsigned int rx_packets;
-       unsigned int tx_packets;
-       unsigned int rx_bytes;
-       unsigned int tx_bytes;
-       unsigned int rx_errors;
-       unsigned int tx_errors;
-       unsigned int rx_dropped;
-       unsigned int tx_dropped;
-       unsigned int time;
-};
-
 struct connman_stats {
        connman_bool_t valid;
        connman_bool_t enabled;
@@ -486,104 +474,6 @@ static void stats_stop(struct connman_service *service)
        stats->enabled = FALSE;
 }
 
-static int stats_load(struct connman_service *service, GKeyFile *keyfile)
-{
-       struct connman_stats_data *data;
-
-       /* home */
-       data = &service->stats.data;
-       data->rx_packets = g_key_file_get_integer(keyfile,
-                       service->identifier, "Home.rx_packets", NULL);
-       data->tx_packets = g_key_file_get_integer(keyfile,
-                       service->identifier, "Home.tx_packets", NULL);
-       data->rx_bytes = g_key_file_get_integer(keyfile,
-                       service->identifier, "Home.rx_bytes", NULL);
-       data->tx_bytes = g_key_file_get_integer(keyfile,
-                       service->identifier, "Home.tx_bytes", NULL);
-       data->rx_errors = g_key_file_get_integer(keyfile,
-                       service->identifier, "Home.rx_errors", NULL);
-       data->tx_errors = g_key_file_get_integer(keyfile,
-                       service->identifier, "Home.tx_errors", NULL);
-       data->rx_dropped = g_key_file_get_integer(keyfile,
-                       service->identifier, "Home.rx_dropped", NULL);
-       data->tx_dropped = g_key_file_get_integer(keyfile,
-                       service->identifier, "Home.tx_dropped", NULL);
-       data->time = g_key_file_get_integer(keyfile,
-                       service->identifier, "Home.time", NULL);
-
-       /* roaming */
-       data = &service->stats_roaming.data;
-       data->rx_packets = g_key_file_get_integer(keyfile,
-                       service->identifier, "Roaming.rx_packets", NULL);
-       data->tx_packets = g_key_file_get_integer(keyfile,
-                       service->identifier, "Roaming.tx_packets", NULL);
-       data->rx_bytes = g_key_file_get_integer(keyfile,
-                       service->identifier, "Roaming.rx_bytes", NULL);
-       data->tx_bytes = g_key_file_get_integer(keyfile,
-                       service->identifier, "Roaming.tx_bytes", NULL);
-       data->rx_errors = g_key_file_get_integer(keyfile,
-                       service->identifier, "Roaming.rx_errors", NULL);
-       data->tx_errors = g_key_file_get_integer(keyfile,
-                       service->identifier, "Roaming.tx_errors", NULL);
-       data->rx_dropped = g_key_file_get_integer(keyfile,
-                       service->identifier, "Roaming.rx_dropped", NULL);
-       data->tx_dropped = g_key_file_get_integer(keyfile,
-                       service->identifier, "Roaming.tx_dropped", NULL);
-       data->time = g_key_file_get_integer(keyfile,
-                       service->identifier, "Roaming.time", NULL);
-
-       return 0;
-}
-
-static int stats_save(struct connman_service *service, GKeyFile *keyfile)
-{
-       struct connman_stats_data *data;
-
-       /* home */
-       data = &service->stats.data;
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Home.rx_packets", data->rx_packets);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Home.tx_packets", data->tx_packets);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Home.rx_bytes", data->rx_bytes);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Home.tx_bytes", data->tx_bytes);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Home.rx_errors", data->rx_errors);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Home.tx_errors", data->tx_errors);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Home.rx_dropped", data->rx_dropped);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Home.tx_dropped", data->tx_dropped);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Home.time", data->time);
-
-       /* roaming */
-       data = &service->stats_roaming.data;
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Roaming.rx_packets", data->rx_packets);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Roaming.tx_packets", data->tx_packets);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Roaming.rx_bytes", data->rx_bytes);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Roaming.tx_bytes", data->tx_bytes);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Roaming.rx_errors", data->rx_errors);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Roaming.tx_errors", data->tx_errors);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Roaming.rx_dropped", data->rx_dropped);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Roaming.tx_dropped", data->tx_dropped);
-       g_key_file_set_integer(keyfile, service->identifier,
-               "Roaming.time", data->time);
-
-       return 0;
-}
-
 static void reset_stats(struct connman_service *service)
 {
        DBG("service %p", service);
@@ -1206,6 +1096,8 @@ void __connman_service_notify(struct connman_ipconfig *ipconfig,
        gpointer key, value;
        const char *counter;
        struct connman_stats_counter *counters;
+       struct connman_stats_data *data;
+       int err;
 
        service = connman_ipconfig_get_data(ipconfig);
        if (service == NULL)
@@ -1220,6 +1112,12 @@ void __connman_service_notify(struct connman_ipconfig *ipconfig,
                rx_errors, tx_errors,
                rx_dropped, tx_dropped);
 
+       data = &stats_get(service)->data;
+       err = __connman_stats_update(service, service->roaming, data);
+       if (err < 0)
+               connman_error("Failed to store statistics for %s",
+                               service->identifier);
+
        g_hash_table_iter_init(&iter, service->counter_table);
        while (g_hash_table_iter_next(&iter, &key, &value)) {
                counter = key;
@@ -2317,6 +2215,7 @@ static void service_free(gpointer user_data)
 
        stats_stop(service);
        __connman_storage_save_service(service);
+       __connman_stats_service_unregister(service);
 
        service->path = NULL;
 
@@ -3356,6 +3255,10 @@ static int service_register(struct connman_service *service)
 
        __connman_storage_load_service(service);
 
+       __connman_stats_service_register(service);
+       __connman_stats_get(service, FALSE, &service->stats.data);
+       __connman_stats_get(service, TRUE, &service->stats_roaming.data);
+
        g_dbus_register_interface(connection, service->path,
                                        CONNMAN_SERVICE_INTERFACE,
                                        service_methods, service_signals,
@@ -3529,6 +3432,11 @@ struct connman_service *__connman_service_lookup_from_index(int index)
        return NULL;
 }
 
+const char *__connman_service_get_ident(struct connman_service *service)
+{
+       return service->identifier;
+}
+
 const char *__connman_service_get_path(struct connman_service *service)
 {
        return service->path;
@@ -4090,7 +3998,6 @@ static int service_load(struct connman_service *service)
                service->domains = NULL;
        }
 
-       stats_load(service, keyfile);
 done:
        g_key_file_free(keyfile);
 
@@ -4252,8 +4159,6 @@ update:
                g_key_file_remove_key(keyfile, service->identifier,
                                                        "Domains", NULL);
 
-       stats_save(service, keyfile);
-
        data = g_key_file_to_data(keyfile, &length, NULL);
 
        if (g_file_set_contents(pathname, data, length, NULL) == FALSE)
diff --git a/src/stats.c b/src/stats.c
new file mode 100644 (file)
index 0000000..08f3dd2
--- /dev/null
@@ -0,0 +1,431 @@
+/*
+ *
+ *  Connection Manager
+ *
+ *  Copyright (C) 2010  BMW Car IT GmbH. All rights reserved.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License version 2 as
+ *  published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#define _GNU_SOURCE
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "connman.h"
+
+#define MAGIC 0xFA00B915
+
+/*
+ * Statistics counters are stored into a ring buffer which is stored
+ * into a file
+ *
+ * File properties:
+ *   The ring buffer is mmap to a file
+ *   Initialy only the smallest possible amount of disk space is allocated
+ *   The files grow to the configured maximal size
+ *   The grows by _SC_PAGESIZE step size
+ *   For each service home and service roaming counter set a file is created
+ *   Each file has a header where the indexes are stored
+ *
+ * Entries properties:
+ *   The entries are fixed sized (stats_record)
+ *   Each entry has a timestamp
+ *
+ * Ring buffer properties:
+ *   There are to indexes 'begin' and 'end'
+ *   'begin' points to the oldest entry
+ *   'end' points to the newest/current entry
+ *   If 'begin' == 'end' then the buffer is empty
+ *   If 'end' + 1 == 'begin then it's full
+ *   The ring buffer is valid in the range (begin, end]
+ *   'first' points to the first entry in the ring buffer
+ *   'last' points to the last entry in the ring buffer
+ */
+
+struct stats_file_header {
+       unsigned int magic;
+       unsigned int begin;
+       unsigned int end;
+};
+
+struct stats_record {
+       time_t ts;
+       struct connman_stats_data data;
+};
+
+struct stats_file {
+       int fd;
+       char *name;
+       char *addr;
+       size_t len;
+       size_t max_len;
+
+       /* cached values */
+       struct stats_record *first;
+       struct stats_record *last;
+};
+
+struct stats_handle {
+       struct stats_file home;
+       struct stats_file roaming;
+};
+
+GHashTable *stats_hash = NULL;
+
+static struct stats_file_header *get_hdr(struct stats_file *file)
+{
+       return (struct stats_file_header *)file->addr;
+}
+
+static struct stats_record *get_begin(struct stats_file *file)
+{
+       unsigned int off = get_hdr(file)->begin;
+
+       return (struct stats_record *)(file->addr + off);
+}
+
+static struct stats_record *get_end(struct stats_file *file)
+{
+       unsigned int off = get_hdr(file)->end;
+
+       return (struct stats_record *)(file->addr + off);
+}
+
+static void set_begin(struct stats_file *file, struct stats_record *begin)
+{
+       struct stats_file_header *hdr;
+
+       hdr = get_hdr(file);
+       hdr->begin = (char *)begin - file->addr;
+}
+
+static void set_end(struct stats_file *file, struct stats_record *end)
+{
+       struct stats_file_header *hdr;
+
+       hdr = get_hdr(file);
+       hdr->end = (char *)end - file->addr;
+}
+
+static struct stats_record *get_next(struct stats_file *file,
+                                       struct stats_record *cur)
+{
+       cur++;
+
+       if (cur > file->last)
+               cur = file->first;
+
+       return cur;
+}
+
+static struct stats_file *stats_file_get(struct stats_handle *handle,
+                                               connman_bool_t roaming)
+{
+       struct stats_file *file;
+
+       if (roaming == FALSE)
+               file = &handle->home;
+       else
+               file = &handle->roaming;
+
+       return file;
+}
+
+static void stats_close_file(struct stats_file *file)
+{
+       msync(file->addr, file->len, MS_SYNC);
+
+       munmap(file->addr, file->len);
+       file->addr = NULL;
+
+       close(file->fd);
+       file->fd = -1;
+
+       g_free(file->name);
+}
+
+static void stats_free(gpointer user_data)
+{
+       struct stats_handle *handle = user_data;
+
+       stats_close_file(&handle->home);
+       stats_close_file(&handle->roaming);
+
+       g_free(handle);
+}
+
+static int stats_create(struct connman_service *service)
+{
+       struct stats_handle *handle;
+
+       handle = g_try_new0(struct stats_handle, 1);
+       if (handle == NULL)
+               return -ENOMEM;
+
+       g_hash_table_insert(stats_hash, service, handle);
+
+       return 0;
+}
+
+static void update_first(struct stats_file *file)
+{
+       file->first = (struct stats_record *)
+                       (file->addr + sizeof(struct stats_file_header));
+}
+
+static void update_last(struct stats_file *file)
+{
+       unsigned int max_entries;
+
+       max_entries = (file->len - sizeof(struct stats_file_header)) /
+                       sizeof(struct stats_record);
+       file->last = file->first + max_entries - 1;
+}
+
+static void stats_file_update_cache(struct stats_file *file)
+{
+       update_first(file);
+       update_last(file);
+}
+
+static int stats_file_remap(struct stats_file *file, size_t size)
+{
+       size_t page_size, new_size;
+       void *addr;
+       int err;
+
+       page_size = sysconf(_SC_PAGESIZE);
+       new_size = (size + page_size - 1) & ~(page_size - 1);
+
+       err = ftruncate(file->fd, new_size);
+       if (err < 0) {
+               connman_error("ftrunctate error %s for %s",
+                               strerror(errno), file->name);
+               return -errno;
+       }
+
+       if (file->addr == NULL) {
+               addr = mmap(NULL, new_size, PROT_READ | PROT_WRITE,
+                               MAP_SHARED, file->fd, 0);
+       } else {
+               addr = mremap(file->addr, file->len, new_size, MREMAP_MAYMOVE);
+       }
+
+       if (addr == MAP_FAILED) {
+               connman_error("mmap error %s for %s",
+                               strerror(errno), file->name);
+               return -errno;
+       }
+
+       file->addr = addr;
+       file->len = new_size;
+
+       stats_file_update_cache(file);
+
+       return 0;
+}
+
+static int stats_open_file(struct connman_service *service,
+                               struct stats_file *file,
+                               connman_bool_t roaming)
+{
+       struct stat stat;
+       char *name;
+       int err;
+       size_t size;
+       struct stats_file_header *hdr;
+
+       if (roaming == FALSE) {
+               name = g_strdup_printf("%s/%s.data", STORAGEDIR,
+                                       __connman_service_get_ident(service));
+       } else {
+               name = g_strdup_printf("%s/%s-roaming.data", STORAGEDIR,
+                                       __connman_service_get_ident(service));
+       }
+
+       file->name = name;
+       file->fd = open(name, O_RDWR | O_CREAT, 0644);
+
+       if (file->fd == -1) {
+               connman_error("open error %s for %s",
+                               strerror(errno), file->name);
+               return -errno;
+       }
+
+       file->max_len = STATS_MAX_FILE_SIZE;
+
+       err = fstat(file->fd, &stat);
+       if (err < 0)
+               return -errno;
+
+       if (stat.st_size < sysconf(_SC_PAGESIZE))
+               size = sysconf(_SC_PAGESIZE);
+       else
+               size = (size_t)stat.st_size;
+
+       err = stats_file_remap(file, size);
+       if (err < 0)
+               return err;
+
+       hdr = get_hdr(file);
+
+       if (hdr->magic != MAGIC ||
+                       hdr->begin < sizeof(struct stats_file_header) ||
+                       hdr->end < sizeof(struct stats_file_header) ||
+                       hdr->begin > file->len ||
+                       hdr->end > file->len) {
+               connman_warn("invalid file header for %s", file->name);
+
+               hdr->magic = MAGIC;
+               hdr->begin = sizeof(struct stats_file_header);
+               hdr->end = sizeof(struct stats_file_header);
+       }
+
+       return 0;
+}
+
+static int stats_open(struct connman_service *service,
+                       struct stats_handle *handle)
+{
+       int err;
+
+       err = stats_open_file(service, &handle->home, FALSE);
+       if (err < 0)
+               return err;
+
+       err = stats_open_file(service, &handle->roaming, TRUE);
+       if (err < 0)
+               return err;
+
+       return 0;
+}
+
+int __connman_stats_service_register(struct connman_service *service)
+{
+       struct stats_handle *handle;
+       int err;
+
+       DBG("service %p", service);
+
+       handle = g_hash_table_lookup(stats_hash, service);
+       if (handle == NULL) {
+               err = stats_create(service);
+
+               if (err < 0)
+                       return err;
+
+               handle = g_hash_table_lookup(stats_hash, service);
+       }
+
+       err = stats_open(service, handle);
+       if (err < 0)
+               g_hash_table_remove(stats_hash, service);
+
+       return err;
+}
+
+void __connman_stats_service_unregister(struct connman_service *service)
+{
+       DBG("service %p", service);
+
+       g_hash_table_remove(stats_hash, service);
+}
+
+int  __connman_stats_update(struct connman_service *service,
+                               connman_bool_t roaming,
+                               struct connman_stats_data *data)
+{
+       struct stats_handle *handle;
+       struct stats_file *file;
+       struct stats_record *next;
+       int err;
+
+       handle = g_hash_table_lookup(stats_hash, service);
+       if (handle == NULL)
+               return -EEXIST;
+
+       file = stats_file_get(handle, roaming);
+
+       if (file->len < file->max_len &&
+                       file->last == get_end(file)) {
+               DBG("grow file %s", file->name);
+
+               err = stats_file_remap(file, file->len + sysconf(_SC_PAGESIZE));
+               if (err < 0)
+                       return err;
+       }
+
+       next = get_next(file, get_end(file));
+
+       if (next == get_begin(file))
+               set_begin(file, get_next(file, next));
+
+       next->ts = time(NULL);
+       memcpy(&next->data, data, sizeof(struct connman_stats_data));
+
+       set_end(file, next);
+
+       return 0;
+}
+
+int __connman_stats_get(struct connman_service *service,
+                               connman_bool_t roaming,
+                               struct connman_stats_data *data)
+{
+       struct stats_handle *handle;
+       struct stats_file *file;
+       struct stats_file_header *hdr;
+
+       handle = g_hash_table_lookup(stats_hash, service);
+       if (handle == NULL)
+               return -EEXIST;
+
+       file = stats_file_get(handle, roaming);
+       hdr = get_hdr(file);
+
+       if (hdr->begin != hdr->end) {
+               memcpy(data, &get_end(file)->data,
+                       sizeof(struct connman_stats_data));
+       }
+
+       return 0;
+}
+
+int __connman_stats_init(void)
+{
+       DBG("");
+
+       stats_hash = g_hash_table_new_full(g_direct_hash, g_direct_equal,
+                                                       NULL, stats_free);
+
+       return 0;
+}
+
+void __connman_stats_cleanup(void)
+{
+       DBG("");
+
+       g_hash_table_destroy(stats_hash);
+       stats_hash = NULL;
+}