dlog_logger_SOURCES = \
external/sd-daemon/sd-daemon.c \
+ external/fastlz/fastlz.c \
src/logger/logger.c \
src/logger/logger_privileges.c \
src/logger/dlogutil_line.c \
src/logger/fd_entity.c \
src/logger/log_buffer.c \
src/logger/log_storage.c \
+ src/logger/log_compressed_storage.c \
src/logger/qos.c \
src/logger/qos_distributions.c \
src/logger/reader_common.c \
src/logger/reader_logger.c \
src/logger/reader_pipe.c \
+ src/logger/reader_memory.c \
src/logger/subreader_dlogutil.c \
src/logger/subreader_file.c \
+ src/logger/subreader_memory.c \
src/logger/subreader_metrics.c \
src/logger/socket.c \
src/logger/writer.c \
DLOGUTIL_MODE_CONTINUOUS,
DLOGUTIL_MODE_DUMP,
DLOGUTIL_MODE_MONITOR,
+ DLOGUTIL_MODE_COMPRESSED_MEMORY_DUMP,
} dlogutil_mode_e;
typedef enum {
unsigned int buffers;
unsigned int dump_size;
dlogutil_mode_e mode;
+ char *compress;
};
#ifdef __cplusplus
* @retval TIZEN_ERROR_INVALID_PARAMETER Config was NULL
* @see dlogutil_config_mode_set_monitor()
* @see dlogutil_config_mode_set_dump()
+ * @see dlogutil_config_mode_set_compressed_memory_dump()
*/
int dlogutil_config_mode_set_continuous(dlogutil_config_s *config);
* @retval TIZEN_ERROR_INVALID_PARAMETER Config was NULL
* @see dlogutil_config_mode_set_continuous()
* @see dlogutil_config_mode_set_dump()
+ * @see dlogutil_config_mode_set_compressed_memory_dump()
*/
int dlogutil_config_mode_set_monitor(dlogutil_config_s *config);
* @retval TIZEN_ERROR_INVALID_PARAMETER Config was NULL
* @see dlogutil_config_mode_set_continuous()
* @see dlogutil_config_mode_set_monitor()
+ * @see dlogutil_config_mode_set_compressed_memory_dump()
*/
int dlogutil_config_mode_set_dump(dlogutil_config_s *config, unsigned int entry_count);
+/**
+ * @brief Set log retrieval mode to dumping compressed historical logs
+ * @since_tizen 7.0
+ * @remarks This is similar to `cat /var/log/dlog/xyz`. After dumping all the logs, dlogutil_get_log() will signal this by returning TIZEN_ERROR_NO_DATA.
+ * @param[in,out] config The configuration struct
+ * @param[in] compression_buffer The name of the compression storage entry
+ * @return An error code
+ * @retval TIZEN_ERROR_NONE Success
+ * @retval TIZEN_ERROR_OUT_OF_MEMORY Not enough memory. Parameters left unchanged
+ * @retval TIZEN_ERROR_INVALID_PARAMETER Config or compression_buffer was NULL
+ * @see dlogutil_config_mode_set_continuous()
+ * @see dlogutil_config_mode_set_monitor()
+ * @see dlogutil_config_mode_set_dump()
+ */
+int dlogutil_config_mode_set_compressed_memory_dump(dlogutil_config_s *config, const char *compression_buffer);
+
/**
* @brief Finalizes the config into a state struct by connecting to buffers
DLOG_REQ_STDOUT = 6 << 1,
DLOG_REQ_GLOBAL_ENABLE_DISABLE_STDOUT = 7 << 1,
DLOG_REQ_GET_STDOUT = 8 << 1,
+ DLOG_REQ_HANDLE_COMPRESSED_LOGUTIL = 9 << 1,
};
enum {
dlogutil_mode_e mode; /**< Log printing mode */
unsigned dump_size; /**< If dump mode is enabled, number of logs to be dumped, or #DLOGUTIL_MAX_DUMP_SIZE if infinite */
dlogutil_sorting_order_e sort_by; /**< The timestamp type to be sorted on */
+ char *compression; /**< The compression storage name */
};
const char *which_option; /**< An statically allocated string meaning an option, including the initial dash(es) */
const char *bad_contents; /**< An string from input argument which has been not a valid argument */
/// Transition into a state that allows reading and printing out logs
/// Return values > 0 mean everything went fine but we don't want to print
- int (*prepare_print)(struct fd_info *fdi, int dump, bool monitor, struct log_filter *filter_object);
+ int (*prepare_print)(struct fd_info *fdi, int dump, bool monitor, struct log_filter *filter_object, char *compress);
/// Clear the buffer
int (*clear)(struct fd_info *fdi);
fdi->priv_data = NULL;
}
-static int logger_prepare_print(struct fd_info *fdi, int dump, bool monitor, struct log_filter *filter_object)
+static int logger_prepare_print(struct fd_info *fdi, int dump, bool monitor, struct log_filter *filter_object, char *compress)
{
assert(fdi);
struct logger_priv_data *const lpd = (struct logger_priv_data *)fdi->priv_data;
assert(lpd);
assert(filter_object);
+ assert(!compress);
lpd->monitor = monitor;
}
/**
- * @brief Send a logger request
- * @details Send a logging request to the daemon
+ * @brief Send a standard logger request
+ * @details Send a standard (not compressed memory) logging request to the daemon
* @param[in] filters Log filters to use
* @param[in] dump Whether dumping or not
* @param[in] sock_fd Socket file descriptor
* @return 0 on success, -errno on failure
*/
-static int send_logger_request(struct log_filter *filters, int dump, bool monitor, int sock_fd)
+static int send_logger_request(struct log_filter *filters, int dump, bool monitor, int sock_fd, char *compress)
{
assert(filters);
assert(sock_fd >= 0);
assert(!dump || !monitor);
+ static const char compress_opt_str[] = " --compress ";
size_t request_string_len = get_filter_size(filters) +
+ (compress ? (strlen(compress) + sizeof compress_opt_str - 1) : 0) +
(dump ? 3 : 0) + // " -d"
(monitor ? 3 : 0); // " -m"
len += sizeof monitor_str - 1;
}
+ if (compress) {
+ strncat(request_string, compress_opt_str, request_string_len - len);
+ len += sizeof compress_opt_str - 1;
+
+ strncat(request_string, compress, request_string_len - len);
+ len += strlen(compress);
+ }
+
int needed = 0;
for (list_head iter = log_filter_get_list(filters); iter; list_next(&iter)) {
len += needed;
request_string[len - 1] = '\0';
- return send_dlog_request(sock_fd, DLOG_REQ_HANDLE_LOGUTIL, request_string, len);
+ return send_dlog_request(sock_fd, compress ? DLOG_REQ_HANDLE_COMPRESSED_LOGUTIL : DLOG_REQ_HANDLE_LOGUTIL, request_string, len);
}
/**
fdi->priv_data = NULL;
}
-static int pipe_prepare_print(struct fd_info *fdi, int dump, bool monitor, struct log_filter *filter_object)
+/* NB: compressed logs are handled almost exactly the same as regular logs,
+ * with only the setup being slightly different (see below). This is because
+ * the decompression is done entirely in the daemon (so the logs we receive
+ * are already decompressed and look the same as usual).
+ *
+ * The potential downside of this design would usually be that the daemon
+ * is critical so it would be better to relegate the load (so as not to
+ * delay incoming logs) but the memory logs are only supposed to be dumped
+ * manually and quite rarely so performance isn't a worry, while this
+ * lets us keep compression and decompression in the same executable. */
+static int pipe_prepare_print(struct fd_info *fdi, int dump, bool monitor, struct log_filter *filter_object, char *compress)
{
assert(filter_object);
assert(fdi);
assert(ppd);
assert(ppd->sock_fd >= 0);
- int r = send_logger_request(filter_object, dump, monitor, ppd->sock_fd);
+ int r = send_logger_request(filter_object, dump, monitor, ppd->sock_fd, compress);
if (r < 0)
return r;
log_id_t aliased[LOG_ID_MAX];
for (int i = 0; i < LOG_ID_MAX; ++i)
aliased[i] = LOG_ID_INVALID;
- int fdi_cnt = create_initial_fdis(&fdi_ptrs, config->buffers, is_pipe, &conf, aliased);
+ int fdi_cnt = create_initial_fdis(&fdi_ptrs, config->buffers, is_pipe, config->mode == DLOGUTIL_MODE_COMPRESSED_MEMORY_DUMP, &conf, aliased);
if (fdi_cnt < 0)
return TIZEN_ERROR_IO_ERROR;
if (fdi_cnt == 0)
return TIZEN_ERROR_NONE;
}
+EXPORT_API int dlogutil_config_mode_set_compressed_memory_dump(dlogutil_config_s *config, const char *compress_buffer)
+{
+ CHECK_PARAM(config);
+ CHECK_PARAM(compress_buffer);
+
+ char *copy = strdup(compress_buffer);
+ if (!copy)
+ return TIZEN_ERROR_OUT_OF_MEMORY;
+
+ free(config->compress);
+ config->compress = copy;
+ config->mode = DLOGUTIL_MODE_COMPRESSED_MEMORY_DUMP;
+
+ return TIZEN_ERROR_NONE;
+}
+
EXPORT_API int dlogutil_get_log(dlogutil_state_s *state, int timeout, dlogutil_entry_s **entry_out)
{
CHECK_PARAM(state);
&& !log_config_get_boolean(conf, "handle_kmsg", true);
}
-int create_initial_fdis(struct fd_info ***fdis, int enabled_buffers, bool is_pipe, const struct log_config *conf, log_id_t aliased[LOG_ID_MAX])
+int create_initial_fdis(struct fd_info ***fdis, int enabled_buffers, bool is_pipe, bool is_compressed_memory, const struct log_config *conf, log_id_t aliased[LOG_ID_MAX])
{
assert(fdis);
assert(conf);
continue;
struct fd_info *fdi;
- switch (i) {
- case LOG_ID_KMSG:
- case LOG_ID_SYSLOG:
+ if (is_compressed_memory) {
+ /* Ignore `is_pipe`. This feature is done
+ * via pipes always, even on the Android
+ * Logger backend. */
fdi = fdi_create(&ops_pipe, i);
- break;
- default:
- fdi = fdi_create(is_pipe ? &ops_pipe : &ops_logger, i);
- break;
+ } else {
+ switch (i) {
+ case LOG_ID_KMSG:
+ case LOG_ID_SYSLOG:
+ fdi = fdi_create(&ops_pipe, i);
+ break;
+ default:
+ fdi = fdi_create(is_pipe ? &ops_pipe : &ops_logger, i);
+ break;
+ }
}
if (!fdi)
return -ENOMEM;
return TIZEN_ERROR_NONE;
}
-static int state_prepare_single_print(dlogutil_state_s *state, int nfds)
+static int state_prepare_single_print(dlogutil_state_s *state, int nfds, char *compress)
{
assert(state);
assert(state->mode != DLOGUTIL_MODE_NONPRINTING);
- int r = state->data_fds[nfds]->ops->prepare_print(state->data_fds[nfds], state->dump_size, state->mode == DLOGUTIL_MODE_MONITOR, state->filter_object);
+ int r = state->data_fds[nfds]->ops->prepare_print(state->data_fds[nfds], state->dump_size, state->mode == DLOGUTIL_MODE_MONITOR, state->filter_object, compress);
if (r < 0)
return TIZEN_ERROR_IO_ERROR;
if (r > 0) {
return TIZEN_ERROR_NONE;
}
-static int state_prepare_prints(dlogutil_state_s *state)
+static int state_prepare_prints(dlogutil_state_s *state, char *compress)
{
assert(state);
assert(state->mode != DLOGUTIL_MODE_NONPRINTING);
return TIZEN_ERROR_OUT_OF_MEMORY;
for (int nfds = 0; nfds < state->fd_count; ++nfds) {
- int r = state_prepare_single_print(state, nfds);
+ int r = state_prepare_single_print(state, nfds, compress);
if (r != TIZEN_ERROR_NONE)
return r;
}
return TIZEN_ERROR_OUT_OF_MEMORY;
if (state->mode != DLOGUTIL_MODE_NONPRINTING) {
- int r = state_prepare_prints(state);
+ int r = state_prepare_prints(state, config->compress);
if (r != TIZEN_ERROR_NONE)
return r;
}
log_id_t aliased[LOG_ID_MAX];
} dlogutil_state_s;
-int create_initial_fdis(struct fd_info ***fdis, int enabled_buffers, bool is_pipe, const struct log_config *conf, log_id_t aliased[LOG_ID_MAX]);
+int create_initial_fdis(struct fd_info ***fdis, int enabled_buffers, bool is_pipe, bool is_compressed_memory, const struct log_config *conf, log_id_t aliased[LOG_ID_MAX]);
int dlogutil_state_init(dlogutil_state_s *state, struct fd_info ***data_fds_ptr, int fd_count, dlogutil_config_s *config, bool sorting_needed, dlogutil_sorting_order_e real_sort_by, const struct log_config *conf, log_id_t aliased[LOG_ID_MAX]);
int do_print_once(dlogutil_state_s *state, int timeout, dlogutil_entry_s **out);
params->is_dumping = false;
params->buf_id = LOG_ID_INVALID;
params->file_path = NULL;
+ params->compression = NULL;
params->filter = log_filter_new();
if (!params->filter)
{
logfile_free(¶ms->file);
log_filter_free(params->filter);
+ free(params->compression);
free(params->file_path);
}
return -ENOMEM;
}
+ params->compression = pr.compression ? strdup(pr.compression) : NULL;
params->file.rotate_size_kbytes = pr.rotate_size_kbytes;
params->file.max_rotated = pr.max_rotated;
params->file.format.format = pr.format;
#pragma once
#include <log_file.h>
+#include <util_parser.h>
/**
* @brief Settings for output buffering functionality
struct dlogutil_line_params {
bool monitor; /**< Whether the monitor mode is enabled */
bool is_dumping; /**< Whether the dump mode is enabled */
+ char *compression; /**< Compressed memory backup parameter */
struct log_file file; /**< The file information structure */
log_id_t buf_id; /**< The buffer being dumped, or 0 if none selected */
char *file_path; /**< The file to be written to, or NULL if none selected */
#include "log_buffer.h"
#include "logger_internal.h"
#include "subreader_dlogutil.h"
+#include "reader_memory.h"
#include <ptrs_list.h>
/**
/**
* @brief Service util request
- * @details Handle a request from util
+ * @details Handle a standard (not compressed memory) request from util
* @param[in] server The logger server
* @param[in] wr The writer who sent the request
* @param[in] msg The message containing the request
goto cleanup;
}
+ if (params.compression) {
+ /* Memory compression is only available from the text
+ * config. libdlogutil clients have a separate request
+ * type to obtain data. Eventually it would be good to
+ * merge them together, but that requires some thought
+ * put into it. */
+ goto cleanup;
+ }
+
if (params.file_path) {
/* Do not trust writer-based readers (only config-based).
* The control socket's privilege checks are fairly lenient
return retval;
}
+static int service_writer_handle_req_compressed_memory_util(struct logger *server, struct writer *wr, struct dlog_control_msg *msg)
+{
+ assert(server);
+ assert(wr);
+ assert(msg);
+
+ assert(msg->request == DLOG_REQ_HANDLE_COMPRESSED_LOGUTIL);
+
+ if (msg->length <= sizeof(struct dlog_control_msg) ||
+ msg->length > sizeof(struct dlog_control_msg) + MAX_LOGGER_REQUEST_LEN)
+ return -EINVAL;
+
+ if (msg->data[msg->length - sizeof(struct dlog_control_msg)] != 0)
+ return -EINVAL;
+
+ __attribute__((cleanup(reader_free_ptr))) struct reader_memory *reader = NULL;
+
+ int retval;
+ __attribute__((cleanup(free_dlogutil_line_params))) struct dlogutil_line_params params;
+ if (!initialize_dlogutil_line_params(¶ms, (struct buf_params) { })) {
+ /* TODO: cleanup discards this value, so there isn't much
+ * point setting it. Ideally it would be attached to the
+ * reply but that's a protocol change so not worth it atm */
+ // retval = -ENOMEM;
+ goto cleanup;
+ }
+
+ retval = get_dlogutil_line_params(msg->data, ¶ms);
+ if (retval < 0) {
+ // retval = -ENOMEM; // see above
+ goto cleanup;
+ }
+
+ if (params.file_path) {
+ /* Do not trust writer-based readers (only config-based).
+ * The control socket's privilege checks are fairly lenient
+ * so this prevents people from asking us to overwrite
+ * some potentially important files at logger privilege.
+ *
+ * At some point it would be good to be able to skip the
+ * middleman and become able to write to a file directly
+ * though. The daemon should become able to receive an
+ * opened file descriptor from a writer. */
+ // retval = -EPERM; // see above
+ goto cleanup;
+ }
+
+ if (!params.compression)
+ goto cleanup;
+
+ retval = reader_memory_init_with_writer(&reader, wr, server, params.compression, params.monitor, params.is_dumping);
+ if (retval != 0)
+ goto cleanup;
+
+ int pipe_fd[2] = { -1, -1 };
+ retval = create_fifo_fds(server, pipe_fd, 0, reader->is_dumping);
+ if (retval < 0)
+ goto cleanup;
+
+ retval = reader_add_subreader_dlogutil(&reader->common, params.filter, pipe_fd[1]);
+ if (retval != 0)
+ goto cleanup;
+
+ set_write_fd_entity(&reader->common.fd_entity_sink, pipe_fd[1]);
+ retval = send_pipe(wr->fd_entity.fd, pipe_fd[0]);
+ if (pipe_fd[0] > 0)
+ close(pipe_fd[0]);
+ if (retval)
+ goto cleanup;
+
+ retval = add_reader_memory(server, reader);
+ if (retval < 0)
+ goto cleanup;
+
+ reader = NULL;
+ return 0;
+
+cleanup:
+ /* NB: reply success means that stuff is still stable enough that the client
+ * can probably get a second chance, so return a success from the whole func
+ * so as not to drop the client */
+ retval = send_dlog_reply(wr->fd_entity.fd, DLOG_REQ_HANDLE_COMPRESSED_LOGUTIL, DLOG_REQ_RESULT_ERR, NULL, 0);
+ if (retval < 0)
+ printf("ERROR: both create_reader_from_dlogutil_line() and send_dlog_reply() failed\n");
+
+ return retval;
+}
+
static int service_writer_handle_req_get_usage(struct logger *server, struct writer *wr, struct dlog_control_msg *msg)
{
assert(server);
switch (msg->request) {
case DLOG_REQ_CLEAR: ret = service_writer_handle_req_clear (server, wr, msg); break;
case DLOG_REQ_HANDLE_LOGUTIL: ret = service_writer_handle_req_util (server, wr, msg); break;
+ case DLOG_REQ_HANDLE_COMPRESSED_LOGUTIL: ret = service_writer_handle_req_compressed_memory_util (server, wr, msg); break;
case DLOG_REQ_GET_CAPACITY: ret = service_writer_handle_req_get_capacity (server, wr, msg); break;
case DLOG_REQ_GET_USAGE: ret = service_writer_handle_req_get_usage (server, wr, msg); break;
case DLOG_REQ_GLOBAL_ENABLE_DISABLE_STDOUT: ret = service_writer_handle_req_global_enable_disable_stdout(server, wr, msg); break;
#include "logger_privileges.h"
#include "subreader_file.h"
#include "subreader_metrics.h"
+#include "subreader_memory.h"
#include <metrics.h>
#include <getopt.h>
return ret;
}
+int add_reader_memory(struct logger *server, struct reader_memory *reader)
+{
+ assert(reader);
+ assert(server);
+
+ int ret = add_reader_common(server, &reader->common);
+ if (ret < 0)
+ return ret;
+
+ // `readers_logger` actually accepts any readers
+ if (!list_add(&server->readers_logger, reader)) {
+ if (reader->common.fd_entity_sink.fd >= 0)
+ remove_fd_entity(&server->epoll_common, &reader->common.fd_entity_sink);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
static int add_reader_logger(struct logger *server, struct reader_logger *reader)
{
assert(reader);
return ret;
}
+static int create_memory_subreader_for_common(struct dlogutil_line_params *params, struct reader_common *reader, struct logger *server)
+{
+ struct log_compressed_storage *storage = log_compressed_storage_create(params->file.rotate_size_kbytes, params->compression);
+ if (!storage)
+ return -ENOMEM;
+
+ list_add(&server->compressed_memories, storage);
+
+ return reader_add_subreader_memory(reader, params->filter, storage);
+}
+
+static int create_memory_subreader_from_dlogutil_line(struct dlogutil_line_params *params, struct logger *server)
+{
+ assert(params);
+ assert(params->compression);
+
+ if (params->file_path) // We're writing to memory, not to file
+ return -EINVAL;
+
+ if (params->buf_id == LOG_ID_INVALID)
+ return -EINVAL;
+
+ struct reader_logger *const reader = g_backend.logger_readers[params->buf_id];
+ if (!reader)
+ return -ENOENT;
+
+ return create_memory_subreader_for_common(params, &reader->common, server);
+}
+
static int create_logger_subreader_from_dlogutil_line(struct dlogutil_line_params *params)
{
+ assert(params);
+ assert(!params->compression);
+
if (params->file_path) {
int retval = logfile_set_path(¶ms->file, params->file_path);
if (retval < 0)
if (params->buf_id == LOG_ID_INVALID)
return -EINVAL;
- if (params->file.path == NULL)
+ if ((params->file.path == NULL && !params->compression)
+ || (params->file.path != NULL && params->compression))
return -EINVAL;
retval = reader_pipe_init(&reader, params->buf_id, server, params->monitor, params->is_dumping);
if (retval != 0)
return retval;
- retval = reader_add_subreader_file(&reader->common, params->filter, ¶ms->file, server->buffers[params->buf_id]->sort_by);
- if (retval != 0)
- return retval;
+ if (params->file.path == NULL) {
+ assert(params->compression);
+ retval = create_memory_subreader_for_common(params, &reader->common, server);
+ if (retval != 0)
+ return retval;
+ } else {
+ /* FIXME: in theory these could be added independently (1 reader 2 subs),
+ * but so far that has not been needed and it sounds too fragile to do in haste. */
+ assert(!params->compression);
+ retval = reader_add_subreader_file(&reader->common, params->filter, ¶ms->file, server->buffers[params->buf_id]->sort_by);
+ if (retval != 0)
+ return retval;
+ }
*rd = reader;
reader = NULL;
int r;
if (g_backend.use_logger_by_default && is_core_buffer(params.buf_id)) {
- r = create_logger_subreader_from_dlogutil_line(¶ms);
+ r = params.compression
+ ? create_memory_subreader_from_dlogutil_line(¶ms, server)
+ : create_logger_subreader_from_dlogutil_line(¶ms)
+ ;
} else {
struct reader_pipe *reader = NULL;
r = create_reader_pipe_from_dlogutil_line(¶ms, server, &reader);
#include "fd_entity.h"
#include "reader_common.h"
#include "reader_logger.h"
+#include "reader_memory.h"
#include "reader_pipe.h"
#include "socket.h"
#include "dlogutil_line.h"
struct buf_params buf_params;
int exiting;
struct qos_module qos;
+ list_head compressed_memories;
};
struct logger_config_data {
void logger_add_writer(struct logger *l, struct writer *wr);
int create_fifo_fds(struct logger *server, int pipe_fd[2], int flags, bool dump);
int add_reader_pipe(struct logger *server, struct reader_pipe *reader);
+int add_reader_memory(struct logger *server, struct reader_memory *reader);
void flush_logfile_timely(struct log_file *file, struct timespec ts, int flush_time);
int get_now(struct now_t *now);
--- /dev/null
+#include "reader_memory.h"
+#include "logger_internal.h"
+#include "log_compressed_storage.h"
+#include "compression_common.h"
+#include "fastlz.h"
+
+static void reader_memory_free(struct reader_common *_reader)
+{
+ struct reader_memory *reader = (struct reader_memory *) _reader;
+
+ if (reader->log_compressed_storage_reader_ptr)
+ log_compressed_storage_release_reader(reader->log_compressed_storage_reader_ptr);
+ free(reader->uncompressed_chunk);
+}
+
+static int print_out_logs(struct reader_common *_reader, struct now_t _time);
+
+static void unpin_and_free(struct reader_memory *reader, struct logger *server)
+{
+ list_remove(&server->readers_logger, reader);
+ remove_reader_fd_entities(server, &reader->common);
+ reader_free(&reader->common);
+}
+
+static void dispatch_event_reader_memory(struct logger *server, struct epoll_event *event, void *userdata)
+{
+ struct reader_memory *const rm = (struct reader_memory *) userdata;
+ assert(rm);
+
+ if (event->events & (EPOLLHUP | EPOLLERR)) {
+ unpin_and_free(rm, server);
+ return;
+ }
+
+ struct now_t now;
+ int r = get_now(&now);
+ if (r < 0) {
+ unpin_and_free(rm, server);
+ return ;
+ }
+
+ r = rm->common.service_reader(&rm->common, now);
+ if (r != 0) {
+ unpin_and_free(rm, server);
+ return;
+ }
+}
+
+static struct reader_memory *reader_memory_alloc(bool monitor, bool is_dumping)
+{
+ struct reader_memory *ret = calloc(1, sizeof(*ret));
+ if (!ret)
+ return NULL;
+
+ init_fd_entity(&ret->common.fd_entity_sink , NULL, NULL);
+ init_fd_entity(&ret->common.fd_entity_source, NULL, NULL);
+ ret->common.free_reader = reader_memory_free;
+
+ ret->uncompressed_chunk = malloc(COMPRESSION_CHUNK_SIZE);
+ if (!ret->uncompressed_chunk) {
+ free(ret);
+ return NULL;
+ }
+ ret->uncompressed_chunk_size = 0;
+ ret->uncompressed_chunk_index = 0;
+
+ ret->common.service_reader = print_out_logs;
+ ret->monitor = monitor;
+ ret->is_dumping = is_dumping;
+ ret->log_compressed_storage_reader_ptr = NULL;
+
+ return ret;
+}
+
+bool find_str(void *element, void *userdata)
+{
+ return !strcmp(log_compressed_storage_get_name((log_compressed_storage *) element), (char *) userdata);
+}
+
+void ignore_dropped_log(const struct compression_entry *ce, void *user_data)
+{
+ /* A compressed chunk is typically too large to try and salvage it.
+ * Drop it, it's quite unlikely that this actually happens given
+ * that a client should process logs quite fast compared to how
+ * slow they are produced. */
+
+ fprintf(stderr, "reader_memory: dropped a compressed chunk of length %zu / %zu\n"
+ , ce->size_out
+ , ce->size_in
+ );
+}
+
+int reader_memory_init_with_writer(struct reader_memory **reader, struct writer *writer, struct logger *server,
+ char *compress, bool monitor, bool is_dumping)
+{
+ assert(reader);
+ assert(writer);
+ assert(writer->buf_ptr);
+ assert(server);
+ assert(compress);
+
+ log_compressed_storage *storage = list_find_if(server->compressed_memories, compress, find_str);
+ if (!storage)
+ return -EINVAL;
+
+ __attribute__((cleanup(reader_free_ptr))) struct reader_memory *ret = reader_memory_alloc(monitor, is_dumping);
+ if (!ret)
+ return -ENOMEM;
+
+ ret->log_compressed_storage_reader_ptr = log_compressed_storage_new_reader(storage,
+ is_dumping, monitor, ignore_dropped_log, ret);
+ if (!ret->log_compressed_storage_reader_ptr)
+ return -ENOMEM;
+
+ init_fd_entity(&ret->common.fd_entity_sink, dispatch_event_reader_memory, ret);
+ init_fd_entity(&ret->common.fd_entity_source, dispatch_event_reader_memory, ret);
+
+ *reader = ret;
+ ret = NULL;
+ return 0;
+}
+
+int spend_queued_chunk(struct reader_memory *reader)
+{
+ while (reader->uncompressed_chunk_index < reader->uncompressed_chunk_size) {
+ dlogutil_entry_s *const due = (dlogutil_entry_s *)(reader->uncompressed_chunk + reader->uncompressed_chunk_index);
+ if (!due->len)
+ return EINVAL;
+
+ reader->uncompressed_chunk_index += due->len;
+
+ int r = list_foreach_ret(reader->common.subs, due, subreader_apply_log);
+ if (r != 0)
+ return r;
+ }
+
+ return 0;
+}
+
+void reader_memory_decompress_chunk(struct reader_memory *reader, const struct compression_entry *ce)
+{
+ if (reader->uncompressed_chunk_index < reader->uncompressed_chunk_size)
+ return;
+
+ /* Could also allocate the chunk here instead of a "static" large one,
+ * which saves some memory overhead in the optimistic case, but
+ * adds a lot of juggling around instead. */
+
+ reader->uncompressed_chunk_index = 0;
+ reader->uncompressed_chunk_size = ce->size_in;
+
+ if (!ce->size_out){
+ // Data wasn't compressed
+ memcpy(reader->uncompressed_chunk, ce->compressed_data, ce->size_in);
+ } else {
+ int decompressed_size = fastlz_decompress(ce->compressed_data, ce->size_out, reader->uncompressed_chunk, ce->size_in);
+ assert((size_t) decompressed_size == ce->size_in);
+ }
+}
+
+/**
+ * @brief Print out logs
+ * @details Make sure the reader is up to date on printed logs
+ * @param[in] reader The reader to read the data
+ * @param[in] _time Unused timestamps
+ * @return 0 if data remains for the next iteration, 1 if the buffer is to be removed, else -1
+ */
+static int print_out_logs(struct reader_common *_reader, struct now_t _time)
+{
+ struct reader_memory *const reader = (struct reader_memory *) _reader;
+ assert(reader);
+
+ int r = reader_flush(_reader, (struct timespec){0, 0}, 0);
+ if (r != 0)
+ return r > 0;
+
+ r = spend_queued_chunk(reader);
+ if (r != 0)
+ return r > 0;
+
+ while (log_compressed_storage_reader_is_new_entry_available(reader->log_compressed_storage_reader_ptr)) {
+ const struct compression_entry *ce = log_compressed_storage_reader_get_new_entry(reader->log_compressed_storage_reader_ptr);
+ assert(ce); // `is_new_entry_available` guarantees this
+
+ reader_memory_decompress_chunk(reader, ce);
+
+ r = spend_queued_chunk(reader);
+ if (r != 0)
+ return r > 0;
+ }
+
+ /* Treat all readers as dumping. This is because monitoring makes
+ * little sense here (monitor the source instead!) and because
+ * these readers have quite a lot of overhead (so it's best if
+ * they don't linger around too long, even if by accident). */
+ return 1;
+}
+
--- /dev/null
+#pragma once
+
+#include "reader_common.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+
+struct log_compressed_storage_reader;
+struct writer;
+struct logger;
+
+struct reader_memory {
+ struct reader_common common;
+ struct log_compressed_storage_reader *log_compressed_storage_reader_ptr;
+ bool is_dumping;
+ bool monitor;
+ char *uncompressed_chunk;
+ size_t uncompressed_chunk_size;
+ size_t uncompressed_chunk_index;
+};
+
+int reader_memory_init_with_writer(struct reader_memory **reader, struct writer *writer, struct logger *server,
+ char *compress, bool monitor, bool is_dumping);
--- /dev/null
+#include "queued_entry.h"
+#include "subreader_memory.h"
+#include "compression_common.h"
+
+/* Don't flush, even on an explicit flush request, if
+ * the buffer is not filled at least this much.
+ *
+ * Should be lower than the full chunk size by at least
+ * a full log entry including metadata (about 4200).
+ *
+ * Keep in mind flushes occur periodically, every 99 logs
+ * and/or 600ms by default, so while it would be good to
+ * respect flush requests they happen a bit too frequently
+ * to do that at the moment (since compressing single logs
+ * gives fairly underwhelming results). */
+#define NO_FLUSH_SIZE (COMPRESSION_CHUNK_SIZE - 10000)
+
+static void subreader_memory_free(void *userdata)
+{
+ struct subreader_memory *const srm = (struct subreader_memory *) userdata;
+ assert(srm);
+
+ free(srm->to_be_compressed);
+}
+
+static int subreader_memory_flush(void *userdata, struct timespec ts, int flush_time)
+{
+ struct subreader_memory *const srm = (struct subreader_memory *) userdata;
+ assert(srm);
+
+ /* Compression minimum buffer requirement.
+ * Keep this even if excessive flushing is
+ * fixed and the check below is removed. */
+ if (srm->to_be_compressed_size < 16)
+ return 0;
+
+ if (srm->to_be_compressed_size < NO_FLUSH_SIZE)
+ return 0;
+
+ /* Disregard failures (other than a mention), since
+ * we can't do anything about insufficient memory. */
+ if (!log_compressed_storage_add_new_entry(srm->storage, srm->to_be_compressed, srm->to_be_compressed_size)) {
+ fprintf(stderr, "subreader_memory: failed to compress a chunk of size %zu\n"
+ , srm->to_be_compressed_size
+ );
+ }
+ srm->to_be_compressed_size = 0;
+
+ return 0;
+}
+
+static int subreader_memory_apply_log(const struct subreader_common *sub, const struct dlogutil_entry *due)
+{
+ assert(sub);
+ assert(due);
+
+ struct subreader_memory *const srm = (struct subreader_memory *) sub->sub_userdata;
+ assert(srm);
+
+ if (due->len + srm->to_be_compressed_size > COMPRESSION_CHUNK_SIZE) {
+ subreader_memory_flush(srm, (struct timespec){0, 0}, 0);
+ if (due->len + srm->to_be_compressed_size > COMPRESSION_CHUNK_SIZE)
+ return -1;
+ }
+
+ memcpy(srm->to_be_compressed + srm->to_be_compressed_size, due, due->len);
+ srm->to_be_compressed_size += due->len;
+
+ return 0;
+}
+
+int reader_add_subreader_memory(struct reader_common *reader, struct log_filter *filter, struct log_compressed_storage *storage)
+{
+ assert(reader);
+ assert(filter);
+ assert(storage);
+
+ struct subreader_common *const sub = malloc(sizeof *sub);
+ struct subreader_memory *const srm = malloc(sizeof *srm);
+ if (!sub || !srm)
+ goto free_structs;
+
+ srm->storage = storage;
+ srm->to_be_compressed_size = 0;
+ srm->to_be_compressed = malloc(COMPRESSION_CHUNK_SIZE); // FIXME: Maybe should just be `char to_be_compressed[COMPRESSION_CHUNK_SIZE]` instead? Looks too huge though
+ if (!srm->to_be_compressed)
+ goto free_srm_innards;
+
+ sub->sub_userdata = srm;
+ sub->sub_destroy = subreader_memory_free;
+ sub->sub_apply_log = subreader_memory_apply_log;
+ sub->sub_flush = subreader_memory_flush;
+ sub->filter = log_filter_move(filter);
+
+ list_add(&reader->subs, sub);
+
+ return 0;
+
+free_srm_innards:
+ free(srm->to_be_compressed);
+
+free_structs:
+ free(srm);
+ free(sub);
+
+ return -ENOMEM;
+}
+
--- /dev/null
+#pragma once
+
+#include "reader_common.h"
+#include "log_compressed_storage.h"
+
+struct subreader_memory {
+ struct log_compressed_storage *storage;
+ char *to_be_compressed;
+ size_t to_be_compressed_size;
+};
+
+int reader_add_subreader_memory(struct reader_common *reader, struct log_filter *filter, struct log_compressed_storage *storage);
+
* @retval <0 Failure as denoted by value: -errno
*/
static int do_print(dlogutil_mode_e mode, unsigned int dump_size, int enabled_buffers, dlogutil_sorting_order_e sort_by,
- dlogutil_config_s *config, struct log_file *l_file)
+ dlogutil_config_s *config, struct log_file *l_file, const char *compress)
{
/* Optimisation for short-lived (i.e. dumping) instances.
*
dlogutil_config_buffer_add(config, (log_id_t) i);
dlogutil_state_s *state;
- if (mode == DLOGUTIL_MODE_MONITOR)
- r = dlogutil_config_mode_set_monitor(config);
- else if (mode == DLOGUTIL_MODE_CONTINUOUS)
- r = dlogutil_config_mode_set_continuous(config);
- else
- r = dlogutil_config_mode_set_dump(config, dump_size);
+ switch (mode) {
+ case DLOGUTIL_MODE_MONITOR: r = dlogutil_config_mode_set_monitor (config ); break;
+ case DLOGUTIL_MODE_CONTINUOUS: r = dlogutil_config_mode_set_continuous (config ); break;
+ case DLOGUTIL_MODE_DUMP: r = dlogutil_config_mode_set_dump (config, dump_size); break;
+ case DLOGUTIL_MODE_COMPRESSED_MEMORY_DUMP: r = dlogutil_config_mode_set_compressed_memory_dump(config, compress ); break;
+ default: assert(false);
+ }
if (!r)
r = dlogutil_config_connect(config, &state);
int r;
switch (pr.action) {
case ACTION_PRINT: {
- r = do_print(pr.mode, pr.dump_size, enabled_buffers, pr.sort_by, config, &l_file);
+ r = do_print(pr.mode, pr.dump_size, enabled_buffers, pr.sort_by, config, &l_file, pr.compression);
break;
}
case ACTION_GET_CAPACITY: {
size_t rotate_size_kbytes = DEFAULT_ROTATE_SIZE_KB;
size_t max_rotated = DEFAULT_ROTATE_NUM_FILES;
const char *file_path = NULL;
+ char *compression = NULL;
size_t write_buffer_size = 0;
int enabled_buffers = 0;
action_e action = ACTION_PRINT;
while (1) {
static const struct option long_options[] = {
- {"tid" , required_argument, NULL, 0},
- {"pid" , required_argument, NULL, 1},
- {"version", no_argument, NULL, 2},
- {"color" , required_argument, NULL, 3},
- {"sort-by", required_argument, NULL, 4},
- {"help" , no_argument, NULL, 'h'},
+ {"tid" , required_argument, NULL, 0},
+ {"pid" , required_argument, NULL, 1},
+ {"version" , no_argument, NULL, 2},
+ {"color" , required_argument, NULL, 3},
+ {"sort-by" , required_argument, NULL, 4},
+ {"compress", required_argument, NULL, 5},
+ {"help" , no_argument, NULL, 'h'},
{0}
};
int option = getopt_long(argc, argv, "cdmt:gsf:r:n:v:b:u:e:h", long_options, NULL);
return (struct parse_result) { .status = PARSE_BAD_SORT_BY, .bad_contents = optarg, };
}
break;
+ case 5: /* memory compression */
+ mode = DLOGUTIL_MODE_COMPRESSED_MEMORY_DUMP;
+ compression = optarg;
+ break;
case 'd':
mode = DLOGUTIL_MODE_DUMP;
break;
.mode = mode,
.dump_size = dump_size,
.sort_by = sort_by,
+ .compression = compression,
};
filterspecs = NULL;
pid_filters = NULL;
expected_ioctl = LOGGER_GET_LOG_LEN;
ioctl_ret = -67;
- assert(-67 == ops_logger.prepare_print(&fdi, 123, false, (struct log_filter *) 0xCA5CADE));
+ assert(-67 == ops_logger.prepare_print(&fdi, 123, false, (struct log_filter *) 0xCA5CADE, NULL));
expected_ioctl = -1;
- assert(!ops_logger.prepare_print(&fdi, 0, false, (struct log_filter *) 0xCA5CADE));
+ assert(!ops_logger.prepare_print(&fdi, 0, false, (struct log_filter *) 0xCA5CADE, NULL));
fail_read = true;
assert(-334 == ops_logger.read(&fdi));
expected_ioctl = LOGGER_GET_LOG_LEN;
ioctl_ret = 0;
- assert(1 == ops_logger.prepare_print(&fdi, 123, false, (struct log_filter *) 0xCA5CADE));
+ assert(1 == ops_logger.prepare_print(&fdi, 123, false, (struct log_filter *) 0xCA5CADE, NULL));
monitor_more_reads(3, &fdi);
monitor_more_reads(6, &fdi);
correct_send_data = "dlogutil filter0:V *:S";
correct_send_datalen = strlen(correct_send_data) + 1;
send_return = -1;
- assert(ops_pipe.prepare_print(&info, false, false, filter) == -1);
+ assert(ops_pipe.prepare_print(&info, false, false, filter, NULL) == -1);
correct_send_data = "dlogutil -d filter0:V *:S";
correct_send_datalen = strlen(correct_send_data) + 1;
- assert(ops_pipe.prepare_print(&info, true, false, filter) == -1);
+ assert(ops_pipe.prepare_print(&info, true, false, filter, NULL) == -1);
log_filter_set_filterspec(filter, "filter1");
correct_send_data = "dlogutil filter1:V filter0:V *:S";
correct_send_datalen = strlen(correct_send_data) + 1;
- assert(ops_pipe.prepare_print(&info, false, false, filter) == -1);
+ assert(ops_pipe.prepare_print(&info, false, false, filter, NULL) == -1);
log_filter_set_tid(filter, 123);
log_filter_set_pid(filter, 456);
correct_send_data = "dlogutil --pid 456 --tid 123 filter1:V filter0:V *:S";
correct_send_datalen = strlen(correct_send_data) + 1;
- assert(ops_pipe.prepare_print(&info, false, false, filter) == -1);
+ assert(ops_pipe.prepare_print(&info, false, false, filter, NULL) == -1);
+
+ correct_request = DLOG_REQ_HANDLE_COMPRESSED_LOGUTIL;
+ correct_send_data = "dlogutil --compress membackup --pid 456 --tid 123 filter1:V filter0:V *:S";
+ correct_send_datalen = strlen(correct_send_data) + 1;
+ assert(ops_pipe.prepare_print(&info, false, false, filter, (char *) "membackup") == -1);
send_return = 0;
recv_pipe_fail = true;
- assert(ops_pipe.prepare_print(&info, false, false, filter) == -EIO);
+ assert(ops_pipe.prepare_print(&info, false, false, filter, (char *) "membackup") == -EIO);
recv_pipe_fail = false;
// Too long filter list test
/* This is a long filter. In fact, it's too long to be accepted
* by ops_pipe.prepare_print, which should return -E2BIG. */
log_filter_set_filterspec(filter, too_long_filter);
- assert(ops_pipe.prepare_print(&info, false, false, filter) == -E2BIG);
+ assert(ops_pipe.prepare_print(&info, false, false, filter, (char *) "membackup") == -E2BIG);
log_filter_free(filter);
correct_sockfd = 0;
// NB: both strings end in a space; the input filter should be parsed correctly despite this
strncat(very_big_string_for_comparison, "*:S", sizeof(very_big_string_for_comparison) - 1);
- correct_sockfd = 3;
+ correct_sockfd = 4;
correct_request = DLOG_REQ_HANDLE_LOGUTIL;
send_return = 0;
correct_send_datalen = strlen(very_big_string_for_comparison) + 1;
/* long data test */
log_filter_set_filterspec(filter, just_big_enough_filter);
- assert(ops_pipe.prepare_print(&info, false, false, filter) == 0);
+ assert(ops_pipe.prepare_print(&info, false, false, filter, NULL) == 0);
correct_request = DLOG_REQ_CLEAR;
correct_send_data = NULL;
recv_pipe_fail = false;
correct_send_data = "dlogutil --pid 456 --tid 123 filter1:V filter0:V *:S";
correct_send_datalen = strlen(correct_send_data) + 1;
- assert(ops_pipe.prepare_print(&info, false, false, filter) == 0);
+ assert(ops_pipe.prepare_print(&info, false, false, filter, NULL) == 0);
assert(info.fd == 2);
+ correct_request = DLOG_REQ_HANDLE_COMPRESSED_LOGUTIL;
+ correct_send_data = "dlogutil --compress membackup --pid 456 --tid 123 filter1:V filter0:V *:S";
+ correct_send_datalen = strlen(correct_send_data) + 1;
+ assert(ops_pipe.prepare_print(&info, false, false, filter, (char *) "membackup") == 0);
+ assert(info.fd == 3);
+
log_filter_free(filter);
- correct_sockfd = 2;
+ correct_sockfd = 3;
read_errno = 0;
read_datalen = sizeof(dlogutil_entry_s) + 4;
assert(ops_pipe.has_log(&info));
assert(!ops_pipe.extract_entry(&info));
__real_free(read_data);
-
correct_sockfd = 1;
correct_request = DLOG_REQ_CLEAR;
correct_send_data = NULL;
projects / platform / core / system / dlog.git / commitdiff