From 8580d1f73db36e9383e674e388b4fb55828c0c66 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 2 Oct 2015 23:21:59 +0200 Subject: [PATCH] journal: rework vacuuming logic Implement a maximum limit on number of journal files to keep around. Enforcing a limit is useful on this since our performance when viewing pays a heavy penalty for each journal file to interleve. This setting is turned on now by default, and set to 100. Also, actully implement what 348ced909724a1331b85d57aede80a102a00e428 promised: use whatever we find on disk at startup as lower bound on how much disk space we can use. That commit introduced some provisions to implement this, but actually never did. This also adds "journalctl --vacuum-files=" to vacuum files on disk by their number explicitly. --- man/journalctl.xml | 26 ++- man/journald.conf.xml | 23 ++- src/journal/journal-file.c | 64 +++++-- src/journal/journal-file.h | 13 +- src/journal/journal-vacuum.c | 98 ++++++---- src/journal/journal-vacuum.h | 6 +- src/journal/journalctl.c | 24 ++- src/journal/journald-gperf.gperf | 2 + src/journal/journald-server.c | 320 +++++++++++++++++--------------- src/journal/journald-server.h | 9 +- src/journal/journald.c | 8 +- src/journal/journald.conf | 2 + src/journal/test-journal-interleaving.c | 4 +- src/journal/test-journal.c | 4 +- 14 files changed, 365 insertions(+), 238 deletions(-) diff --git a/man/journalctl.xml b/man/journalctl.xml index 305f1b9..3efcfbd 100644 --- a/man/journalctl.xml +++ b/man/journalctl.xml @@ -649,6 +649,7 @@ + Removes archived journal files until the disk space they use falls below the specified size (specified with @@ -658,15 +659,24 @@ timespan (specified with the usual s, min, h, days, months, - weeks, years - suffixes). Note that running - has only indirect effect on the output shown by + weeks, years suffixes), + or no more than the specified number of separate journal files + remain. Note that running has + only indirect effect on the output shown by as the latter includes active - journal files, while the former only operates on archived - journal files. and - may be combined in a single - invocation to enforce both a size and time limit on the - archived journal files. + journal files, while the the vacuuming operation only operates + on archived journal files. Similar, + might not actually reduce the + number of journal files to below the specified number, as it + will not remove active journal + files. , + and + may be combined in a single + invocation to enforce any combination of a size, a time and a + number of files limit on the archived journal + files. Specifying any of these three parameters as zero is + equivalent to not enforcing the specific limit, and is thus + redundant. diff --git a/man/journald.conf.xml b/man/journald.conf.xml index d6fe45d..109ca96 100644 --- a/man/journald.conf.xml +++ b/man/journald.conf.xml @@ -173,9 +173,11 @@ SystemMaxUse= SystemKeepFree= SystemMaxFileSize= + SystemMaxFiles= RuntimeMaxUse= RuntimeKeepFree= RuntimeMaxFileSize= + RuntimeMaxFiles= Enforce size limits on the journal files stored. The options prefixed with System @@ -197,8 +199,7 @@ names not ending with .journal or .journal~, so only such files, located in the appropriate directories, are taken into account when - calculating current disk usage. - + calculating current disk usage. SystemMaxUse= and RuntimeMaxUse= control how much disk space @@ -212,13 +213,14 @@ The first pair defaults to 10% and the second to 15% of the size of the respective file system. If the file system is nearly full and either SystemKeepFree= or - RuntimeKeepFree= is violated when - systemd-journald is started, the value will be raised to + RuntimeKeepFree= are violated when + systemd-journald is started, the limit will be raised to the percentage that is actually free. This means that if there was enough free space before and journal files were created, and subsequently something else causes the file system to fill up, journald will stop using more space, but it will not be - removing existing files to go reduce footprint either. + removing existing files to reduce footprint again + either. SystemMaxFileSize= and RuntimeMaxFileSize= control how large @@ -228,13 +230,22 @@ eighth of the values configured with SystemMaxUse= and RuntimeMaxUse=, so that usually seven - rotated journal files are kept as history. + rotated journal files are kept as history. Specify values in bytes or use K, M, G, T, P, E as units for the specified sizes (equal to 1024, 1024²,... bytes). Note that size limits are enforced synchronously when journal files are extended, and no explicit rotation step triggered by time is needed. + + SystemMaxFiles= and + RuntimeMaxFiles= control how many + individual journal files to keep at maximum. Note that only + archived files are deleted to reduce the number of files until + this limit is reached; active files will stay around. This + means that in effect there might still be more journal files + around in total than this limit after a vacuuming operation is + complete. This setting defaults to 100. diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c index 085d5f3..1071c6d 100644 --- a/src/journal/journal-file.c +++ b/src/journal/journal-file.c @@ -49,6 +49,9 @@ #define DEFAULT_MAX_USE_LOWER (1ULL*1024ULL*1024ULL) /* 1 MiB */ #define DEFAULT_MAX_USE_UPPER (4ULL*1024ULL*1024ULL*1024ULL) /* 4 GiB */ +/* This is the default minimal use limit, how much we'll use even if keep_free suggests otherwise. */ +#define DEFAULT_MIN_USE (1ULL*1024ULL*1024ULL) /* 1 MiB */ + /* This is the upper bound if we deduce max_size from max_use */ #define DEFAULT_MAX_SIZE_UPPER (128ULL*1024ULL*1024ULL) /* 128 MiB */ @@ -60,6 +63,9 @@ * size */ #define DEFAULT_KEEP_FREE (1024ULL*1024ULL) /* 1 MB */ +/* This is the default maximum number of journal files to keep around. */ +#define DEFAULT_N_MAX_FILES (100) + /* n_data was the first entry we added after the initial file format design */ #define HEADER_SIZE_MIN ALIGN64(offsetof(Header, n_data)) @@ -2957,16 +2963,35 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6 return r; } +void journal_reset_metrics(JournalMetrics *m) { + assert(m); + + /* Set everything to "pick automatic values". */ + + *m = (JournalMetrics) { + .min_use = (uint64_t) -1, + .max_use = (uint64_t) -1, + .min_size = (uint64_t) -1, + .max_size = (uint64_t) -1, + .keep_free = (uint64_t) -1, + .n_max_files = (uint64_t) -1, + }; +} + void journal_default_metrics(JournalMetrics *m, int fd) { - uint64_t fs_size = 0; + char a[FORMAT_BYTES_MAX], b[FORMAT_BYTES_MAX], c[FORMAT_BYTES_MAX], d[FORMAT_BYTES_MAX], e[FORMAT_BYTES_MAX]; struct statvfs ss; - char a[FORMAT_BYTES_MAX], b[FORMAT_BYTES_MAX], c[FORMAT_BYTES_MAX], d[FORMAT_BYTES_MAX]; + uint64_t fs_size; assert(m); assert(fd >= 0); if (fstatvfs(fd, &ss) >= 0) fs_size = ss.f_frsize * ss.f_blocks; + else { + log_debug_errno(errno, "Failed to detremine disk size: %m"); + fs_size = 0; + } if (m->max_use == (uint64_t) -1) { @@ -2983,10 +3008,16 @@ void journal_default_metrics(JournalMetrics *m, int fd) { } else { m->max_use = PAGE_ALIGN(m->max_use); - if (m->max_use < JOURNAL_FILE_SIZE_MIN*2) + if (m->max_use != 0 && m->max_use < JOURNAL_FILE_SIZE_MIN*2) m->max_use = JOURNAL_FILE_SIZE_MIN*2; } + if (m->min_use == (uint64_t) -1) + m->min_use = DEFAULT_MIN_USE; + + if (m->min_use > m->max_use) + m->min_use = m->max_use; + if (m->max_size == (uint64_t) -1) { m->max_size = PAGE_ALIGN(m->max_use / 8); /* 8 chunks */ @@ -2995,11 +3026,13 @@ void journal_default_metrics(JournalMetrics *m, int fd) { } else m->max_size = PAGE_ALIGN(m->max_size); - if (m->max_size < JOURNAL_FILE_SIZE_MIN) - m->max_size = JOURNAL_FILE_SIZE_MIN; + if (m->max_size != 0) { + if (m->max_size < JOURNAL_FILE_SIZE_MIN) + m->max_size = JOURNAL_FILE_SIZE_MIN; - if (m->max_size*2 > m->max_use) - m->max_use = m->max_size*2; + if (m->max_use != 0 && m->max_size*2 > m->max_use) + m->max_use = m->max_size*2; + } if (m->min_size == (uint64_t) -1) m->min_size = JOURNAL_FILE_SIZE_MIN; @@ -3009,7 +3042,7 @@ void journal_default_metrics(JournalMetrics *m, int fd) { if (m->min_size < JOURNAL_FILE_SIZE_MIN) m->min_size = JOURNAL_FILE_SIZE_MIN; - if (m->min_size > m->max_size) + if (m->max_size != 0 && m->min_size > m->max_size) m->max_size = m->min_size; } @@ -3025,11 +3058,16 @@ void journal_default_metrics(JournalMetrics *m, int fd) { m->keep_free = DEFAULT_KEEP_FREE; } - log_debug("Fixed max_use=%s max_size=%s min_size=%s keep_free=%s", - format_bytes(a, sizeof(a), m->max_use), - format_bytes(b, sizeof(b), m->max_size), - format_bytes(c, sizeof(c), m->min_size), - format_bytes(d, sizeof(d), m->keep_free)); + if (m->n_max_files == (uint64_t) -1) + m->n_max_files = DEFAULT_N_MAX_FILES; + + log_debug("Fixed min_use=%s max_use=%s max_size=%s min_size=%s keep_free=%s n_max_files=%" PRIu64, + format_bytes(a, sizeof(a), m->min_use), + format_bytes(b, sizeof(b), m->max_use), + format_bytes(c, sizeof(c), m->max_size), + format_bytes(d, sizeof(d), m->min_size), + format_bytes(e, sizeof(e), m->keep_free), + m->n_max_files); } int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to) { diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h index 52e6b7a..f2c0735 100644 --- a/src/journal/journal-file.h +++ b/src/journal/journal-file.h @@ -36,11 +36,13 @@ #include "hashmap.h" typedef struct JournalMetrics { - uint64_t max_use; - uint64_t use; - uint64_t max_size; - uint64_t min_size; - uint64_t keep_free; + /* For all these: -1 means "pick automatically", and 0 means "no limit enforced" */ + uint64_t max_size; /* how large journal files grow at max */ + uint64_t min_size; /* how large journal files grow at least */ + uint64_t max_use; /* how much disk space to use in total at max, keep_free permitting */ + uint64_t min_use; /* how much disk space to use in total at least, even if keep_free says not to */ + uint64_t keep_free; /* how much to keep free on disk */ + uint64_t n_max_files; /* how many files to keep around at max */ } JournalMetrics; typedef enum direction { @@ -223,6 +225,7 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal); void journal_file_post_change(JournalFile *f); +void journal_reset_metrics(JournalMetrics *m); void journal_default_metrics(JournalMetrics *m, int fd); int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to); diff --git a/src/journal/journal-vacuum.c b/src/journal/journal-vacuum.c index 394f7be..a394066 100644 --- a/src/journal/journal-vacuum.c +++ b/src/journal/journal-vacuum.c @@ -140,22 +140,24 @@ static int journal_file_empty(int dir_fd, const char *name) { int journal_directory_vacuum( const char *directory, uint64_t max_use, + uint64_t n_max_files, usec_t max_retention_usec, usec_t *oldest_usec, bool verbose) { _cleanup_closedir_ DIR *d = NULL; - int r = 0; struct vacuum_info *list = NULL; - unsigned n_list = 0, i; + unsigned n_list = 0, i, n_active_files = 0; size_t n_allocated = 0; uint64_t sum = 0, freed = 0; usec_t retention_limit = 0; char sbytes[FORMAT_BYTES_MAX]; + struct dirent *de; + int r; assert(directory); - if (max_use <= 0 && max_retention_usec <= 0) + if (max_use <= 0 && max_retention_usec <= 0 && n_max_files <= 0) return 0; if (max_retention_usec > 0) { @@ -170,27 +172,20 @@ int journal_directory_vacuum( if (!d) return -errno; - for (;;) { - struct dirent *de; - size_t q; - struct stat st; - char *p; + FOREACH_DIRENT_ALL(de, d, r = -errno; goto finish) { + unsigned long long seqnum = 0, realtime; + _cleanup_free_ char *p = NULL; sd_id128_t seqnum_id; bool have_seqnum; + uint64_t size; + struct stat st; + size_t q; - errno = 0; - de = readdir(d); - if (!de && errno != 0) { - r = -errno; - goto finish; - } - - if (!de) - break; - - if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) + if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) { + log_debug_errno(errno, "Failed to stat file %s while vacuuming, ignoring: %m", de->d_name); continue; + } if (!S_ISREG(st.st_mode)) continue; @@ -199,15 +194,20 @@ int journal_directory_vacuum( if (endswith(de->d_name, ".journal")) { - /* Vacuum archived files */ + /* Vacuum archived files. Active files are + * left around */ - if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) + if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) { + n_active_files++; continue; + } if (de->d_name[q-8-16-1] != '-' || de->d_name[q-8-16-1-16-1] != '-' || - de->d_name[q-8-16-1-16-1-32-1] != '@') + de->d_name[q-8-16-1-16-1-32-1] != '@') { + n_active_files++; continue; + } p = strdup(de->d_name); if (!p) { @@ -218,11 +218,13 @@ int journal_directory_vacuum( de->d_name[q-8-16-1-16-1] = 0; if (sd_id128_from_string(de->d_name + q-8-16-1-16-1-32, &seqnum_id) < 0) { free(p); + n_active_files++; continue; } if (sscanf(de->d_name + q-8-16-1-16, "%16llx-%16llx.journal", &seqnum, &realtime) != 2) { free(p); + n_active_files++; continue; } @@ -233,12 +235,16 @@ int journal_directory_vacuum( /* Vacuum corrupted files */ - if (q < 1 + 16 + 1 + 16 + 8 + 1) + if (q < 1 + 16 + 1 + 16 + 8 + 1) { + n_active_files ++; continue; + } if (de->d_name[q-1-8-16-1] != '-' || - de->d_name[q-1-8-16-1-16-1] != '@') + de->d_name[q-1-8-16-1-16-1] != '@') { + n_active_files ++; continue; + } p = strdup(de->d_name); if (!p) { @@ -248,54 +254,68 @@ int journal_directory_vacuum( if (sscanf(de->d_name + q-1-8-16-1-16, "%16llx-%16llx.journal~", &realtime, &tmp) != 2) { free(p); + n_active_files ++; continue; } have_seqnum = false; - } else - /* We do not vacuum active files or unknown files! */ + } else { + /* We do not vacuum unknown files! */ + log_debug("Not vacuuming unknown file %s.", de->d_name); continue; + } - if (journal_file_empty(dirfd(d), p)) { - /* Always vacuum empty non-online files. */ + size = 512UL * (uint64_t) st.st_blocks; - uint64_t size = 512UL * (uint64_t) st.st_blocks; + r = journal_file_empty(dirfd(d), p); + if (r < 0) { + log_debug_errno(r, "Failed check if %s is empty, ignoring: %m", p); + continue; + } + if (r > 0) { + /* Always vacuum empty non-online files. */ if (unlinkat(dirfd(d), p, 0) >= 0) { - log_full(verbose ? LOG_INFO : LOG_DEBUG, "Deleted empty archived journal %s/%s (%s).", directory, p, format_bytes(sbytes, sizeof(sbytes), size)); + + log_full(verbose ? LOG_INFO : LOG_DEBUG, + "Deleted empty archived journal %s/%s (%s).", directory, p, format_bytes(sbytes, sizeof(sbytes), size)); + freed += size; } else if (errno != ENOENT) log_warning_errno(errno, "Failed to delete empty archived journal %s/%s: %m", directory, p); - free(p); continue; } - patch_realtime(directory, p, &st, &realtime); + patch_realtime(dirfd(d), p, &st, &realtime); if (!GREEDY_REALLOC(list, n_allocated, n_list + 1)) { - free(p); r = -ENOMEM; goto finish; } list[n_list].filename = p; - list[n_list].usage = 512UL * (uint64_t) st.st_blocks; + list[n_list].usage = size; list[n_list].seqnum = seqnum; list[n_list].realtime = realtime; list[n_list].seqnum_id = seqnum_id; list[n_list].have_seqnum = have_seqnum; - - sum += list[n_list].usage; - n_list ++; + + p = NULL; + sum += size; } qsort_safe(list, n_list, sizeof(struct vacuum_info), vacuum_compare); for (i = 0; i < n_list; i++) { + unsigned left; + + left = n_active_files + n_list - i; + if ((max_retention_usec <= 0 || list[i].realtime >= retention_limit) && - (max_use <= 0 || sum <= max_use)) + (max_use <= 0 || sum <= max_use) && + (n_max_files <= 0 || left <= n_max_files)) break; if (unlinkat(dirfd(d), list[i].filename, 0) >= 0) { @@ -314,6 +334,8 @@ int journal_directory_vacuum( if (oldest_usec && i < n_list && (*oldest_usec == 0 || list[i].realtime < *oldest_usec)) *oldest_usec = list[i].realtime; + r = 0; + finish: for (i = 0; i < n_list; i++) free(list[i].filename); diff --git a/src/journal/journal-vacuum.h b/src/journal/journal-vacuum.h index c45cc31..49ab90a 100644 --- a/src/journal/journal-vacuum.h +++ b/src/journal/journal-vacuum.h @@ -21,5 +21,9 @@ along with systemd; If not, see . ***/ +#include +#include -int journal_directory_vacuum(const char *directory, uint64_t max_use, usec_t max_retention_usec, usec_t *oldest_usec, bool vacuum); +#include "time-util.h" + +int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t n_max_files, usec_t max_retention_usec, usec_t *oldest_usec, bool verbose); diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c index 28ccb80..d9851db 100644 --- a/src/journal/journalctl.c +++ b/src/journal/journalctl.c @@ -106,8 +106,9 @@ static bool arg_reverse = false; static int arg_journal_type = 0; static const char *arg_root = NULL; static const char *arg_machine = NULL; -static uint64_t arg_vacuum_size = (uint64_t) -1; -static usec_t arg_vacuum_time = USEC_INFINITY; +static uint64_t arg_vacuum_size = 0; +static uint64_t arg_vacuum_n_files = 0; +static usec_t arg_vacuum_time = 0; static enum { ACTION_SHOW, @@ -235,7 +236,8 @@ static void help(void) { " --new-id128 Generate a new 128-bit ID\n" " --disk-usage Show total disk usage of all journal files\n" " --vacuum-size=BYTES Reduce disk usage below specified size\n" - " --vacuum-time=TIME Remove journal files older than specified date\n" + " --vacuum-files=INT Leave only the specified number of journal files\n" + " --vacuum-time=TIME Remove journal files older than specified time\n" " --flush Flush all journal data from /run into /var\n" " --rotate Request immediate rotation of the journal files\n" " --header Show journal header information\n" @@ -281,6 +283,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_FLUSH, ARG_ROTATE, ARG_VACUUM_SIZE, + ARG_VACUUM_FILES, ARG_VACUUM_TIME, }; @@ -335,6 +338,7 @@ static int parse_argv(int argc, char *argv[]) { { "flush", no_argument, NULL, ARG_FLUSH }, { "rotate", no_argument, NULL, ARG_ROTATE }, { "vacuum-size", required_argument, NULL, ARG_VACUUM_SIZE }, + { "vacuum-files", required_argument, NULL, ARG_VACUUM_FILES }, { "vacuum-time", required_argument, NULL, ARG_VACUUM_TIME }, {} }; @@ -540,6 +544,16 @@ static int parse_argv(int argc, char *argv[]) { arg_action = ACTION_VACUUM; break; + case ARG_VACUUM_FILES: + r = safe_atou64(optarg, &arg_vacuum_n_files); + if (r < 0) { + log_error("Failed to parse vacuum files: %s", optarg); + return r; + } + + arg_action = ACTION_VACUUM; + break; + case ARG_VACUUM_TIME: r = parse_sec(optarg, &arg_vacuum_time); if (r < 0) { @@ -1929,9 +1943,9 @@ int main(int argc, char *argv[]) { if (d->is_root) continue; - q = journal_directory_vacuum(d->path, arg_vacuum_size, arg_vacuum_time, NULL, true); + q = journal_directory_vacuum(d->path, arg_vacuum_size, arg_vacuum_n_files, arg_vacuum_time, NULL, true); if (q < 0) { - log_error_errno(q, "Failed to vacuum: %m"); + log_error_errno(q, "Failed to vacuum %s: %m", d->path); r = q; } } diff --git a/src/journal/journald-gperf.gperf b/src/journal/journald-gperf.gperf index bf7c773..c154610 100644 --- a/src/journal/journald-gperf.gperf +++ b/src/journal/journald-gperf.gperf @@ -24,9 +24,11 @@ Journal.RateLimitBurst, config_parse_unsigned, 0, offsetof(Server, rate_li Journal.SystemMaxUse, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.max_use) Journal.SystemMaxFileSize, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.max_size) Journal.SystemKeepFree, config_parse_iec_uint64, 0, offsetof(Server, system_metrics.keep_free) +Journal.SystemMaxFiles, config_parse_uint64, 0, offsetof(Server, system_metrics.n_max_files) Journal.RuntimeMaxUse, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.max_use) Journal.RuntimeMaxFileSize, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.max_size) Journal.RuntimeKeepFree, config_parse_iec_uint64, 0, offsetof(Server, runtime_metrics.keep_free) +Journal.RuntimeMaxFiles, config_parse_uint64, 0, offsetof(Server, runtime_metrics.n_max_files) Journal.MaxRetentionSec, config_parse_sec, 0, offsetof(Server, max_retention_usec) Journal.MaxFileSec, config_parse_sec, 0, offsetof(Server, max_file_usec) Journal.ForwardToSyslog, config_parse_bool, 0, offsetof(Server, forward_to_syslog) diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 296e28e..fb172b7 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -19,45 +19,44 @@ along with systemd; If not, see . ***/ -#include -#include #include -#include -#include - #ifdef HAVE_SELINUX #include #endif +#include +#include +#include +#include -#include - +#include "libudev.h" +#include "sd-daemon.h" #include "sd-journal.h" #include "sd-messages.h" -#include "sd-daemon.h" -#include "mkdir.h" -#include "rm-rf.h" -#include "hashmap.h" -#include "journal-file.h" -#include "socket-util.h" + +#include "acl-util.h" #include "cgroup-util.h" -#include "missing.h" #include "conf-parser.h" -#include "selinux-util.h" -#include "acl-util.h" #include "formats-util.h" -#include "process-util.h" +#include "hashmap.h" #include "hostname-util.h" +#include "missing.h" +#include "mkdir.h" +#include "process-util.h" +#include "rm-rf.h" +#include "selinux-util.h" #include "signal-util.h" +#include "socket-util.h" +#include "journal-authenticate.h" +#include "journal-file.h" #include "journal-internal.h" #include "journal-vacuum.h" -#include "journal-authenticate.h" -#include "journald-rate-limit.h" +#include "journald-audit.h" #include "journald-kmsg.h" -#include "journald-syslog.h" -#include "journald-stream.h" #include "journald-native.h" -#include "journald-audit.h" +#include "journald-rate-limit.h" #include "journald-server.h" +#include "journald-stream.h" +#include "journald-syslog.h" #define USER_JOURNALS_MAX 1024 @@ -66,88 +65,61 @@ #define DEFAULT_RATE_LIMIT_BURST 1000 #define DEFAULT_MAX_FILE_USEC USEC_PER_MONTH -#define RECHECK_AVAILABLE_SPACE_USEC (30*USEC_PER_SEC) - -static const char* const storage_table[_STORAGE_MAX] = { - [STORAGE_AUTO] = "auto", - [STORAGE_VOLATILE] = "volatile", - [STORAGE_PERSISTENT] = "persistent", - [STORAGE_NONE] = "none" -}; - -DEFINE_STRING_TABLE_LOOKUP(storage, Storage); -DEFINE_CONFIG_PARSE_ENUM(config_parse_storage, storage, Storage, "Failed to parse storage setting"); +#define RECHECK_SPACE_USEC (30*USEC_PER_SEC) -static const char* const split_mode_table[_SPLIT_MAX] = { - [SPLIT_LOGIN] = "login", - [SPLIT_UID] = "uid", - [SPLIT_NONE] = "none", -}; - -DEFINE_STRING_TABLE_LOOKUP(split_mode, SplitMode); -DEFINE_CONFIG_PARSE_ENUM(config_parse_split_mode, split_mode, SplitMode, "Failed to parse split mode setting"); - -static uint64_t available_space(Server *s, bool verbose) { - char ids[33]; - _cleanup_free_ char *p = NULL; - sd_id128_t machine; - struct statvfs ss; - uint64_t sum = 0, ss_avail = 0, avail = 0; - int r; +static int determine_space_for( + Server *s, + JournalMetrics *metrics, + const char *path, + const char *name, + bool verbose, + bool patch_min_use, + uint64_t *available, + uint64_t *limit) { + + uint64_t sum = 0, ss_avail, avail; _cleanup_closedir_ DIR *d = NULL; + struct dirent *de; + struct statvfs ss; + const char *p; usec_t ts; - const char *f; - JournalMetrics *m; - ts = now(CLOCK_MONOTONIC); - - if (s->cached_available_space_timestamp + RECHECK_AVAILABLE_SPACE_USEC > ts - && !verbose) - return s->cached_available_space; + assert(s); + assert(metrics); + assert(path); + assert(name); - r = sd_id128_get_machine(&machine); - if (r < 0) - return 0; + ts = now(CLOCK_MONOTONIC); - if (s->system_journal) { - f = "/var/log/journal/"; - m = &s->system_metrics; - } else { - f = "/run/log/journal/"; - m = &s->runtime_metrics; - } + if (!verbose && s->cached_space_timestamp + RECHECK_SPACE_USEC > ts) { - assert(m); + if (available) + *available = s->cached_space_available; + if (limit) + *limit = s->cached_space_limit; - p = strappend(f, sd_id128_to_string(machine, ids)); - if (!p) return 0; + } + p = strjoina(path, SERVER_MACHINE_ID(s)); d = opendir(p); if (!d) - return 0; + return log_full_errno(errno == ENOENT ? LOG_DEBUG : LOG_ERR, errno, "Failed to open %s: %m", p); if (fstatvfs(dirfd(d), &ss) < 0) - return 0; + return log_error_errno(errno, "Failed to fstatvfs(%s): %m", p); - for (;;) { + FOREACH_DIRENT_ALL(de, d, break) { struct stat st; - struct dirent *de; - - errno = 0; - de = readdir(d); - if (!de && errno != 0) - return 0; - - if (!de) - break; if (!endswith(de->d_name, ".journal") && !endswith(de->d_name, ".journal~")) continue; - if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) + if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) { + log_debug_errno(errno, "Failed to stat %s/%s, ignoring: %m", p, de->d_name); continue; + } if (!S_ISREG(st.st_mode)) continue; @@ -155,39 +127,66 @@ static uint64_t available_space(Server *s, bool verbose) { sum += (uint64_t) st.st_blocks * 512UL; } - ss_avail = ss.f_bsize * ss.f_bavail; + /* If request, then let's bump the min_use limit to the + * current usage on disk. We do this when starting up and + * first opening the journal files. This way sudden spikes in + * disk usage will not cause journald to vacuum files without + * bounds. Note that this means that only a restart of + * journald will make it reset this value. */ - /* If we reached a high mark, we will always allow this much - * again, unless usage goes above max_use. This watermark - * value is cached so that we don't give up space on pressure, - * but hover below the maximum usage. */ + if (patch_min_use) + metrics->min_use = MAX(metrics->min_use, sum); - if (m->use < sum) - m->use = sum; - - avail = LESS_BY(ss_avail, m->keep_free); + ss_avail = ss.f_bsize * ss.f_bavail; + avail = LESS_BY(ss_avail, metrics->keep_free); - s->cached_available_space = LESS_BY(MIN(m->max_use, avail), sum); - s->cached_available_space_timestamp = ts; + s->cached_space_limit = MIN(MAX(sum + avail, metrics->min_use), metrics->max_use); + s->cached_space_available = LESS_BY(s->cached_space_limit, sum); + s->cached_space_timestamp = ts; if (verbose) { char fb1[FORMAT_BYTES_MAX], fb2[FORMAT_BYTES_MAX], fb3[FORMAT_BYTES_MAX], - fb4[FORMAT_BYTES_MAX], fb5[FORMAT_BYTES_MAX]; + fb4[FORMAT_BYTES_MAX], fb5[FORMAT_BYTES_MAX], fb6[FORMAT_BYTES_MAX]; server_driver_message(s, SD_MESSAGE_JOURNAL_USAGE, - "%s is currently using %s.\n" + "%s (%s) is currently using %s.\n" "Maximum allowed usage is set to %s.\n" "Leaving at least %s free (of currently available %s of space).\n" - "Enforced usage limit is thus %s.", - s->system_journal ? "Permanent journal (/var/log/journal/)" : "Runtime journal (/run/log/journal/)", + "Enforced usage limit is thus %s, of which %s are still available.", + name, path, format_bytes(fb1, sizeof(fb1), sum), - format_bytes(fb2, sizeof(fb2), m->max_use), - format_bytes(fb3, sizeof(fb3), m->keep_free), + format_bytes(fb2, sizeof(fb2), metrics->max_use), + format_bytes(fb3, sizeof(fb3), metrics->keep_free), format_bytes(fb4, sizeof(fb4), ss_avail), - format_bytes(fb5, sizeof(fb5), s->cached_available_space + sum)); + format_bytes(fb5, sizeof(fb5), s->cached_space_limit), + format_bytes(fb6, sizeof(fb6), s->cached_space_available)); + } + + if (available) + *available = s->cached_space_available; + if (limit) + *limit = s->cached_space_limit; + + return 1; +} + +static int determine_space(Server *s, bool verbose, bool patch_min_use, uint64_t *available, uint64_t *limit) { + JournalMetrics *metrics; + const char *path, *name; + + assert(s); + + if (s->system_journal) { + path = "/var/log/journal/"; + metrics = &s->system_metrics; + name = "System journal"; + } else { + path = "/run/log/journal/"; + metrics = &s->runtime_metrics; + name = "Runtime journal"; } - return s->cached_available_space; + return determine_space_for(s, metrics, path, name, verbose, patch_min_use, available, limit); } void server_fix_perms(Server *s, JournalFile *f, uid_t uid) { @@ -326,8 +325,8 @@ void server_rotate(Server *s) { log_debug("Rotating..."); - do_rotate(s, &s->runtime_journal, "runtime", false, 0); - do_rotate(s, &s->system_journal, "system", s->seal, 0); + (void) do_rotate(s, &s->runtime_journal, "runtime", false, 0); + (void) do_rotate(s, &s->system_journal, "system", s->seal, 0); ORDERED_HASHMAP_FOREACH_KEY(f, k, s->user_journals, i) { r = do_rotate(s, &f, "user", s->seal, PTR_TO_UINT32(k)); @@ -368,43 +367,50 @@ void server_sync(Server *s) { static void do_vacuum( Server *s, - const char *id, JournalFile *f, - const char* path, - JournalMetrics *metrics) { + JournalMetrics *metrics, + const char *path, + const char *name, + bool verbose, + bool patch_min_use) { const char *p; + uint64_t limit; int r; + assert(s); + assert(metrics); + assert(path); + assert(name); + if (!f) return; - p = strjoina(path, id); - r = journal_directory_vacuum(p, metrics->max_use, s->max_retention_usec, &s->oldest_file_usec, false); + p = strjoina(path, SERVER_MACHINE_ID(s)); + + limit = metrics->max_use; + (void) determine_space_for(s, metrics, path, name, verbose, patch_min_use, NULL, &limit); + + r = journal_directory_vacuum(p, limit, metrics->n_max_files, s->max_retention_usec, &s->oldest_file_usec, verbose); if (r < 0 && r != -ENOENT) - log_error_errno(r, "Failed to vacuum %s: %m", p); + log_warning_errno(r, "Failed to vacuum %s, ignoring: %m", p); } -void server_vacuum(Server *s) { - char ids[33]; - sd_id128_t machine; - int r; +int server_vacuum(Server *s, bool verbose, bool patch_min_use) { + assert(s); log_debug("Vacuuming..."); s->oldest_file_usec = 0; - r = sd_id128_get_machine(&machine); - if (r < 0) { - log_error_errno(r, "Failed to get machine ID: %m"); - return; - } - sd_id128_to_string(machine, ids); + do_vacuum(s, s->system_journal, &s->system_metrics, "/var/log/journal/", "System journal", verbose, patch_min_use); + do_vacuum(s, s->runtime_journal, &s->runtime_metrics, "/run/log/journal/", "Runtime journal", verbose, patch_min_use); - do_vacuum(s, ids, s->system_journal, "/var/log/journal/", &s->system_metrics); - do_vacuum(s, ids, s->runtime_journal, "/run/log/journal/", &s->runtime_metrics); + s->cached_space_limit = 0; + s->cached_space_available = 0; + s->cached_space_timestamp = 0; - s->cached_available_space_timestamp = 0; + return 0; } static void server_cache_machine_id(Server *s) { @@ -502,7 +508,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned if (journal_file_rotate_suggested(f, s->max_file_usec)) { log_debug("%s: Journal header limits reached or header out-of-date, rotating.", f->path); server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); vacuumed = true; f = find_journal(s, uid); @@ -522,7 +528,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned } server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); f = find_journal(s, uid); if (!f) @@ -867,6 +873,7 @@ void server_dispatch_message( int rl, r; _cleanup_free_ char *path = NULL; + uint64_t available = 0; char *c; assert(s); @@ -906,9 +913,8 @@ void server_dispatch_message( } } - rl = journal_rate_limit_test(s->rate_limit, path, - priority & LOG_PRIMASK, available_space(s, false)); - + (void) determine_space(s, false, false, &available, NULL); + rl = journal_rate_limit_test(s->rate_limit, path, priority & LOG_PRIMASK, available); if (rl == 0) return; @@ -923,16 +929,8 @@ finish: static int system_journal_open(Server *s, bool flush_requested) { - int r; const char *fn; - sd_id128_t machine; - char ids[33]; - - r = sd_id128_get_machine(&machine); - if (r < 0) - return log_error_errno(r, "Failed to get machine id: %m"); - - sd_id128_to_string(machine, ids); + int r; if (!s->system_journal && (s->storage == STORAGE_PERSISTENT || s->storage == STORAGE_AUTO) && @@ -948,15 +946,15 @@ static int system_journal_open(Server *s, bool flush_requested) { if (s->storage == STORAGE_PERSISTENT) (void) mkdir_p("/var/log/journal/", 0755); - fn = strjoina("/var/log/journal/", ids); + fn = strjoina("/var/log/journal/", SERVER_MACHINE_ID(s)); (void) mkdir(fn, 0755); fn = strjoina(fn, "/system.journal"); r = journal_file_open_reliably(fn, O_RDWR|O_CREAT, 0640, s->compress, s->seal, &s->system_metrics, s->mmap, NULL, &s->system_journal); - - if (r >= 0) + if (r >= 0) { server_fix_perms(s, s->system_journal, 0); - else if (r < 0) { + (void) determine_space_for(s, &s->system_metrics, "/var/log/journal/", "System journal", true, true, NULL, NULL); + } else if (r < 0) { if (r != -ENOENT && r != -EROFS) log_warning_errno(r, "Failed to open system journal: %m"); @@ -967,7 +965,7 @@ static int system_journal_open(Server *s, bool flush_requested) { if (!s->runtime_journal && (s->storage != STORAGE_NONE)) { - fn = strjoina("/run/log/journal/", ids, "/system.journal", NULL); + fn = strjoina("/run/log/journal/", SERVER_MACHINE_ID(s), "/system.journal"); if (s->system_journal) { @@ -997,12 +995,12 @@ static int system_journal_open(Server *s, bool flush_requested) { return log_error_errno(r, "Failed to open runtime journal: %m"); } - if (s->runtime_journal) + if (s->runtime_journal) { server_fix_perms(s, s->runtime_journal, 0); + (void) determine_space_for(s, &s->runtime_metrics, "/run/log/journal/", "Runtime journal", true, true, NULL, NULL); + } } - available_space(s, true); - return r; } @@ -1023,7 +1021,7 @@ int server_flush_to_var(Server *s) { if (!s->runtime_journal) return 0; - system_journal_open(s, true); + (void) system_journal_open(s, true); if (!s->system_journal) return 0; @@ -1067,7 +1065,7 @@ int server_flush_to_var(Server *s) { } server_rotate(s); - server_vacuum(s); + server_vacuum(s, false, false); if (!s->system_journal) { log_notice("Didn't flush runtime journal since rotation of system journal wasn't successful."); @@ -1231,7 +1229,7 @@ static int dispatch_sigusr1(sd_event_source *es, const struct signalfd_siginfo * server_flush_to_var(s); server_sync(s); - server_vacuum(s); + server_vacuum(s, false, false); touch("/run/systemd/journal/flushed"); @@ -1245,7 +1243,7 @@ static int dispatch_sigusr2(sd_event_source *es, const struct signalfd_siginfo * log_info("Received request to rotate journal from PID %"PRIu32, si->ssi_pid); server_rotate(s); - server_vacuum(s); + server_vacuum(s, true, true); return 0; } @@ -1472,18 +1470,19 @@ int server_init(Server *s) { s->max_level_console = LOG_INFO; s->max_level_wall = LOG_EMERG; - memset(&s->system_metrics, 0xFF, sizeof(s->system_metrics)); - memset(&s->runtime_metrics, 0xFF, sizeof(s->runtime_metrics)); + journal_reset_metrics(&s->system_metrics); + journal_reset_metrics(&s->runtime_metrics); server_parse_config_file(s); server_parse_proc_cmdline(s); + if (!!s->rate_limit_interval ^ !!s->rate_limit_burst) { log_debug("Setting both rate limit interval and burst from "USEC_FMT",%u to 0,0", s->rate_limit_interval, s->rate_limit_burst); s->rate_limit_interval = s->rate_limit_burst = 0; } - mkdir_p("/run/systemd/journal", 0755); + (void) mkdir_p("/run/systemd/journal", 0755); s->user_journals = ordered_hashmap_new(NULL); if (!s->user_journals) @@ -1682,3 +1681,22 @@ void server_done(Server *s) { udev_unref(s->udev); } + +static const char* const storage_table[_STORAGE_MAX] = { + [STORAGE_AUTO] = "auto", + [STORAGE_VOLATILE] = "volatile", + [STORAGE_PERSISTENT] = "persistent", + [STORAGE_NONE] = "none" +}; + +DEFINE_STRING_TABLE_LOOKUP(storage, Storage); +DEFINE_CONFIG_PARSE_ENUM(config_parse_storage, storage, Storage, "Failed to parse storage setting"); + +static const char* const split_mode_table[_SPLIT_MAX] = { + [SPLIT_LOGIN] = "login", + [SPLIT_UID] = "uid", + [SPLIT_NONE] = "none", +}; + +DEFINE_STRING_TABLE_LOOKUP(split_mode, SplitMode); +DEFINE_CONFIG_PARSE_ENUM(config_parse_split_mode, split_mode, SplitMode, "Failed to parse split mode setting"); diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h index d954c51..535c0ab 100644 --- a/src/journal/journald-server.h +++ b/src/journal/journald-server.h @@ -100,8 +100,9 @@ typedef struct Server { unsigned n_forward_syslog_missed; usec_t last_warn_forward_syslog_missed; - uint64_t cached_available_space; - usec_t cached_available_space_timestamp; + uint64_t cached_space_available; + uint64_t cached_space_limit; + usec_t cached_space_timestamp; uint64_t var_available_timestamp; @@ -141,6 +142,8 @@ typedef struct Server { char *cgroup_root; } Server; +#define SERVER_MACHINE_ID(s) ((s)->machine_id_field + strlen("_MACHINE_ID=")) + #define N_IOVEC_META_FIELDS 20 #define N_IOVEC_KERNEL_FIELDS 64 #define N_IOVEC_UDEV_FIELDS 32 @@ -166,7 +169,7 @@ void server_fix_perms(Server *s, JournalFile *f, uid_t uid); int server_init(Server *s); void server_done(Server *s); void server_sync(Server *s); -void server_vacuum(Server *s); +int server_vacuum(Server *s, bool verbose, bool patch_min_use); void server_rotate(Server *s); int server_schedule_sync(Server *s, int priority); int server_flush_to_var(Server *s); diff --git a/src/journal/journald.c b/src/journal/journald.c index b2624c6..83236ce 100644 --- a/src/journal/journald.c +++ b/src/journal/journald.c @@ -21,8 +21,8 @@ #include -#include "systemd/sd-messages.h" -#include "systemd/sd-daemon.h" +#include "sd-messages.h" +#include "sd-daemon.h" #include "journal-authenticate.h" #include "journald-server.h" @@ -54,7 +54,7 @@ int main(int argc, char *argv[]) { if (r < 0) goto finish; - server_vacuum(&server); + server_vacuum(&server, false, false); server_flush_to_var(&server); server_flush_dev_kmsg(&server); @@ -82,7 +82,7 @@ int main(int argc, char *argv[]) { if (server.oldest_file_usec + server.max_retention_usec < n) { log_info("Retention time reached."); server_rotate(&server); - server_vacuum(&server); + server_vacuum(&server, false, false); continue; } diff --git a/src/journal/journald.conf b/src/journal/journald.conf index 47eefe9..7beb96c 100644 --- a/src/journal/journald.conf +++ b/src/journal/journald.conf @@ -22,9 +22,11 @@ #SystemMaxUse= #SystemKeepFree= #SystemMaxFileSize= +#SystemMaxFiles=100 #RuntimeMaxUse= #RuntimeKeepFree= #RuntimeMaxFileSize= +#RuntimeMaxFiles=100 #MaxRetentionSec= #MaxFileSec=1month #ForwardToSyslog=no diff --git a/src/journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c index adefa1b..8069339 100644 --- a/src/journal/test-journal-interleaving.c +++ b/src/journal/test-journal-interleaving.c @@ -197,7 +197,7 @@ static void test_skip(void (*setup)(void)) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } @@ -282,7 +282,7 @@ static void test_sequence_numbers(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } diff --git a/src/journal/test-journal.c b/src/journal/test-journal.c index caaab25..01d4bc9 100644 --- a/src/journal/test-journal.c +++ b/src/journal/test-journal.c @@ -116,7 +116,7 @@ static void test_non_empty(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } @@ -155,7 +155,7 @@ static void test_empty(void) { if (arg_keep) log_info("Not removing %s", t); else { - journal_directory_vacuum(".", 3000000, 0, NULL, true); + journal_directory_vacuum(".", 3000000, 0, 0, NULL, true); assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } -- 2.7.4