#include "journal-remote.h"
static int do_rotate(JournalFile **f, bool compress, bool seal) {
- int r = journal_file_rotate(f, compress, seal, NULL);
+ int r = journal_file_rotate(f, compress, (uint64_t) -1, seal, NULL);
if (r < 0) {
if (*f)
log_error_errno(r, "Failed to rotate %s: %m", (*f)->path);
r = journal_file_open_reliably(output,
O_RDWR|O_CREAT, 0640,
- arg_compress, arg_seal,
+ arg_compress, (uint64_t) -1, arg_seal,
&w->metrics,
w->mmap, NULL,
NULL, &w->journal);
}
r = sd_event_add_time(s->events, &d->timer_event,
- CLOCK_MONOTONIC, UINT64_MAX, 0,
+ CLOCK_MONOTONIC, (uint64_t) -1, 0,
null_timer_event_handler, d);
if (r < 0) {
log_error_errno(r, "Failed to add timer_event: %m");
#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem))
#define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem))
-#define COMPRESSION_SIZE_THRESHOLD (512ULL)
+#define DEFAULT_COMPRESS_THRESHOLD (512ULL)
+#define MIN_COMPRESS_THRESHOLD (8ULL)
/* This is the minimum journal file size */
#define JOURNAL_FILE_SIZE_MIN (512ULL*1024ULL) /* 512 KiB */
o->data.hash = htole64(hash);
#if HAVE_XZ || HAVE_LZ4
- if (JOURNAL_FILE_COMPRESS(f) && size >= COMPRESSION_SIZE_THRESHOLD) {
+ if (JOURNAL_FILE_COMPRESS(f) && size >= f->compress_threshold_bytes) {
size_t rsize = 0;
compression = compress_blob(data, size, o->data.payload, size - 1, &rsize);
int flags,
mode_t mode,
bool compress,
+ uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
JournalFile *f;
void *h;
int r;
+ char bytes[FORMAT_BYTES_MAX];
assert(ret);
assert(fd >= 0 || fname);
#elif HAVE_XZ
f->compress_xz = compress;
#endif
+
+ if (compress_threshold_bytes == (uint64_t) -1)
+ f->compress_threshold_bytes = DEFAULT_COMPRESS_THRESHOLD;
+ else
+ f->compress_threshold_bytes = MAX(MIN_COMPRESS_THRESHOLD, compress_threshold_bytes);
+
#if HAVE_GCRYPT
f->seal = seal;
#endif
+ log_debug("Journal effective settings seal=%s compress=%s compress_threshold_bytes=%s",
+ yes_no(f->seal), yes_no(JOURNAL_FILE_COMPRESS(f)),
+ format_bytes(bytes, sizeof(bytes), f->compress_threshold_bytes));
+
if (mmap_cache)
f->mmap = mmap_cache_ref(mmap_cache);
else {
return r;
}
-int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes) {
+int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes) {
_cleanup_free_ char *p = NULL;
size_t l;
JournalFile *old_file, *new_file = NULL;
* we archive them */
old_file->defrag_on_close = true;
- r = journal_file_open(-1, old_file->path, old_file->flags, old_file->mode, compress, seal, NULL, old_file->mmap, deferred_closes, old_file, &new_file);
+ r = journal_file_open(-1, old_file->path, old_file->flags, old_file->mode, compress,
+ compress_threshold_bytes, seal, NULL, old_file->mmap, deferred_closes,
+ old_file, &new_file);
if (deferred_closes &&
set_put(deferred_closes, old_file) >= 0)
int flags,
mode_t mode,
bool compress,
+ uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
size_t l;
_cleanup_free_ char *p = NULL;
- r = journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
+ r = journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache,
+ deferred_closes, template, ret);
if (!IN_SET(r,
-EBADMSG, /* Corrupted */
-ENODATA, /* Truncated */
log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname);
- return journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
+ return journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache,
+ deferred_closes, template, ret);
}
int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) {
unsigned last_seen_generation;
+ uint64_t compress_threshold_bytes;
#if HAVE_XZ || HAVE_LZ4
void *compress_buffer;
size_t compress_buffer_size;
int flags,
mode_t mode,
bool compress,
+ uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
int flags,
mode_t mode,
bool compress,
+ uint64_t compress_threshold_bytes,
bool seal,
JournalMetrics *metrics,
MMapCache *mmap_cache,
void journal_file_dump(JournalFile *f);
void journal_file_print_header(JournalFile *f);
-int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes);
+int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes);
void journal_file_post_change(JournalFile *f);
int journal_file_enable_post_change_timer(JournalFile *f, sd_event *e, usec_t t);
assert(ret);
if (reliably)
- r = journal_file_open_reliably(fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
+ r = journal_file_open_reliably(fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
else
- r = journal_file_open(-1, fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
+ r = journal_file_open(-1, fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
if (r < 0)
return r;
if (!*f)
return -EINVAL;
- r = journal_file_rotate(f, s->compress, seal, s->deferred_closes);
+ r = journal_file_rotate(f, s->compress, (uint64_t) -1, seal, s->deferred_closes);
if (r < 0) {
if (*f)
return log_error_errno(r, "Failed to rotate %s: %m", (*f)->path);
goto finish;
}
- r = journal_file_open(fd, path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, NULL, &f);
+ r = journal_file_open(fd, path, O_RDONLY, 0, false, 0, false, NULL, j->mmap, NULL, NULL, &f);
if (r < 0) {
log_debug_errno(r, "Failed to open journal file %s: %m", path);
goto finish;
assert_se(mkdtemp(dn));
fn = strappend(dn, "/test.journal");
- r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, false, NULL, NULL, NULL, NULL, &new_journal);
+ r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, 0, false, NULL, NULL, NULL, NULL, &new_journal);
assert_se(r >= 0);
r = sd_journal_open(&j, 0);
static JournalFile *test_open(const char *name) {
JournalFile *f;
- assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, false, NULL, NULL, NULL, NULL, &f));
+ assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f));
return f;
}
assert_se(chdir(t) >= 0);
assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0644,
- true, false, NULL, NULL, NULL, NULL, &one) == 0);
+ true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0);
append_number(one, 1, &seqnum);
printf("seqnum=%"PRIu64"\n", seqnum);
memcpy(&seqnum_id, &one->header->seqnum_id, sizeof(sd_id128_t));
assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0644,
- true, false, NULL, NULL, NULL, one, &two) == 0);
+ true, (uint64_t) -1, false, NULL, NULL, NULL, one, &two) == 0);
assert_se(two->header->state == STATE_ONLINE);
assert_se(!sd_id128_equal(two->header->file_id, one->header->file_id));
seqnum = 0;
assert_se(journal_file_open(-1, "two.journal", O_RDWR, 0,
- true, false, NULL, NULL, NULL, NULL, &two) == 0);
+ true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0);
assert_se(sd_id128_equal(two->header->seqnum_id, seqnum_id));
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
- assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &one) == 0);
- assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &two) == 0);
- assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &three) == 0);
+ assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0);
+ assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0);
+ assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &three) == 0);
for (i = 0; i < N_ENTRIES; i++) {
char *p, *q;
JournalFile *f;
int r;
- r = journal_file_open(-1, fn, O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f);
+ r = journal_file_open(-1, fn, O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f);
if (r < 0)
return r;
log_info("Generating...");
- assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
+ assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
for (n = 0; n < N_ENTRIES; n++) {
struct iovec iovec;
log_info("Verifying...");
- assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
+ assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
/* journal_file_print_header(f); */
journal_file_dump(f);
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
- assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f) == 0);
+ assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f) == 0);
dual_timestamp_get(&ts);
assert_se(journal_file_move_to_entry_by_seqnum(f, 10, DIRECTION_DOWN, &o, NULL) == 0);
- journal_file_rotate(&f, true, true, NULL);
- journal_file_rotate(&f, true, true, NULL);
+ journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
+ journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
(void) journal_file_close(f);
assert_se(mkdtemp(t));
assert_se(chdir(t) >= 0);
- assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, false, NULL, NULL, NULL, NULL, &f1) == 0);
+ assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f1) == 0);
- assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &f2) == 0);
+ assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f2) == 0);
- assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, true, NULL, NULL, NULL, NULL, &f3) == 0);
+ assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f3) == 0);
- assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f4) == 0);
+ assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f4) == 0);
journal_file_print_header(f1);
puts("");
(void) journal_file_close(f4);
}
+#if HAVE_XZ || HAVE_LZ4
+static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) {
+ dual_timestamp ts;
+ JournalFile *f;
+ struct iovec iovec;
+ Object *o;
+ uint64_t p;
+ char t[] = "/tmp/journal-XXXXXX";
+ char data[2048] = {0};
+ bool is_compressed;
+ int r;
+
+ assert_se(data_size <= sizeof(data));
+
+ log_set_max_level(LOG_DEBUG);
+
+ assert_se(mkdtemp(t));
+ assert_se(chdir(t) >= 0);
+
+ assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0);
+
+ dual_timestamp_get(&ts);
+
+ iovec.iov_base = (void*) data;
+ iovec.iov_len = data_size;
+ assert_se(journal_file_append_entry(f, &ts, &iovec, 1, NULL, NULL, NULL) == 0);
+
+#if HAVE_GCRYPT
+ journal_file_append_tag(f);
+#endif
+ journal_file_dump(f);
+
+ /* We have to partially reimplement some of the dump logic, because the normal next_entry does the
+ * decompression for us. */
+ p = le64toh(f->header->header_size);
+ while (true) {
+ r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o);
+ assert_se(r == 0);
+ if (o->object.type == OBJECT_DATA)
+ break;
+
+ assert_se(p < le64toh(f->header->tail_object_offset));
+ p = p + ALIGN64(le64toh(o->object.size));
+ }
+
+ is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0;
+
+ (void) journal_file_close(f);
+
+ log_info("Done...");
+
+ if (arg_keep)
+ log_info("Not removing %s", t);
+ else {
+ journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
+
+ assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+ }
+
+ puts("------------------------------------------------------------");
+
+ return is_compressed;
+}
+
+static void test_min_compress_size(void) {
+ /* Note that XZ will actually fail to compress anything under 80 bytes, so you have to choose the limits
+ * carefully */
+
+ /* DEFAULT_MIN_COMPRESS_SIZE is 512 */
+ assert_se(!check_compressed((uint64_t) -1, 255));
+ assert_se(check_compressed((uint64_t) -1, 513));
+
+ /* compress everything */
+ assert_se(check_compressed(0, 96));
+ assert_se(check_compressed(8, 96));
+
+ /* Ensure we don't try to compress less than 8 bytes */
+ assert_se(!check_compressed(0, 7));
+
+ /* check boundary conditions */
+ assert_se(check_compressed(256, 256));
+ assert_se(!check_compressed(256, 255));
+}
+#endif
+
int main(int argc, char *argv[]) {
arg_keep = argc > 1;
test_non_empty();
test_empty();
+#if HAVE_XZ || HAVE_LZ4
+ test_min_compress_size();
+#endif
return 0;
}