1 // Copyright (C) 2002-2004 Andrew Tridgell
2 // Copyright (C) 2009-2018 Joel Rosdahl
4 // This program is free software; you can redistribute it and/or modify it
5 // under the terms of the GNU General Public License as published by the Free
6 // Software Foundation; either version 3 of the License, or (at your option)
9 // This program is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 // You should have received a copy of the GNU General Public License along with
15 // this program; if not, write to the Free Software Foundation, Inc., 51
16 // Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 // Routines to handle the stats files. The stats file is stored one per cache
19 // subdirectory to make this more scalable.
24 #include <sys/types.h>
32 extern char *stats_file;
33 extern struct conf *conf;
34 extern unsigned lock_staleness_limit;
35 extern char *primary_config_path;
36 extern char *secondary_config_path;
38 static struct counters *counter_updates;
40 #define FLAG_NOZERO 1 // don't zero with the -z option
41 #define FLAG_ALWAYS 2 // always show, even if zero
42 #define FLAG_NEVER 4 // never show
44 static void display_size_times_1024(uint64_t size);
46 // Statistics fields in display order.
61 "cache hit (preprocessed)",
79 "called for preprocessing",
85 "multiple source files",
91 "compiler produced stdout",
97 "compiler produced no output",
103 "compiler produced empty output",
115 "ccache internal error",
121 "preprocessor error",
127 "can't use precompiled header",
133 "couldn't find the compiler",
139 "cache file missing",
145 "bad compiler arguments",
151 "unsupported source language",
157 "compiler check failed",
163 "autoconf compile/link",
168 STATS_UNSUPPORTED_OPTION,
169 "unsupported compiler option",
174 STATS_UNSUPPORTED_DIRECTIVE,
175 "unsupported code directive",
187 "output to a non-regular file",
199 "error hashing extra file",
205 "cleanups performed",
213 FLAG_NOZERO|FLAG_ALWAYS
218 display_size_times_1024,
219 FLAG_NOZERO|FLAG_ALWAYS
222 STATS_OBSOLETE_MAXFILES,
225 FLAG_NOZERO|FLAG_NEVER
228 STATS_OBSOLETE_MAXSIZE,
231 FLAG_NOZERO|FLAG_NEVER
235 "stats last zeroed at",
248 display_size(uint64_t size)
250 char *s = format_human_readable_size(size);
256 display_size_times_1024(uint64_t size)
258 display_size(size * 1024);
261 // Parse a stats file from a buffer, adding to the counters.
263 parse_stats(struct counters *counters, const char *buf)
269 long val = strtol(p, &p2, 10);
273 if (counters->size < i + 1) {
274 counters_resize(counters, i + 1);
276 counters->data[i] += val;
282 // Write out a stats file.
284 stats_write(const char *path, struct counters *counters)
287 if (stat(path, &st) != 0 && errno == ENOENT) {
288 // New stats, update zero timestamp.
291 stats_timestamp(now, counters);
293 char *tmp_file = format("%s.tmp", path);
294 FILE *f = create_tmp_file(&tmp_file, "wb");
295 for (size_t i = 0; i < counters->size; i++) {
296 if (fprintf(f, "%u\n", counters->data[i]) < 0) {
297 fatal("Failed to write to %s", tmp_file);
301 x_rename(tmp_file, path);
306 init_counter_updates(void)
308 if (!counter_updates) {
309 counter_updates = counters_init(STATS_END);
313 // Record that a number of bytes and files have been added to the cache. Size
316 stats_update_size(int64_t size, int files)
318 init_counter_updates();
319 counter_updates->data[STATS_NUMFILES] += files;
320 counter_updates->data[STATS_TOTALSIZE] += size / 1024;
323 // Read in the stats from one directory and add to the counters.
325 stats_read(const char *sfile, struct counters *counters)
327 char *data = read_text_file(sfile, 1024);
329 parse_stats(counters, data);
334 // Set the timestamp when the counters were last zeroed out.
336 stats_timestamp(time_t time, struct counters *counters)
338 counters->data[STATS_ZEROTIMESTAMP] = (unsigned) time;
341 // Write counter updates in counter_updates to disk.
351 if (!counter_updates) {
355 bool should_flush = false;
356 for (int i = 0; i < STATS_END; ++i) {
357 if (counter_updates->data[i] > 0) {
369 // A NULL stats_file means that we didn't get past calculate_object_hash(),
370 // so we just choose one of stats files in the 16 subdirectories.
371 stats_dir = format("%s/%x", conf->cache_dir, hash_from_int(getpid()) % 16);
372 stats_file = format("%s/stats", stats_dir);
376 if (!lockfile_acquire(stats_file, lock_staleness_limit)) {
380 struct counters *counters = counters_init(STATS_END);
381 stats_read(stats_file, counters);
382 for (int i = 0; i < STATS_END; ++i) {
383 counters->data[i] += counter_updates->data[i];
385 stats_write(stats_file, counters);
386 lockfile_release(stats_file);
388 if (!str_eq(conf->log_file, "")) {
389 for (int i = 0; i < STATS_END; ++i) {
390 if (counter_updates->data[stats_info[i].stat] != 0
391 && !(stats_info[i].flags & FLAG_NOZERO)) {
392 cc_log("Result: %s", stats_info[i].message);
397 char *subdir = dirname(stats_file);
398 bool need_cleanup = false;
400 if (conf->max_files != 0
401 && counters->data[STATS_NUMFILES] > conf->max_files / 16) {
402 cc_log("Need to clean up %s since it holds %u files (limit: %u files)",
404 counters->data[STATS_NUMFILES],
405 conf->max_files / 16);
408 if (conf->max_size != 0
409 && counters->data[STATS_TOTALSIZE] > conf->max_size / 1024 / 16) {
410 cc_log("Need to clean up %s since it holds %u KiB (limit: %lu KiB)",
412 counters->data[STATS_TOTALSIZE],
413 (unsigned long)conf->max_size / 1024 / 16);
418 clean_up_dir(conf, subdir, conf->limit_multiple);
422 counters_free(counters);
425 // Update a normal stat.
427 stats_update(enum stats stat)
429 assert(stat > STATS_NONE && stat < STATS_END);
430 init_counter_updates();
431 counter_updates->data[stat]++;
434 // Get the pending update of a counter value.
436 stats_get_pending(enum stats stat)
438 init_counter_updates();
439 return counter_updates->data[stat];
442 // Sum and display the total stats for all cache dirs.
444 stats_summary(struct conf *conf)
446 struct counters *counters = counters_init(STATS_END);
451 // Add up the stats in each directory.
452 for (int dir = -1; dir <= 0xF; dir++) {
456 fname = format("%s/stats", conf->cache_dir);
458 fname = format("%s/%1x/stats", conf->cache_dir, dir);
461 counters->data[STATS_ZEROTIMESTAMP] = 0; // Don't add
462 stats_read(fname, counters);
463 time_t current = (time_t) counters->data[STATS_ZEROTIMESTAMP];
464 if (current != 0 && (oldest == 0 || current < oldest)) {
470 printf("cache directory %s\n", conf->cache_dir);
471 printf("primary config %s\n",
472 primary_config_path ? primary_config_path : "");
473 printf("secondary config (readonly) %s\n",
474 secondary_config_path ? secondary_config_path : "");
476 struct tm *tm = localtime(&oldest);
478 strftime(timestamp, sizeof(timestamp), "%c", tm);
479 printf("stats zero time %s\n", timestamp);
482 // ...and display them.
483 for (int i = 0; stats_info[i].message; i++) {
484 enum stats stat = stats_info[i].stat;
486 if (stats_info[i].flags & FLAG_NEVER) {
489 if (counters->data[stat] == 0 && !(stats_info[i].flags & FLAG_ALWAYS)) {
493 printf("%-31s ", stats_info[i].message);
494 if (stats_info[i].fn) {
495 stats_info[i].fn(counters->data[stat]);
498 printf("%8u\n", counters->data[stat]);
501 if (stat == STATS_TOCACHE) {
502 unsigned direct = counters->data[STATS_CACHEHIT_DIR];
503 unsigned preprocessed = counters->data[STATS_CACHEHIT_CPP];
504 unsigned hit = direct + preprocessed;
505 unsigned miss = counters->data[STATS_TOCACHE];
506 unsigned total = hit + miss;
507 double percent = total > 0 ? (100.0f * hit) / total : 0.0f;
508 printf("cache hit rate %6.2f %%\n", percent);
512 if (conf->max_files != 0) {
513 printf("max files %8u\n", conf->max_files);
515 if (conf->max_size != 0) {
516 printf("max cache size ");
517 display_size(conf->max_size);
521 counters_free(counters);
524 // Zero all the stats structures.
530 char *fname = format("%s/stats", conf->cache_dir);
534 for (int dir = 0; dir <= 0xF; dir++) {
535 struct counters *counters = counters_init(STATS_END);
537 fname = format("%s/%1x/stats", conf->cache_dir, dir);
538 if (stat(fname, &st) != 0) {
539 // No point in trying to reset the stats file if it doesn't exist.
543 if (lockfile_acquire(fname, lock_staleness_limit)) {
544 stats_read(fname, counters);
545 for (unsigned i = 0; stats_info[i].message; i++) {
546 if (!(stats_info[i].flags & FLAG_NOZERO)) {
547 counters->data[stats_info[i].stat] = 0;
550 stats_timestamp(time(NULL), counters);
551 stats_write(fname, counters);
552 lockfile_release(fname);
554 counters_free(counters);
559 // Get the per-directory limits.
561 stats_get_obsolete_limits(const char *dir, unsigned *maxfiles,
564 struct counters *counters = counters_init(STATS_END);
565 char *sname = format("%s/stats", dir);
566 stats_read(sname, counters);
567 *maxfiles = counters->data[STATS_OBSOLETE_MAXFILES];
568 *maxsize = (uint64_t)counters->data[STATS_OBSOLETE_MAXSIZE] * 1024;
570 counters_free(counters);
573 // Set the per-directory sizes.
575 stats_set_sizes(const char *dir, unsigned num_files, uint64_t total_size)
577 struct counters *counters = counters_init(STATS_END);
578 char *statsfile = format("%s/stats", dir);
579 if (lockfile_acquire(statsfile, lock_staleness_limit)) {
580 stats_read(statsfile, counters);
581 counters->data[STATS_NUMFILES] = num_files;
582 counters->data[STATS_TOTALSIZE] = total_size / 1024;
583 stats_write(statsfile, counters);
584 lockfile_release(statsfile);
587 counters_free(counters);
590 // Count directory cleanup run.
592 stats_add_cleanup(const char *dir, unsigned count)
594 struct counters *counters = counters_init(STATS_END);
595 char *statsfile = format("%s/stats", dir);
596 if (lockfile_acquire(statsfile, lock_staleness_limit)) {
597 stats_read(statsfile, counters);
598 counters->data[STATS_NUMCLEANUPS] += count;
599 stats_write(statsfile, counters);
600 lockfile_release(statsfile);
603 counters_free(counters);