Detailed benchmark outputs for functions

This patch adds an option to get detailed benchmark output for
functions.  Invoking the benchmark with 'make DETAILED=1 bench' causes
each benchmark program to store a mean execution time for each input
it works on.  This is useful to give a more comprehensive picture of
performance of functions compared to just the single mean figure.

diff --git a/ChangeLog b/ChangeLog
index 072747b..081bf0b 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,15 @@
 2014-03-29  Siddhesh Poyarekar  <siddhesh@redhat.com>
 
+       * benchtests/Makefile (DETAILED_OPT): New make option.
+       (bench-func): Run benchmark program with -d if DETAILED_OPT is
+       set.
+       * benchtests/bench-skeleton.c: Include stdbool.h.
+       (main): Store and print timings per input.
+       * benchtests/scripts/bench.py (STRUCT_TEMPLATE): Add timing
+       member to each argument value.
+       (EPILOGUE): Define new macros RESULT and RESULT_ACCUM.
+       (_print_arg_data): Initialize per-input timing to 0.
+
        * benchtests/Makefile (timing-type): New binary.
        (bench-clean): Also remove bench-timing-type.
        (bench): New target for timing-type.

diff --git a/benchtests/Makefile b/benchtests/Makefile
index be11708..f5488c1 100644 (file)
--- a/benchtests/Makefile
+++ b/benchtests/Makefile
@@ -86,6 +86,12 @@ ifdef USE_CLOCK_GETTIME
 CPPFLAGS-nonlib += -DUSE_CLOCK_GETTIME
 endif
 
+DETAILED_OPT :=
+
+ifdef DETAILED
+DETAILED_OPT := -d
+endif
+
 # This makes sure CPPFLAGS-nonlib and CFLAGS-nonlib are passed
 # for all these modules.
 cpp-srcs-left := $(binaries-benchset:=.c) $(binaries-bench:=.c)
@@ -126,7 +132,7 @@ bench-func: $(binaries-bench)
            echo ","; \
          fi; \
          echo "Running $${run}" >&2; \
-         $(run-bench); \
+         $(run-bench) $(DETAILED_OPT); \
        done; \
        echo "  }"; \
        echo "}"; } > $(objpfx)bench.out-tmp; \

diff --git a/benchtests/bench-skeleton.c b/benchtests/bench-skeleton.c
index faef7eb..0c7d744 100644 (file)
--- a/benchtests/bench-skeleton.c
+++ b/benchtests/bench-skeleton.c
@@ -18,6 +18,7 @@
 
 #include <string.h>
 #include <stdint.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <time.h>
 #include <inttypes.h>
@@ -48,6 +49,10 @@ main (int argc, char **argv)
   unsigned long i, k;
   struct timespec runtime;
   timing_t start, end;
+  bool detailed = false;
+
+  if (argc == 2 && !strcmp (argv[1], "-d"))
+    detailed = true;
 
   startup();
 
@@ -72,6 +77,7 @@ main (int argc, char **argv)
 
       double d_total_i = 0;
       timing_t total = 0, max = 0, min = 0x7fffffffffffffff;
+      int64_t c = 0;
       while (1)
        {
          for (i = 0; i < NUM_SAMPLES (v); i++)
@@ -91,8 +97,13 @@ main (int argc, char **argv)
                min = cur;
 
              TIMING_ACCUM (total, cur);
+             /* Accumulate timings for the value.  In the end we will divide
+                by the total iterations.  */
+             RESULT_ACCUM (cur, v, i, c * iters, (c + 1) * iters);
+
              d_total_i += iters;
            }
+         c++;
          struct timespec curtime;
 
          memset (&curtime, 0, sizeof (curtime));
@@ -114,6 +125,17 @@ main (int argc, char **argv)
              d_total_s, d_total_i, max / d_iters, min / d_iters,
              d_total_s / d_total_i);
 
+      if (detailed)
+       {
+         printf (",\n\"timings\": [");
+         for (int i = 0; i < NUM_SAMPLES (v); i++)
+           {
+             if (i > 0)
+               putc (',', stdout);
+             printf ("%g", RESULT (v, i));
+           }
+         puts ("]");
+       }
       puts ("}");
     }
 

diff --git a/benchtests/scripts/bench.py b/benchtests/scripts/bench.py
index 90317b5..492c764 100755 (executable)
--- a/benchtests/scripts/bench.py
+++ b/benchtests/scripts/bench.py
@@ -50,6 +50,7 @@ STRUCT_TEMPLATE = '''
 struct args
 {
 %(args)s
+  double timing;
 };
 
 struct _variants
@@ -80,6 +81,9 @@ struct _variants variants[%(num_variants)d] = {
 
 # Epilogue for the generated source file.
 EPILOGUE = '''
+#define RESULT(__v, __i) (variants[(__v)].in[(__i)].timing)
+#define RESULT_ACCUM(r, v, i, old, new) \\
+        ((RESULT ((v), (i))) = (RESULT ((v), (i)) * (old) + (r)) / ((new) + 1))
 #define BENCH_FUNC(i, j) ({%(getret)s CALL_BENCH_FUNC (i, j);})
 #define FUNCNAME "%(func)s"
 #include "bench-skeleton.c"'''
@@ -168,7 +172,7 @@ def _print_arg_data(func, directives, all_vals):
     # Now print the values.
     variants = []
     for (k, vals), i in zip(all_vals.items(), itertools.count()):
-        out = ['  {%s},' % v for v in vals]
+        out = ['  {%s, 0},' % v for v in vals]
 
         # Members for the variants structure list that we will
         # print later.