rtla/osnoise: Add osnoise top mode
authorDaniel Bristot de Oliveira <bristot@kernel.org>
Fri, 10 Dec 2021 18:11:23 +0000 (19:11 +0100)
committerSteven Rostedt <rostedt@goodmis.org>
Thu, 13 Jan 2022 22:02:42 +0000 (17:02 -0500)
The rtla osnoise tool is an interface for the osnoise tracer. The
osnoise tracer dispatches a kernel thread per-cpu. These threads read
the time in a loop while with preemption, softirqs and IRQs enabled,
thus allowing all the sources of osnoise during its execution. The
osnoise threads take note of the entry and exit point of any source
of interferences, increasing a per-cpu interference counter. The
osnoise tracer also saves an interference counter for each source
of interference.

The rtla osnoise top mode displays information about the periodic
summary from the osnoise tracer.

One example of rtla osnoise top output is:

[root@alien ~]# rtla osnoise top -c 0-3 -d 1m -q -r 900000 -P F:1
                                         Operating System Noise
duration:   0 00:01:00 | time is in us
CPU Period       Runtime        Noise  % CPU Aval   Max Noise   Max Single          HW          NMI          IRQ      Softirq       Thread
  0 #58         52200000         1031    99.99802          91           60           0            0        52285            0          101
  1 #59         53100000            5    99.99999           5            5           0            9        53122            0           18
  2 #59         53100000            7    99.99998           7            7           0            8        53115            0           18
  3 #59         53100000         8274    99.98441         277           23           0            9        53778            0          660

"rtla osnoise top --help" works and provide information about the
available options.

Link: https://lkml.kernel.org/r/0d796993abf587ae5a170bb8415c49368d4999e1.1639158831.git.bristot@kernel.org
Cc: Tao Zhou <tao.zhou@linux.dev>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Tom Zanussi <zanussi@kernel.org>
Cc: Masami Hiramatsu <mhiramat@kernel.org>
Cc: Juri Lelli <juri.lelli@redhat.com>
Cc: Clark Williams <williams@redhat.com>
Cc: John Kacur <jkacur@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Daniel Bristot de Oliveira <bristot@kernel.org>
Cc: linux-rt-users@vger.kernel.org
Cc: linux-trace-devel@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Daniel Bristot de Oliveira <bristot@kernel.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
tools/tracing/rtla/src/osnoise.c
tools/tracing/rtla/src/osnoise.h
tools/tracing/rtla/src/osnoise_top.c [new file with mode: 0644]

index e525404..fbf4f9b 100644 (file)
@@ -844,9 +844,25 @@ int osnoise_main(int argc, char *argv[])
        if (argc == 0)
                goto usage;
 
+       /*
+        * if osnoise was called without any argument, run the
+        * default cmdline.
+        */
+       if (argc == 1) {
+               osnoise_top_main(argc, argv);
+               exit(0);
+       }
+
        if ((strcmp(argv[1], "-h") == 0) || (strcmp(argv[1], "--help") == 0)) {
                osnoise_usage();
                exit(0);
+       } else if (strncmp(argv[1], "-", 1) == 0) {
+               /* the user skipped the tool, call the default one */
+               osnoise_top_main(argc, argv);
+               exit(0);
+       } else if (strcmp(argv[1], "top") == 0) {
+               osnoise_top_main(argc-1, &argv[1]);
+               exit(0);
        }
 
 usage:
index 1184fc6..a4b114e 100644 (file)
@@ -86,4 +86,5 @@ void osnoise_destroy_tool(struct osnoise_tool *top);
 struct osnoise_tool *osnoise_init_tool(char *tool_name);
 struct osnoise_tool *osnoise_init_trace_tool(char *tracer);
 
+int osnoise_top_main(int argc, char **argv);
 int osnoise_main(int argc, char **argv);
diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c
new file mode 100644 (file)
index 0000000..332b2ac
--- /dev/null
@@ -0,0 +1,579 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2021 Red Hat Inc, Daniel Bristot de Oliveira <bristot@kernel.org>
+ */
+
+#include <getopt.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <time.h>
+
+#include "osnoise.h"
+#include "utils.h"
+
+/*
+ * osnoise top parameters
+ */
+struct osnoise_top_params {
+       char                    *cpus;
+       char                    *monitored_cpus;
+       char                    *trace_output;
+       unsigned long long      runtime;
+       unsigned long long      period;
+       long long               stop_us;
+       long long               stop_total_us;
+       int                     sleep_time;
+       int                     duration;
+       int                     quiet;
+       int                     set_sched;
+       struct sched_attr       sched_param;
+};
+
+struct osnoise_top_cpu {
+       unsigned long long      sum_runtime;
+       unsigned long long      sum_noise;
+       unsigned long long      max_noise;
+       unsigned long long      max_sample;
+
+       unsigned long long      hw_count;
+       unsigned long long      nmi_count;
+       unsigned long long      irq_count;
+       unsigned long long      softirq_count;
+       unsigned long long      thread_count;
+
+       int                     sum_cycles;
+};
+
+struct osnoise_top_data {
+       struct osnoise_top_cpu  *cpu_data;
+       int                     nr_cpus;
+};
+
+/*
+ * osnoise_free_top - free runtime data
+ */
+static void
+osnoise_free_top(struct osnoise_top_data *data)
+{
+       free(data->cpu_data);
+       free(data);
+}
+
+/*
+ * osnoise_alloc_histogram - alloc runtime data
+ */
+static struct osnoise_top_data *osnoise_alloc_top(int nr_cpus)
+{
+       struct osnoise_top_data *data;
+
+       data = calloc(1, sizeof(*data));
+       if (!data)
+               return NULL;
+
+       data->nr_cpus = nr_cpus;
+
+       /* one set of histograms per CPU */
+       data->cpu_data = calloc(1, sizeof(*data->cpu_data) * nr_cpus);
+       if (!data->cpu_data)
+               goto cleanup;
+
+       return data;
+
+cleanup:
+       osnoise_free_top(data);
+       return NULL;
+}
+
+/*
+ * osnoise_top_handler - this is the handler for osnoise tracer events
+ */
+static int
+osnoise_top_handler(struct trace_seq *s, struct tep_record *record,
+                   struct tep_event *event, void *context)
+{
+       struct trace_instance *trace = context;
+       struct osnoise_tool *tool;
+       unsigned long long val;
+       struct osnoise_top_cpu *cpu_data;
+       struct osnoise_top_data *data;
+       int cpu = record->cpu;
+
+       tool = container_of(trace, struct osnoise_tool, trace);
+
+       data = tool->data;
+       cpu_data = &data->cpu_data[cpu];
+
+       cpu_data->sum_cycles++;
+
+       tep_get_field_val(s, event, "runtime", record, &val, 1);
+       update_sum(&cpu_data->sum_runtime, &val);
+
+       tep_get_field_val(s, event, "noise", record, &val, 1);
+       update_max(&cpu_data->max_noise, &val);
+       update_sum(&cpu_data->sum_noise, &val);
+
+       tep_get_field_val(s, event, "max_sample", record, &val, 1);
+       update_max(&cpu_data->max_sample, &val);
+
+       tep_get_field_val(s, event, "hw_count", record, &val, 1);
+       update_sum(&cpu_data->hw_count, &val);
+
+       tep_get_field_val(s, event, "nmi_count", record, &val, 1);
+       update_sum(&cpu_data->nmi_count, &val);
+
+       tep_get_field_val(s, event, "irq_count", record, &val, 1);
+       update_sum(&cpu_data->irq_count, &val);
+
+       tep_get_field_val(s, event, "softirq_count", record, &val, 1);
+       update_sum(&cpu_data->softirq_count, &val);
+
+       tep_get_field_val(s, event, "thread_count", record, &val, 1);
+       update_sum(&cpu_data->thread_count, &val);
+
+       return 0;
+}
+
+/*
+ * osnoise_top_header - print the header of the tool output
+ */
+static void osnoise_top_header(struct osnoise_tool *top)
+{
+       struct trace_seq *s = top->trace.seq;
+       char duration[26];
+
+       get_duration(top->start_time, duration, sizeof(duration));
+
+       trace_seq_printf(s, "\033[2;37;40m");
+       trace_seq_printf(s, "                                          Operating System Noise");
+       trace_seq_printf(s, "                                     ");
+       trace_seq_printf(s, "                                     ");
+       trace_seq_printf(s, "\033[0;0;0m");
+       trace_seq_printf(s, "\n");
+
+       trace_seq_printf(s, "duration: %9s | time is in us\n", duration);
+
+       trace_seq_printf(s, "\033[2;30;47m");
+       trace_seq_printf(s, "CPU Period       Runtime ");
+       trace_seq_printf(s, "       Noise ");
+       trace_seq_printf(s, " %% CPU Aval ");
+       trace_seq_printf(s, "  Max Noise   Max Single ");
+       trace_seq_printf(s, "         HW          NMI          IRQ      Softirq       Thread");
+       trace_seq_printf(s, "\033[0;0;0m");
+       trace_seq_printf(s, "\n");
+}
+
+/*
+ * clear_terminal - clears the output terminal
+ */
+static void clear_terminal(struct trace_seq *seq)
+{
+       if (!config_debug)
+               trace_seq_printf(seq, "\033c");
+}
+
+/*
+ * osnoise_top_print - prints the output of a given CPU
+ */
+static void osnoise_top_print(struct osnoise_tool *tool, int cpu)
+{
+       struct trace_seq *s = tool->trace.seq;
+       struct osnoise_top_cpu *cpu_data;
+       struct osnoise_top_data *data;
+       int percentage;
+       int decimal;
+
+       data = tool->data;
+       cpu_data = &data->cpu_data[cpu];
+
+       if (!cpu_data->sum_runtime)
+               return;
+
+       percentage = ((cpu_data->sum_runtime - cpu_data->sum_noise) * 10000000)
+                       / cpu_data->sum_runtime;
+       decimal = percentage % 100000;
+       percentage = percentage / 100000;
+
+       trace_seq_printf(s, "%3d #%-6d %12llu ", cpu, cpu_data->sum_cycles, cpu_data->sum_runtime);
+       trace_seq_printf(s, "%12llu ", cpu_data->sum_noise);
+       trace_seq_printf(s, "  %3d.%05d", percentage, decimal);
+       trace_seq_printf(s, "%12llu %12llu", cpu_data->max_noise, cpu_data->max_sample);
+
+       trace_seq_printf(s, "%12llu ", cpu_data->hw_count);
+       trace_seq_printf(s, "%12llu ", cpu_data->nmi_count);
+       trace_seq_printf(s, "%12llu ", cpu_data->irq_count);
+       trace_seq_printf(s, "%12llu ", cpu_data->softirq_count);
+       trace_seq_printf(s, "%12llu\n", cpu_data->thread_count);
+}
+
+/*
+ * osnoise_print_stats - print data for all cpus
+ */
+static void
+osnoise_print_stats(struct osnoise_top_params *params, struct osnoise_tool *top)
+{
+       struct trace_instance *trace = &top->trace;
+       static int nr_cpus = -1;
+       int i;
+
+       if (nr_cpus == -1)
+               nr_cpus = sysconf(_SC_NPROCESSORS_CONF);
+
+       if (!params->quiet)
+               clear_terminal(trace->seq);
+
+       osnoise_top_header(top);
+
+       for (i = 0; i < nr_cpus; i++) {
+               if (params->cpus && !params->monitored_cpus[i])
+                       continue;
+               osnoise_top_print(top, i);
+       }
+
+       trace_seq_do_printf(trace->seq);
+       trace_seq_reset(trace->seq);
+}
+
+/*
+ * osnoise_top_usage - prints osnoise top usage message
+ */
+void osnoise_top_usage(char *usage)
+{
+       int i;
+
+       static const char * const msg[] = {
+               "  usage: rtla osnoise [top] [-h] [-q] [-D] [-d s] [-p us] [-r us] [-s us] [-S us] [-t[=file]] \\",
+               "         [-c cpu-list] [-P priority]",
+               "",
+               "         -h/--help: print this menu",
+               "         -p/--period us: osnoise period in us",
+               "         -r/--runtime us: osnoise runtime in us",
+               "         -s/--stop us: stop trace if a single sample is higher than the argument in us",
+               "         -S/--stop-total us: stop trace if the total sample is higher than the argument in us",
+               "         -c/--cpus cpu-list: list of cpus to run osnoise threads",
+               "         -d/--duration time[s|m|h|d]: duration of the session",
+               "         -D/--debug: print debug info",
+               "         -t/--trace[=file]: save the stopped trace to [file|osnoise_trace.txt]",
+               "         -q/--quiet print only a summary at the end",
+               "         -P/--priority o:prio|r:prio|f:prio|d:runtime:period : set scheduling parameters",
+               "               o:prio - use SCHED_OTHER with prio",
+               "               r:prio - use SCHED_RR with prio",
+               "               f:prio - use SCHED_FIFO with prio",
+               "               d:runtime[us|ms|s]:period[us|ms|s] - use SCHED_DEADLINE with runtime and period",
+               "                                                      in nanoseconds",
+               NULL,
+       };
+
+       if (usage)
+               fprintf(stderr, "%s\n", usage);
+
+       fprintf(stderr, "rtla osnoise top: a per-cpu summary of the OS noise (version %s)\n",
+                       VERSION);
+
+       for (i = 0; msg[i]; i++)
+               fprintf(stderr, "%s\n", msg[i]);
+       exit(1);
+}
+
+/*
+ * osnoise_top_parse_args - allocs, parse and fill the cmd line parameters
+ */
+struct osnoise_top_params *osnoise_top_parse_args(int argc, char **argv)
+{
+       struct osnoise_top_params *params;
+       int retval;
+       int c;
+
+       params = calloc(1, sizeof(*params));
+       if (!params)
+               exit(1);
+
+       while (1) {
+               static struct option long_options[] = {
+                       {"cpus",                required_argument,      0, 'c'},
+                       {"debug",               no_argument,            0, 'D'},
+                       {"duration",            required_argument,      0, 'd'},
+                       {"help",                no_argument,            0, 'h'},
+                       {"period",              required_argument,      0, 'p'},
+                       {"priority",            required_argument,      0, 'P'},
+                       {"quiet",               no_argument,            0, 'q'},
+                       {"runtime",             required_argument,      0, 'r'},
+                       {"stop",                required_argument,      0, 's'},
+                       {"stop-total",          required_argument,      0, 'S'},
+                       {"trace",               optional_argument,      0, 't'},
+                       {0, 0, 0, 0}
+               };
+
+               /* getopt_long stores the option index here. */
+               int option_index = 0;
+
+               c = getopt_long(argc, argv, "c:d:Dhp:P:qr:s:S:t::",
+                                long_options, &option_index);
+
+               /* Detect the end of the options. */
+               if (c == -1)
+                       break;
+
+               switch (c) {
+               case 'c':
+                       retval = parse_cpu_list(optarg, &params->monitored_cpus);
+                       if (retval)
+                               osnoise_top_usage("\nInvalid -c cpu list\n");
+                       params->cpus = optarg;
+                       break;
+               case 'D':
+                       config_debug = 1;
+                       break;
+               case 'd':
+                       params->duration = parse_seconds_duration(optarg);
+                       if (!params->duration)
+                               osnoise_top_usage("Invalid -D duration\n");
+                       break;
+               case 'h':
+               case '?':
+                       osnoise_top_usage(NULL);
+                       break;
+               case 'p':
+                       params->period = get_llong_from_str(optarg);
+                       if (params->period > 10000000)
+                               osnoise_top_usage("Period longer than 10 s\n");
+                       break;
+               case 'P':
+                       retval = parse_prio(optarg, &params->sched_param);
+                       if (retval == -1)
+                               osnoise_top_usage("Invalid -P priority");
+                       params->set_sched = 1;
+                       break;
+               case 'q':
+                       params->quiet = 1;
+                       break;
+               case 'r':
+                       params->runtime = get_llong_from_str(optarg);
+                       if (params->runtime < 100)
+                               osnoise_top_usage("Runtime shorter than 100 us\n");
+                       break;
+               case 's':
+                       params->stop_us = get_llong_from_str(optarg);
+                       break;
+               case 'S':
+                       params->stop_total_us = get_llong_from_str(optarg);
+                       break;
+               case 't':
+                       if (optarg)
+                               /* skip = */
+                               params->trace_output = &optarg[1];
+                       else
+                               params->trace_output = "osnoise_trace.txt";
+                       break;
+               default:
+                       osnoise_top_usage("Invalid option");
+               }
+       }
+
+       if (geteuid()) {
+               err_msg("osnoise needs root permission\n");
+               exit(EXIT_FAILURE);
+       }
+
+       return params;
+}
+
+/*
+ * osnoise_top_apply_config - apply the top configs to the initialized tool
+ */
+static int
+osnoise_top_apply_config(struct osnoise_tool *tool, struct osnoise_top_params *params)
+{
+       int retval;
+
+       if (!params->sleep_time)
+               params->sleep_time = 1;
+
+       if (params->cpus) {
+               retval = osnoise_set_cpus(tool->context, params->cpus);
+               if (retval) {
+                       err_msg("Failed to apply CPUs config\n");
+                       goto out_err;
+               }
+       }
+
+       if (params->runtime || params->period) {
+               retval = osnoise_set_runtime_period(tool->context,
+                                                   params->runtime,
+                                                   params->period);
+               if (retval) {
+                       err_msg("Failed to set runtime and/or period\n");
+                       goto out_err;
+               }
+       }
+
+       if (params->stop_us) {
+               retval = osnoise_set_stop_us(tool->context, params->stop_us);
+               if (retval) {
+                       err_msg("Failed to set stop us\n");
+                       goto out_err;
+               }
+       }
+
+       if (params->stop_total_us) {
+               retval = osnoise_set_stop_total_us(tool->context, params->stop_total_us);
+               if (retval) {
+                       err_msg("Failed to set stop total us\n");
+                       goto out_err;
+               }
+       }
+
+       return 0;
+
+out_err:
+       return -1;
+}
+
+/*
+ * osnoise_init_top - initialize a osnoise top tool with parameters
+ */
+struct osnoise_tool *osnoise_init_top(struct osnoise_top_params *params)
+{
+       struct osnoise_tool *tool;
+       int nr_cpus;
+
+       nr_cpus = sysconf(_SC_NPROCESSORS_CONF);
+
+       tool = osnoise_init_tool("osnoise_top");
+       if (!tool)
+               return NULL;
+
+       tool->data = osnoise_alloc_top(nr_cpus);
+       if (!tool->data)
+               goto out_err;
+
+       tool->params = params;
+
+       tep_register_event_handler(tool->trace.tep, -1, "ftrace", "osnoise",
+                                  osnoise_top_handler, NULL);
+
+       return tool;
+
+out_err:
+       osnoise_free_top(tool->data);
+       osnoise_destroy_tool(tool);
+       return NULL;
+}
+
+static int stop_tracing;
+static void stop_top(int sig)
+{
+       stop_tracing = 1;
+}
+
+/*
+ * osnoise_top_set_signals - handles the signal to stop the tool
+ */
+static void osnoise_top_set_signals(struct osnoise_top_params *params)
+{
+       signal(SIGINT, stop_top);
+       if (params->duration) {
+               signal(SIGALRM, stop_top);
+               alarm(params->duration);
+       }
+}
+
+int osnoise_top_main(int argc, char **argv)
+{
+       struct osnoise_top_params *params;
+       struct trace_instance *trace;
+       struct osnoise_tool *record;
+       struct osnoise_tool *tool;
+       int return_value = 1;
+       int retval;
+
+       params = osnoise_top_parse_args(argc, argv);
+       if (!params)
+               exit(1);
+
+       tool = osnoise_init_top(params);
+       if (!tool) {
+               err_msg("Could not init osnoise top\n");
+               goto out_exit;
+       }
+
+       retval = osnoise_top_apply_config(tool, params);
+       if (retval) {
+               err_msg("Could not apply config\n");
+               goto out_top;
+       }
+
+       trace = &tool->trace;
+
+       retval = enable_osnoise(trace);
+       if (retval) {
+               err_msg("Failed to enable osnoise tracer\n");
+               goto out_top;
+       }
+
+       if (params->set_sched) {
+               retval = set_comm_sched_attr("osnoise/", &params->sched_param);
+               if (retval) {
+                       err_msg("Failed to set sched parameters\n");
+                       goto out_top;
+               }
+       }
+
+       trace_instance_start(trace);
+
+       if (params->trace_output) {
+               record = osnoise_init_trace_tool("osnoise");
+               if (!record) {
+                       err_msg("Failed to enable the trace instance\n");
+                       goto out_top;
+               }
+               trace_instance_start(&record->trace);
+       }
+
+       tool->start_time = time(NULL);
+       osnoise_top_set_signals(params);
+
+       do {
+               sleep(params->sleep_time);
+
+               retval = tracefs_iterate_raw_events(trace->tep,
+                                                   trace->inst,
+                                                   NULL,
+                                                   0,
+                                                   collect_registered_events,
+                                                   trace);
+               if (retval < 0) {
+                       err_msg("Error iterating on events\n");
+                       goto out_top;
+               }
+
+               if (!params->quiet)
+                       osnoise_print_stats(params, tool);
+
+               if (!tracefs_trace_is_on(trace->inst))
+                       break;
+
+       } while (!stop_tracing);
+
+       osnoise_print_stats(params, tool);
+
+       return_value = 0;
+
+       if (!tracefs_trace_is_on(trace->inst)) {
+               printf("osnoise hit stop tracing\n");
+               if (params->trace_output) {
+                       printf("  Saving trace to %s\n", params->trace_output);
+                       save_trace_to_file(record->trace.inst, params->trace_output);
+               }
+       }
+
+out_top:
+       osnoise_free_top(tool->data);
+       osnoise_destroy_tool(tool);
+       if (params->trace_output)
+               osnoise_destroy_tool(record);
+out_exit:
+       exit(return_value);
+}