1 // SPDX-License-Identifier: GPL-2.0
3 * trace_events_hist - trace event hist triggers
5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
8 #include <linux/module.h>
9 #include <linux/kallsyms.h>
10 #include <linux/security.h>
11 #include <linux/mutex.h>
12 #include <linux/slab.h>
13 #include <linux/stacktrace.h>
14 #include <linux/rculist.h>
15 #include <linux/tracefs.h>
17 /* for gfp flag names */
18 #include <linux/trace_events.h>
19 #include <trace/events/mmflags.h>
21 #include "tracing_map.h"
22 #include "trace_synth.h"
25 C(NONE, "No error"), \
26 C(DUPLICATE_VAR, "Variable already defined"), \
27 C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
28 C(TOO_MANY_VARS, "Too many variables defined"), \
29 C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \
30 C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \
31 C(TRIGGER_EEXIST, "Hist trigger already exists"), \
32 C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
33 C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \
34 C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \
35 C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \
36 C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \
37 C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \
38 C(EVENT_FILE_NOT_FOUND, "Event file not found"), \
39 C(HIST_NOT_FOUND, "Matching event histogram not found"), \
40 C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \
41 C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \
42 C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \
43 C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \
44 C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
45 C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \
46 C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \
47 C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \
48 C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \
49 C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \
50 C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \
51 C(TOO_MANY_PARAMS, "Too many action params"), \
52 C(PARAM_NOT_FOUND, "Couldn't find param"), \
53 C(INVALID_PARAM, "Invalid action param"), \
54 C(ACTION_NOT_FOUND, "No action found"), \
55 C(NO_SAVE_PARAMS, "No params found for save()"), \
56 C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
57 C(ACTION_MISMATCH, "Handler doesn't support action"), \
58 C(NO_CLOSING_PAREN, "No closing paren found"), \
59 C(SUBSYS_NOT_FOUND, "Missing subsystem"), \
60 C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \
61 C(INVALID_REF_KEY, "Using variable references in keys not supported"), \
62 C(VAR_NOT_FOUND, "Couldn't find variable"), \
63 C(FIELD_NOT_FOUND, "Couldn't find field"), \
64 C(EMPTY_ASSIGNMENT, "Empty assignment"), \
65 C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \
66 C(EMPTY_SORT_FIELD, "Empty sort field"), \
67 C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \
68 C(INVALID_SORT_FIELD, "Sort field must be a key or a val"),
71 #define C(a, b) HIST_ERR_##a
78 static const char *err_text[] = { ERRORS };
82 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
83 struct tracing_map_elt *elt,
84 struct ring_buffer_event *rbe,
87 #define HIST_FIELD_OPERANDS_MAX 2
88 #define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
89 #define HIST_ACTIONS_MAX 8
99 * A hist_var (histogram variable) contains variable information for
100 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
101 * flag set. A hist_var has a variable name e.g. ts0, and is
102 * associated with a given histogram trigger, as specified by
103 * hist_data. The hist_var idx is the unique index assigned to the
104 * variable by the hist trigger's tracing_map. The idx is what is
105 * used to set a variable's value and, by a variable reference, to
110 struct hist_trigger_data *hist_data;
115 struct ftrace_event_field *field;
121 unsigned int is_signed;
123 struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
124 struct hist_trigger_data *hist_data;
127 * Variable fields contain variable-specific info in var.
130 enum field_op_id operator;
135 * The name field is used for EXPR and VAR_REF fields. VAR
136 * fields contain the variable name in var.name.
141 * When a histogram trigger is hit, if it has any references
142 * to variables, the values of those variables are collected
143 * into a var_ref_vals array by resolve_var_refs(). The
144 * current value of each variable is read from the tracing_map
145 * using the hist field's hist_var.idx and entered into the
146 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
148 unsigned int var_ref_idx;
152 static u64 hist_field_none(struct hist_field *field,
153 struct tracing_map_elt *elt,
154 struct ring_buffer_event *rbe,
160 static u64 hist_field_counter(struct hist_field *field,
161 struct tracing_map_elt *elt,
162 struct ring_buffer_event *rbe,
168 static u64 hist_field_string(struct hist_field *hist_field,
169 struct tracing_map_elt *elt,
170 struct ring_buffer_event *rbe,
173 char *addr = (char *)(event + hist_field->field->offset);
175 return (u64)(unsigned long)addr;
178 static u64 hist_field_dynstring(struct hist_field *hist_field,
179 struct tracing_map_elt *elt,
180 struct ring_buffer_event *rbe,
183 u32 str_item = *(u32 *)(event + hist_field->field->offset);
184 int str_loc = str_item & 0xffff;
185 char *addr = (char *)(event + str_loc);
187 return (u64)(unsigned long)addr;
190 static u64 hist_field_pstring(struct hist_field *hist_field,
191 struct tracing_map_elt *elt,
192 struct ring_buffer_event *rbe,
195 char **addr = (char **)(event + hist_field->field->offset);
197 return (u64)(unsigned long)*addr;
200 static u64 hist_field_log2(struct hist_field *hist_field,
201 struct tracing_map_elt *elt,
202 struct ring_buffer_event *rbe,
205 struct hist_field *operand = hist_field->operands[0];
207 u64 val = operand->fn(operand, elt, rbe, event);
209 return (u64) ilog2(roundup_pow_of_two(val));
212 static u64 hist_field_plus(struct hist_field *hist_field,
213 struct tracing_map_elt *elt,
214 struct ring_buffer_event *rbe,
217 struct hist_field *operand1 = hist_field->operands[0];
218 struct hist_field *operand2 = hist_field->operands[1];
220 u64 val1 = operand1->fn(operand1, elt, rbe, event);
221 u64 val2 = operand2->fn(operand2, elt, rbe, event);
226 static u64 hist_field_minus(struct hist_field *hist_field,
227 struct tracing_map_elt *elt,
228 struct ring_buffer_event *rbe,
231 struct hist_field *operand1 = hist_field->operands[0];
232 struct hist_field *operand2 = hist_field->operands[1];
234 u64 val1 = operand1->fn(operand1, elt, rbe, event);
235 u64 val2 = operand2->fn(operand2, elt, rbe, event);
240 static u64 hist_field_unary_minus(struct hist_field *hist_field,
241 struct tracing_map_elt *elt,
242 struct ring_buffer_event *rbe,
245 struct hist_field *operand = hist_field->operands[0];
247 s64 sval = (s64)operand->fn(operand, elt, rbe, event);
248 u64 val = (u64)-sval;
253 #define DEFINE_HIST_FIELD_FN(type) \
254 static u64 hist_field_##type(struct hist_field *hist_field, \
255 struct tracing_map_elt *elt, \
256 struct ring_buffer_event *rbe, \
259 type *addr = (type *)(event + hist_field->field->offset); \
261 return (u64)(unsigned long)*addr; \
264 DEFINE_HIST_FIELD_FN(s64);
265 DEFINE_HIST_FIELD_FN(u64);
266 DEFINE_HIST_FIELD_FN(s32);
267 DEFINE_HIST_FIELD_FN(u32);
268 DEFINE_HIST_FIELD_FN(s16);
269 DEFINE_HIST_FIELD_FN(u16);
270 DEFINE_HIST_FIELD_FN(s8);
271 DEFINE_HIST_FIELD_FN(u8);
273 #define for_each_hist_field(i, hist_data) \
274 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
276 #define for_each_hist_val_field(i, hist_data) \
277 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
279 #define for_each_hist_key_field(i, hist_data) \
280 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
282 #define HIST_STACKTRACE_DEPTH 16
283 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
284 #define HIST_STACKTRACE_SKIP 5
286 #define HITCOUNT_IDX 0
287 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
289 enum hist_field_flags {
290 HIST_FIELD_FL_HITCOUNT = 1 << 0,
291 HIST_FIELD_FL_KEY = 1 << 1,
292 HIST_FIELD_FL_STRING = 1 << 2,
293 HIST_FIELD_FL_HEX = 1 << 3,
294 HIST_FIELD_FL_SYM = 1 << 4,
295 HIST_FIELD_FL_SYM_OFFSET = 1 << 5,
296 HIST_FIELD_FL_EXECNAME = 1 << 6,
297 HIST_FIELD_FL_SYSCALL = 1 << 7,
298 HIST_FIELD_FL_STACKTRACE = 1 << 8,
299 HIST_FIELD_FL_LOG2 = 1 << 9,
300 HIST_FIELD_FL_TIMESTAMP = 1 << 10,
301 HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11,
302 HIST_FIELD_FL_VAR = 1 << 12,
303 HIST_FIELD_FL_EXPR = 1 << 13,
304 HIST_FIELD_FL_VAR_REF = 1 << 14,
305 HIST_FIELD_FL_CPU = 1 << 15,
306 HIST_FIELD_FL_ALIAS = 1 << 16,
311 char *name[TRACING_MAP_VARS_MAX];
312 char *expr[TRACING_MAP_VARS_MAX];
315 struct hist_trigger_attrs {
325 unsigned int map_bits;
327 char *assignment_str[TRACING_MAP_VARS_MAX];
328 unsigned int n_assignments;
330 char *action_str[HIST_ACTIONS_MAX];
331 unsigned int n_actions;
333 struct var_defs var_defs;
337 struct hist_field *var;
338 struct hist_field *val;
341 struct field_var_hist {
342 struct hist_trigger_data *hist_data;
346 struct hist_trigger_data {
347 struct hist_field *fields[HIST_FIELDS_MAX];
350 unsigned int n_fields;
352 unsigned int key_size;
353 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
354 unsigned int n_sort_keys;
355 struct trace_event_file *event_file;
356 struct hist_trigger_attrs *attrs;
357 struct tracing_map *map;
358 bool enable_timestamps;
360 struct hist_field *var_refs[TRACING_MAP_VARS_MAX];
361 unsigned int n_var_refs;
363 struct action_data *actions[HIST_ACTIONS_MAX];
364 unsigned int n_actions;
366 struct field_var *field_vars[SYNTH_FIELDS_MAX];
367 unsigned int n_field_vars;
368 unsigned int n_field_var_str;
369 struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX];
370 unsigned int n_field_var_hists;
372 struct field_var *save_vars[SYNTH_FIELDS_MAX];
373 unsigned int n_save_vars;
374 unsigned int n_save_var_str;
379 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
380 struct tracing_map_elt *elt, void *rec,
381 struct ring_buffer_event *rbe, void *key,
382 struct action_data *data, u64 *var_ref_vals);
384 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
399 enum handler_id handler;
400 enum action_id action;
404 unsigned int n_params;
405 char *params[SYNTH_FIELDS_MAX];
408 * When a histogram trigger is hit, the values of any
409 * references to variables, including variables being passed
410 * as parameters to synthetic events, are collected into a
411 * var_ref_vals array. This var_ref_idx array is an array of
412 * indices into the var_ref_vals array, one for each synthetic
413 * event param, and is passed to the synthetic event
416 unsigned int var_ref_idx[TRACING_MAP_VARS_MAX];
417 struct synth_event *synth_event;
418 bool use_trace_keyword;
419 char *synth_event_name;
429 * var_str contains the $-unstripped variable
430 * name referenced by var_ref, and used when
431 * printing the action. Because var_ref
432 * creation is deferred to create_actions(),
433 * we need a per-action way to save it until
434 * then, thus var_str.
439 * var_ref refers to the variable being
440 * tracked e.g onmax($var).
442 struct hist_field *var_ref;
445 * track_var contains the 'invisible' tracking
446 * variable created to keep the current
449 struct hist_field *track_var;
451 check_track_val_fn_t check_val;
452 action_fn_t save_data;
461 unsigned int key_len;
463 struct tracing_map_elt elt;
465 struct action_data *action_data;
466 struct hist_trigger_data *hist_data;
469 struct hist_elt_data {
472 char *field_var_str[SYNTH_FIELDS_MAX];
475 struct snapshot_context {
476 struct tracing_map_elt *elt;
480 static void track_data_free(struct track_data *track_data)
482 struct hist_elt_data *elt_data;
487 kfree(track_data->key);
489 elt_data = track_data->elt.private_data;
491 kfree(elt_data->comm);
498 static struct track_data *track_data_alloc(unsigned int key_len,
499 struct action_data *action_data,
500 struct hist_trigger_data *hist_data)
502 struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
503 struct hist_elt_data *elt_data;
506 return ERR_PTR(-ENOMEM);
508 data->key = kzalloc(key_len, GFP_KERNEL);
510 track_data_free(data);
511 return ERR_PTR(-ENOMEM);
514 data->key_len = key_len;
515 data->action_data = action_data;
516 data->hist_data = hist_data;
518 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
520 track_data_free(data);
521 return ERR_PTR(-ENOMEM);
524 data->elt.private_data = elt_data;
526 elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
527 if (!elt_data->comm) {
528 track_data_free(data);
529 return ERR_PTR(-ENOMEM);
535 static char last_cmd[MAX_FILTER_STR_VAL];
536 static char last_cmd_loc[MAX_FILTER_STR_VAL];
538 static int errpos(char *str)
540 return err_pos(last_cmd, str);
543 static void last_cmd_set(struct trace_event_file *file, char *str)
545 const char *system = NULL, *name = NULL;
546 struct trace_event_call *call;
551 strcpy(last_cmd, "hist:");
552 strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
555 call = file->event_call;
556 system = call->class->system;
558 name = trace_event_name(call);
565 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
568 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
570 tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
574 static void hist_err_clear(void)
577 last_cmd_loc[0] = '\0';
580 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
581 unsigned int *var_ref_idx);
583 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
584 unsigned int *var_ref_idx)
586 struct tracepoint *tp = event->tp;
588 if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
589 struct tracepoint_func *probe_func_ptr;
590 synth_probe_func_t probe_func;
593 if (!(cpu_online(raw_smp_processor_id())))
596 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
597 if (probe_func_ptr) {
599 probe_func = probe_func_ptr->func;
600 __data = probe_func_ptr->data;
601 probe_func(__data, var_ref_vals, var_ref_idx);
602 } while ((++probe_func_ptr)->func);
607 static void action_trace(struct hist_trigger_data *hist_data,
608 struct tracing_map_elt *elt, void *rec,
609 struct ring_buffer_event *rbe, void *key,
610 struct action_data *data, u64 *var_ref_vals)
612 struct synth_event *event = data->synth_event;
614 trace_synth(event, var_ref_vals, data->var_ref_idx);
617 struct hist_var_data {
618 struct list_head list;
619 struct hist_trigger_data *hist_data;
622 static u64 hist_field_timestamp(struct hist_field *hist_field,
623 struct tracing_map_elt *elt,
624 struct ring_buffer_event *rbe,
627 struct hist_trigger_data *hist_data = hist_field->hist_data;
628 struct trace_array *tr = hist_data->event_file->tr;
630 u64 ts = ring_buffer_event_time_stamp(rbe);
632 if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
638 static u64 hist_field_cpu(struct hist_field *hist_field,
639 struct tracing_map_elt *elt,
640 struct ring_buffer_event *rbe,
643 int cpu = smp_processor_id();
649 * check_field_for_var_ref - Check if a VAR_REF field references a variable
650 * @hist_field: The VAR_REF field to check
651 * @var_data: The hist trigger that owns the variable
652 * @var_idx: The trigger variable identifier
654 * Check the given VAR_REF field to see whether or not it references
655 * the given variable associated with the given trigger.
657 * Return: The VAR_REF field if it does reference the variable, NULL if not
659 static struct hist_field *
660 check_field_for_var_ref(struct hist_field *hist_field,
661 struct hist_trigger_data *var_data,
662 unsigned int var_idx)
664 WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
666 if (hist_field && hist_field->var.idx == var_idx &&
667 hist_field->var.hist_data == var_data)
674 * find_var_ref - Check if a trigger has a reference to a trigger variable
675 * @hist_data: The hist trigger that might have a reference to the variable
676 * @var_data: The hist trigger that owns the variable
677 * @var_idx: The trigger variable identifier
679 * Check the list of var_refs[] on the first hist trigger to see
680 * whether any of them are references to the variable on the second
683 * Return: The VAR_REF field referencing the variable if so, NULL if not
685 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
686 struct hist_trigger_data *var_data,
687 unsigned int var_idx)
689 struct hist_field *hist_field;
692 for (i = 0; i < hist_data->n_var_refs; i++) {
693 hist_field = hist_data->var_refs[i];
694 if (check_field_for_var_ref(hist_field, var_data, var_idx))
702 * find_any_var_ref - Check if there is a reference to a given trigger variable
703 * @hist_data: The hist trigger
704 * @var_idx: The trigger variable identifier
706 * Check to see whether the given variable is currently referenced by
709 * The trigger the variable is defined on is explicitly excluded - the
710 * assumption being that a self-reference doesn't prevent a trigger
711 * from being removed.
713 * Return: The VAR_REF field referencing the variable if so, NULL if not
715 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
716 unsigned int var_idx)
718 struct trace_array *tr = hist_data->event_file->tr;
719 struct hist_field *found = NULL;
720 struct hist_var_data *var_data;
722 list_for_each_entry(var_data, &tr->hist_vars, list) {
723 if (var_data->hist_data == hist_data)
725 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
734 * check_var_refs - Check if there is a reference to any of trigger's variables
735 * @hist_data: The hist trigger
737 * A trigger can define one or more variables. If any one of them is
738 * currently referenced by any other trigger, this function will
741 * Typically used to determine whether or not a trigger can be removed
742 * - if there are any references to a trigger's variables, it cannot.
744 * Return: True if there is a reference to any of trigger's variables
746 static bool check_var_refs(struct hist_trigger_data *hist_data)
748 struct hist_field *field;
752 for_each_hist_field(i, hist_data) {
753 field = hist_data->fields[i];
754 if (field && field->flags & HIST_FIELD_FL_VAR) {
755 if (find_any_var_ref(hist_data, field->var.idx)) {
765 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
767 struct trace_array *tr = hist_data->event_file->tr;
768 struct hist_var_data *var_data, *found = NULL;
770 list_for_each_entry(var_data, &tr->hist_vars, list) {
771 if (var_data->hist_data == hist_data) {
780 static bool field_has_hist_vars(struct hist_field *hist_field,
791 if (hist_field->flags & HIST_FIELD_FL_VAR ||
792 hist_field->flags & HIST_FIELD_FL_VAR_REF)
795 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
796 struct hist_field *operand;
798 operand = hist_field->operands[i];
799 if (field_has_hist_vars(operand, level + 1))
806 static bool has_hist_vars(struct hist_trigger_data *hist_data)
808 struct hist_field *hist_field;
811 for_each_hist_field(i, hist_data) {
812 hist_field = hist_data->fields[i];
813 if (field_has_hist_vars(hist_field, 0))
820 static int save_hist_vars(struct hist_trigger_data *hist_data)
822 struct trace_array *tr = hist_data->event_file->tr;
823 struct hist_var_data *var_data;
825 var_data = find_hist_vars(hist_data);
829 if (tracing_check_open_get_tr(tr))
832 var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
838 var_data->hist_data = hist_data;
839 list_add(&var_data->list, &tr->hist_vars);
844 static void remove_hist_vars(struct hist_trigger_data *hist_data)
846 struct trace_array *tr = hist_data->event_file->tr;
847 struct hist_var_data *var_data;
849 var_data = find_hist_vars(hist_data);
853 if (WARN_ON(check_var_refs(hist_data)))
856 list_del(&var_data->list);
863 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
864 const char *var_name)
866 struct hist_field *hist_field, *found = NULL;
869 for_each_hist_field(i, hist_data) {
870 hist_field = hist_data->fields[i];
871 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
872 strcmp(hist_field->var.name, var_name) == 0) {
881 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
882 struct trace_event_file *file,
883 const char *var_name)
885 struct hist_trigger_data *test_data;
886 struct event_trigger_data *test;
887 struct hist_field *hist_field;
889 lockdep_assert_held(&event_mutex);
891 hist_field = find_var_field(hist_data, var_name);
895 list_for_each_entry(test, &file->triggers, list) {
896 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
897 test_data = test->private_data;
898 hist_field = find_var_field(test_data, var_name);
907 static struct trace_event_file *find_var_file(struct trace_array *tr,
912 struct hist_trigger_data *var_hist_data;
913 struct hist_var_data *var_data;
914 struct trace_event_file *file, *found = NULL;
917 return find_event_file(tr, system, event_name);
919 list_for_each_entry(var_data, &tr->hist_vars, list) {
920 var_hist_data = var_data->hist_data;
921 file = var_hist_data->event_file;
925 if (find_var_field(var_hist_data, var_name)) {
927 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
938 static struct hist_field *find_file_var(struct trace_event_file *file,
939 const char *var_name)
941 struct hist_trigger_data *test_data;
942 struct event_trigger_data *test;
943 struct hist_field *hist_field;
945 lockdep_assert_held(&event_mutex);
947 list_for_each_entry(test, &file->triggers, list) {
948 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
949 test_data = test->private_data;
950 hist_field = find_var_field(test_data, var_name);
959 static struct hist_field *
960 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
962 struct trace_array *tr = hist_data->event_file->tr;
963 struct hist_field *hist_field, *found = NULL;
964 struct trace_event_file *file;
967 for (i = 0; i < hist_data->n_actions; i++) {
968 struct action_data *data = hist_data->actions[i];
970 if (data->handler == HANDLER_ONMATCH) {
971 char *system = data->match_data.event_system;
972 char *event_name = data->match_data.event;
974 file = find_var_file(tr, system, event_name, var_name);
977 hist_field = find_file_var(file, var_name);
980 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
982 return ERR_PTR(-EINVAL);
992 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
997 struct trace_array *tr = hist_data->event_file->tr;
998 struct hist_field *hist_field = NULL;
999 struct trace_event_file *file;
1001 if (!system || !event_name) {
1002 hist_field = find_match_var(hist_data, var_name);
1003 if (IS_ERR(hist_field))
1009 file = find_var_file(tr, system, event_name, var_name);
1013 hist_field = find_file_var(file, var_name);
1018 static u64 hist_field_var_ref(struct hist_field *hist_field,
1019 struct tracing_map_elt *elt,
1020 struct ring_buffer_event *rbe,
1023 struct hist_elt_data *elt_data;
1026 if (WARN_ON_ONCE(!elt))
1029 elt_data = elt->private_data;
1030 var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1035 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1036 u64 *var_ref_vals, bool self)
1038 struct hist_trigger_data *var_data;
1039 struct tracing_map_elt *var_elt;
1040 struct hist_field *hist_field;
1041 unsigned int i, var_idx;
1042 bool resolved = true;
1045 for (i = 0; i < hist_data->n_var_refs; i++) {
1046 hist_field = hist_data->var_refs[i];
1047 var_idx = hist_field->var.idx;
1048 var_data = hist_field->var.hist_data;
1050 if (var_data == NULL) {
1055 if ((self && var_data != hist_data) ||
1056 (!self && var_data == hist_data))
1059 var_elt = tracing_map_lookup(var_data->map, key);
1065 if (!tracing_map_var_set(var_elt, var_idx)) {
1070 if (self || !hist_field->read_once)
1071 var_val = tracing_map_read_var(var_elt, var_idx);
1073 var_val = tracing_map_read_var_once(var_elt, var_idx);
1075 var_ref_vals[i] = var_val;
1081 static const char *hist_field_name(struct hist_field *field,
1084 const char *field_name = "";
1090 field_name = field->field->name;
1091 else if (field->flags & HIST_FIELD_FL_LOG2 ||
1092 field->flags & HIST_FIELD_FL_ALIAS)
1093 field_name = hist_field_name(field->operands[0], ++level);
1094 else if (field->flags & HIST_FIELD_FL_CPU)
1096 else if (field->flags & HIST_FIELD_FL_EXPR ||
1097 field->flags & HIST_FIELD_FL_VAR_REF) {
1098 if (field->system) {
1099 static char full_name[MAX_FILTER_STR_VAL];
1101 strcat(full_name, field->system);
1102 strcat(full_name, ".");
1103 strcat(full_name, field->event_name);
1104 strcat(full_name, ".");
1105 strcat(full_name, field->name);
1106 field_name = full_name;
1108 field_name = field->name;
1109 } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1110 field_name = "common_timestamp";
1112 if (field_name == NULL)
1118 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1120 hist_field_fn_t fn = NULL;
1122 switch (field_size) {
1124 if (field_is_signed)
1125 fn = hist_field_s64;
1127 fn = hist_field_u64;
1130 if (field_is_signed)
1131 fn = hist_field_s32;
1133 fn = hist_field_u32;
1136 if (field_is_signed)
1137 fn = hist_field_s16;
1139 fn = hist_field_u16;
1142 if (field_is_signed)
1152 static int parse_map_size(char *str)
1154 unsigned long size, map_bits;
1157 ret = kstrtoul(str, 0, &size);
1161 map_bits = ilog2(roundup_pow_of_two(size));
1162 if (map_bits < TRACING_MAP_BITS_MIN ||
1163 map_bits > TRACING_MAP_BITS_MAX)
1171 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1178 for (i = 0; i < attrs->n_assignments; i++)
1179 kfree(attrs->assignment_str[i]);
1181 for (i = 0; i < attrs->n_actions; i++)
1182 kfree(attrs->action_str[i]);
1185 kfree(attrs->sort_key_str);
1186 kfree(attrs->keys_str);
1187 kfree(attrs->vals_str);
1188 kfree(attrs->clock);
1192 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1196 if (attrs->n_actions >= HIST_ACTIONS_MAX)
1199 if ((str_has_prefix(str, "onmatch(")) ||
1200 (str_has_prefix(str, "onmax(")) ||
1201 (str_has_prefix(str, "onchange("))) {
1202 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1203 if (!attrs->action_str[attrs->n_actions]) {
1213 static int parse_assignment(struct trace_array *tr,
1214 char *str, struct hist_trigger_attrs *attrs)
1218 if ((len = str_has_prefix(str, "key=")) ||
1219 (len = str_has_prefix(str, "keys="))) {
1220 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1221 if (!attrs->keys_str) {
1225 } else if ((len = str_has_prefix(str, "val=")) ||
1226 (len = str_has_prefix(str, "vals=")) ||
1227 (len = str_has_prefix(str, "values="))) {
1228 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1229 if (!attrs->vals_str) {
1233 } else if ((len = str_has_prefix(str, "sort="))) {
1234 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1235 if (!attrs->sort_key_str) {
1239 } else if (str_has_prefix(str, "name=")) {
1240 attrs->name = kstrdup(str, GFP_KERNEL);
1245 } else if ((len = str_has_prefix(str, "clock="))) {
1248 str = strstrip(str);
1249 attrs->clock = kstrdup(str, GFP_KERNEL);
1250 if (!attrs->clock) {
1254 } else if ((len = str_has_prefix(str, "size="))) {
1255 int map_bits = parse_map_size(str + len);
1261 attrs->map_bits = map_bits;
1265 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1266 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1271 assignment = kstrdup(str, GFP_KERNEL);
1277 attrs->assignment_str[attrs->n_assignments++] = assignment;
1283 static struct hist_trigger_attrs *
1284 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1286 struct hist_trigger_attrs *attrs;
1289 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1291 return ERR_PTR(-ENOMEM);
1293 while (trigger_str) {
1294 char *str = strsep(&trigger_str, ":");
1297 rhs = strchr(str, '=');
1299 if (!strlen(++rhs)) {
1301 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1304 ret = parse_assignment(tr, str, attrs);
1307 } else if (strcmp(str, "pause") == 0)
1308 attrs->pause = true;
1309 else if ((strcmp(str, "cont") == 0) ||
1310 (strcmp(str, "continue") == 0))
1312 else if (strcmp(str, "clear") == 0)
1313 attrs->clear = true;
1315 ret = parse_action(str, attrs);
1321 if (!attrs->keys_str) {
1326 if (!attrs->clock) {
1327 attrs->clock = kstrdup("global", GFP_KERNEL);
1328 if (!attrs->clock) {
1336 destroy_hist_trigger_attrs(attrs);
1338 return ERR_PTR(ret);
1341 static inline void save_comm(char *comm, struct task_struct *task)
1344 strcpy(comm, "<idle>");
1348 if (WARN_ON_ONCE(task->pid < 0)) {
1349 strcpy(comm, "<XXX>");
1353 strncpy(comm, task->comm, TASK_COMM_LEN);
1356 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1360 for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1361 kfree(elt_data->field_var_str[i]);
1363 kfree(elt_data->comm);
1367 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1369 struct hist_elt_data *elt_data = elt->private_data;
1371 hist_elt_data_free(elt_data);
1374 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1376 struct hist_trigger_data *hist_data = elt->map->private_data;
1377 unsigned int size = TASK_COMM_LEN;
1378 struct hist_elt_data *elt_data;
1379 struct hist_field *key_field;
1380 unsigned int i, n_str;
1382 elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1386 for_each_hist_key_field(i, hist_data) {
1387 key_field = hist_data->fields[i];
1389 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1390 elt_data->comm = kzalloc(size, GFP_KERNEL);
1391 if (!elt_data->comm) {
1399 n_str = hist_data->n_field_var_str + hist_data->n_save_var_str;
1401 size = STR_VAR_LEN_MAX;
1403 for (i = 0; i < n_str; i++) {
1404 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1405 if (!elt_data->field_var_str[i]) {
1406 hist_elt_data_free(elt_data);
1411 elt->private_data = elt_data;
1416 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1418 struct hist_elt_data *elt_data = elt->private_data;
1421 save_comm(elt_data->comm, current);
1424 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1425 .elt_alloc = hist_trigger_elt_data_alloc,
1426 .elt_free = hist_trigger_elt_data_free,
1427 .elt_init = hist_trigger_elt_data_init,
1430 static const char *get_hist_field_flags(struct hist_field *hist_field)
1432 const char *flags_str = NULL;
1434 if (hist_field->flags & HIST_FIELD_FL_HEX)
1436 else if (hist_field->flags & HIST_FIELD_FL_SYM)
1438 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1439 flags_str = "sym-offset";
1440 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1441 flags_str = "execname";
1442 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1443 flags_str = "syscall";
1444 else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1446 else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1447 flags_str = "usecs";
1452 static void expr_field_str(struct hist_field *field, char *expr)
1454 if (field->flags & HIST_FIELD_FL_VAR_REF)
1457 strcat(expr, hist_field_name(field, 0));
1459 if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1460 const char *flags_str = get_hist_field_flags(field);
1464 strcat(expr, flags_str);
1469 static char *expr_str(struct hist_field *field, unsigned int level)
1476 expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1480 if (!field->operands[0]) {
1481 expr_field_str(field, expr);
1485 if (field->operator == FIELD_OP_UNARY_MINUS) {
1489 subexpr = expr_str(field->operands[0], ++level);
1494 strcat(expr, subexpr);
1502 expr_field_str(field->operands[0], expr);
1504 switch (field->operator) {
1505 case FIELD_OP_MINUS:
1516 expr_field_str(field->operands[1], expr);
1521 static int contains_operator(char *str)
1523 enum field_op_id field_op = FIELD_OP_NONE;
1526 op = strpbrk(str, "+-");
1528 return FIELD_OP_NONE;
1533 field_op = FIELD_OP_UNARY_MINUS;
1535 field_op = FIELD_OP_MINUS;
1538 field_op = FIELD_OP_PLUS;
1547 static void get_hist_field(struct hist_field *hist_field)
1552 static void __destroy_hist_field(struct hist_field *hist_field)
1554 if (--hist_field->ref > 1)
1557 kfree(hist_field->var.name);
1558 kfree(hist_field->name);
1559 kfree(hist_field->type);
1561 kfree(hist_field->system);
1562 kfree(hist_field->event_name);
1567 static void destroy_hist_field(struct hist_field *hist_field,
1578 if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1579 return; /* var refs will be destroyed separately */
1581 for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1582 destroy_hist_field(hist_field->operands[i], level + 1);
1584 __destroy_hist_field(hist_field);
1587 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1588 struct ftrace_event_field *field,
1589 unsigned long flags,
1592 struct hist_field *hist_field;
1594 if (field && is_function_field(field))
1597 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1601 hist_field->ref = 1;
1603 hist_field->hist_data = hist_data;
1605 if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1606 goto out; /* caller will populate */
1608 if (flags & HIST_FIELD_FL_VAR_REF) {
1609 hist_field->fn = hist_field_var_ref;
1613 if (flags & HIST_FIELD_FL_HITCOUNT) {
1614 hist_field->fn = hist_field_counter;
1615 hist_field->size = sizeof(u64);
1616 hist_field->type = kstrdup("u64", GFP_KERNEL);
1617 if (!hist_field->type)
1622 if (flags & HIST_FIELD_FL_STACKTRACE) {
1623 hist_field->fn = hist_field_none;
1627 if (flags & HIST_FIELD_FL_LOG2) {
1628 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1629 hist_field->fn = hist_field_log2;
1630 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1631 hist_field->size = hist_field->operands[0]->size;
1632 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1633 if (!hist_field->type)
1638 if (flags & HIST_FIELD_FL_TIMESTAMP) {
1639 hist_field->fn = hist_field_timestamp;
1640 hist_field->size = sizeof(u64);
1641 hist_field->type = kstrdup("u64", GFP_KERNEL);
1642 if (!hist_field->type)
1647 if (flags & HIST_FIELD_FL_CPU) {
1648 hist_field->fn = hist_field_cpu;
1649 hist_field->size = sizeof(int);
1650 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1651 if (!hist_field->type)
1656 if (WARN_ON_ONCE(!field))
1659 if (is_string_field(field)) {
1660 flags |= HIST_FIELD_FL_STRING;
1662 hist_field->size = MAX_FILTER_STR_VAL;
1663 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1664 if (!hist_field->type)
1667 if (field->filter_type == FILTER_STATIC_STRING)
1668 hist_field->fn = hist_field_string;
1669 else if (field->filter_type == FILTER_DYN_STRING)
1670 hist_field->fn = hist_field_dynstring;
1672 hist_field->fn = hist_field_pstring;
1674 hist_field->size = field->size;
1675 hist_field->is_signed = field->is_signed;
1676 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1677 if (!hist_field->type)
1680 hist_field->fn = select_value_fn(field->size,
1682 if (!hist_field->fn) {
1683 destroy_hist_field(hist_field, 0);
1688 hist_field->field = field;
1689 hist_field->flags = flags;
1692 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1693 if (!hist_field->var.name)
1699 destroy_hist_field(hist_field, 0);
1703 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1707 for (i = 0; i < HIST_FIELDS_MAX; i++) {
1708 if (hist_data->fields[i]) {
1709 destroy_hist_field(hist_data->fields[i], 0);
1710 hist_data->fields[i] = NULL;
1714 for (i = 0; i < hist_data->n_var_refs; i++) {
1715 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1716 __destroy_hist_field(hist_data->var_refs[i]);
1717 hist_data->var_refs[i] = NULL;
1721 static int init_var_ref(struct hist_field *ref_field,
1722 struct hist_field *var_field,
1723 char *system, char *event_name)
1727 ref_field->var.idx = var_field->var.idx;
1728 ref_field->var.hist_data = var_field->hist_data;
1729 ref_field->size = var_field->size;
1730 ref_field->is_signed = var_field->is_signed;
1731 ref_field->flags |= var_field->flags &
1732 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1735 ref_field->system = kstrdup(system, GFP_KERNEL);
1736 if (!ref_field->system)
1741 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1742 if (!ref_field->event_name) {
1748 if (var_field->var.name) {
1749 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1750 if (!ref_field->name) {
1754 } else if (var_field->name) {
1755 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1756 if (!ref_field->name) {
1762 ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1763 if (!ref_field->type) {
1770 kfree(ref_field->system);
1771 kfree(ref_field->event_name);
1772 kfree(ref_field->name);
1777 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1778 struct hist_field *var_field)
1780 struct hist_field *ref_field;
1783 for (i = 0; i < hist_data->n_var_refs; i++) {
1784 ref_field = hist_data->var_refs[i];
1785 if (ref_field->var.idx == var_field->var.idx &&
1786 ref_field->var.hist_data == var_field->hist_data)
1794 * create_var_ref - Create a variable reference and attach it to trigger
1795 * @hist_data: The trigger that will be referencing the variable
1796 * @var_field: The VAR field to create a reference to
1797 * @system: The optional system string
1798 * @event_name: The optional event_name string
1800 * Given a variable hist_field, create a VAR_REF hist_field that
1801 * represents a reference to it.
1803 * This function also adds the reference to the trigger that
1804 * now references the variable.
1806 * Return: The VAR_REF field if successful, NULL if not
1808 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1809 struct hist_field *var_field,
1810 char *system, char *event_name)
1812 unsigned long flags = HIST_FIELD_FL_VAR_REF;
1813 struct hist_field *ref_field;
1816 /* Check if the variable already exists */
1817 for (i = 0; i < hist_data->n_var_refs; i++) {
1818 ref_field = hist_data->var_refs[i];
1819 if (ref_field->var.idx == var_field->var.idx &&
1820 ref_field->var.hist_data == var_field->hist_data) {
1821 get_hist_field(ref_field);
1826 ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1828 if (init_var_ref(ref_field, var_field, system, event_name)) {
1829 destroy_hist_field(ref_field, 0);
1833 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1834 ref_field->var_ref_idx = hist_data->n_var_refs++;
1840 static bool is_var_ref(char *var_name)
1842 if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1848 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1854 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1855 name = hist_data->attrs->var_defs.name[i];
1857 if (strcmp(var_name, name) == 0) {
1858 field = hist_data->attrs->var_defs.expr[i];
1859 if (contains_operator(field) || is_var_ref(field))
1868 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1869 char *system, char *event_name,
1872 struct trace_event_call *call;
1874 if (system && event_name) {
1875 call = hist_data->event_file->event_call;
1877 if (strcmp(system, call->class->system) != 0)
1880 if (strcmp(event_name, trace_event_name(call)) != 0)
1884 if (!!system != !!event_name)
1887 if (!is_var_ref(var_name))
1892 return field_name_from_var(hist_data, var_name);
1895 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1896 char *system, char *event_name,
1899 struct hist_field *var_field = NULL, *ref_field = NULL;
1900 struct trace_array *tr = hist_data->event_file->tr;
1902 if (!is_var_ref(var_name))
1907 var_field = find_event_var(hist_data, system, event_name, var_name);
1909 ref_field = create_var_ref(hist_data, var_field,
1910 system, event_name);
1913 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1918 static struct ftrace_event_field *
1919 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1920 char *field_str, unsigned long *flags)
1922 struct ftrace_event_field *field = NULL;
1923 char *field_name, *modifier, *str;
1924 struct trace_array *tr = file->tr;
1926 modifier = str = kstrdup(field_str, GFP_KERNEL);
1928 return ERR_PTR(-ENOMEM);
1930 field_name = strsep(&modifier, ".");
1932 if (strcmp(modifier, "hex") == 0)
1933 *flags |= HIST_FIELD_FL_HEX;
1934 else if (strcmp(modifier, "sym") == 0)
1935 *flags |= HIST_FIELD_FL_SYM;
1936 else if (strcmp(modifier, "sym-offset") == 0)
1937 *flags |= HIST_FIELD_FL_SYM_OFFSET;
1938 else if ((strcmp(modifier, "execname") == 0) &&
1939 (strcmp(field_name, "common_pid") == 0))
1940 *flags |= HIST_FIELD_FL_EXECNAME;
1941 else if (strcmp(modifier, "syscall") == 0)
1942 *flags |= HIST_FIELD_FL_SYSCALL;
1943 else if (strcmp(modifier, "log2") == 0)
1944 *flags |= HIST_FIELD_FL_LOG2;
1945 else if (strcmp(modifier, "usecs") == 0)
1946 *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1948 hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1949 field = ERR_PTR(-EINVAL);
1954 if (strcmp(field_name, "common_timestamp") == 0) {
1955 *flags |= HIST_FIELD_FL_TIMESTAMP;
1956 hist_data->enable_timestamps = true;
1957 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1958 hist_data->attrs->ts_in_usecs = true;
1959 } else if (strcmp(field_name, "cpu") == 0)
1960 *flags |= HIST_FIELD_FL_CPU;
1962 field = trace_find_event_field(file->event_call, field_name);
1963 if (!field || !field->size) {
1964 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND, errpos(field_name));
1965 field = ERR_PTR(-EINVAL);
1975 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
1976 struct hist_field *var_ref,
1979 struct hist_field *alias = NULL;
1980 unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
1982 alias = create_hist_field(hist_data, NULL, flags, var_name);
1986 alias->fn = var_ref->fn;
1987 alias->operands[0] = var_ref;
1989 if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
1990 destroy_hist_field(alias, 0);
1994 alias->var_ref_idx = var_ref->var_ref_idx;
1999 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2000 struct trace_event_file *file, char *str,
2001 unsigned long *flags, char *var_name)
2003 char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2004 struct ftrace_event_field *field = NULL;
2005 struct hist_field *hist_field = NULL;
2008 s = strchr(str, '.');
2010 s = strchr(++s, '.');
2012 ref_system = strsep(&str, ".");
2017 ref_event = strsep(&str, ".");
2026 s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2028 hist_field = parse_var_ref(hist_data, ref_system,
2029 ref_event, ref_var);
2032 hist_field = create_alias(hist_data, hist_field, var_name);
2043 field = parse_field(hist_data, file, str, flags);
2044 if (IS_ERR(field)) {
2045 ret = PTR_ERR(field);
2049 hist_field = create_hist_field(hist_data, field, *flags, var_name);
2057 return ERR_PTR(ret);
2060 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2061 struct trace_event_file *file,
2062 char *str, unsigned long flags,
2063 char *var_name, unsigned int level);
2065 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2066 struct trace_event_file *file,
2067 char *str, unsigned long flags,
2068 char *var_name, unsigned int level)
2070 struct hist_field *operand1, *expr = NULL;
2071 unsigned long operand_flags;
2075 /* we support only -(xxx) i.e. explicit parens required */
2078 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2083 str++; /* skip leading '-' */
2085 s = strchr(str, '(');
2093 s = strrchr(str, ')');
2097 ret = -EINVAL; /* no closing ')' */
2101 flags |= HIST_FIELD_FL_EXPR;
2102 expr = create_hist_field(hist_data, NULL, flags, var_name);
2109 operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2110 if (IS_ERR(operand1)) {
2111 ret = PTR_ERR(operand1);
2115 expr->flags |= operand1->flags &
2116 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2117 expr->fn = hist_field_unary_minus;
2118 expr->operands[0] = operand1;
2119 expr->operator = FIELD_OP_UNARY_MINUS;
2120 expr->name = expr_str(expr, 0);
2121 expr->type = kstrdup(operand1->type, GFP_KERNEL);
2129 destroy_hist_field(expr, 0);
2130 return ERR_PTR(ret);
2133 static int check_expr_operands(struct trace_array *tr,
2134 struct hist_field *operand1,
2135 struct hist_field *operand2)
2137 unsigned long operand1_flags = operand1->flags;
2138 unsigned long operand2_flags = operand2->flags;
2140 if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2141 (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2142 struct hist_field *var;
2144 var = find_var_field(operand1->var.hist_data, operand1->name);
2147 operand1_flags = var->flags;
2150 if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2151 (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2152 struct hist_field *var;
2154 var = find_var_field(operand2->var.hist_data, operand2->name);
2157 operand2_flags = var->flags;
2160 if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2161 (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2162 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2169 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2170 struct trace_event_file *file,
2171 char *str, unsigned long flags,
2172 char *var_name, unsigned int level)
2174 struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2175 unsigned long operand_flags;
2176 int field_op, ret = -EINVAL;
2177 char *sep, *operand1_str;
2180 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2181 return ERR_PTR(-EINVAL);
2184 field_op = contains_operator(str);
2186 if (field_op == FIELD_OP_NONE)
2187 return parse_atom(hist_data, file, str, &flags, var_name);
2189 if (field_op == FIELD_OP_UNARY_MINUS)
2190 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2193 case FIELD_OP_MINUS:
2203 operand1_str = strsep(&str, sep);
2204 if (!operand1_str || !str)
2208 operand1 = parse_atom(hist_data, file, operand1_str,
2209 &operand_flags, NULL);
2210 if (IS_ERR(operand1)) {
2211 ret = PTR_ERR(operand1);
2216 /* rest of string could be another expression e.g. b+c in a+b+c */
2218 operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2219 if (IS_ERR(operand2)) {
2220 ret = PTR_ERR(operand2);
2225 ret = check_expr_operands(file->tr, operand1, operand2);
2229 flags |= HIST_FIELD_FL_EXPR;
2231 flags |= operand1->flags &
2232 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2234 expr = create_hist_field(hist_data, NULL, flags, var_name);
2240 operand1->read_once = true;
2241 operand2->read_once = true;
2243 expr->operands[0] = operand1;
2244 expr->operands[1] = operand2;
2245 expr->operator = field_op;
2246 expr->name = expr_str(expr, 0);
2247 expr->type = kstrdup(operand1->type, GFP_KERNEL);
2254 case FIELD_OP_MINUS:
2255 expr->fn = hist_field_minus;
2258 expr->fn = hist_field_plus;
2267 destroy_hist_field(operand1, 0);
2268 destroy_hist_field(operand2, 0);
2269 destroy_hist_field(expr, 0);
2271 return ERR_PTR(ret);
2274 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2275 struct trace_event_file *file)
2277 struct event_trigger_data *test;
2279 lockdep_assert_held(&event_mutex);
2281 list_for_each_entry(test, &file->triggers, list) {
2282 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2283 if (test->private_data == hist_data)
2284 return test->filter_str;
2291 static struct event_command trigger_hist_cmd;
2292 static int event_hist_trigger_func(struct event_command *cmd_ops,
2293 struct trace_event_file *file,
2294 char *glob, char *cmd, char *param);
2296 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2297 struct hist_trigger_data *hist_data,
2298 unsigned int n_keys)
2300 struct hist_field *target_hist_field, *hist_field;
2301 unsigned int n, i, j;
2303 if (hist_data->n_fields - hist_data->n_vals != n_keys)
2306 i = hist_data->n_vals;
2307 j = target_hist_data->n_vals;
2309 for (n = 0; n < n_keys; n++) {
2310 hist_field = hist_data->fields[i + n];
2311 target_hist_field = target_hist_data->fields[j + n];
2313 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2315 if (hist_field->size != target_hist_field->size)
2317 if (hist_field->is_signed != target_hist_field->is_signed)
2324 static struct hist_trigger_data *
2325 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2326 struct trace_event_file *file)
2328 struct hist_trigger_data *hist_data;
2329 struct event_trigger_data *test;
2330 unsigned int n_keys;
2332 lockdep_assert_held(&event_mutex);
2334 n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2336 list_for_each_entry(test, &file->triggers, list) {
2337 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2338 hist_data = test->private_data;
2340 if (compatible_keys(target_hist_data, hist_data, n_keys))
2348 static struct trace_event_file *event_file(struct trace_array *tr,
2349 char *system, char *event_name)
2351 struct trace_event_file *file;
2353 file = __find_event_file(tr, system, event_name);
2355 return ERR_PTR(-EINVAL);
2360 static struct hist_field *
2361 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2362 char *system, char *event_name, char *field_name)
2364 struct hist_field *event_var;
2365 char *synthetic_name;
2367 synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2368 if (!synthetic_name)
2369 return ERR_PTR(-ENOMEM);
2371 strcpy(synthetic_name, "synthetic_");
2372 strcat(synthetic_name, field_name);
2374 event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2376 kfree(synthetic_name);
2382 * create_field_var_hist - Automatically create a histogram and var for a field
2383 * @target_hist_data: The target hist trigger
2384 * @subsys_name: Optional subsystem name
2385 * @event_name: Optional event name
2386 * @field_name: The name of the field (and the resulting variable)
2388 * Hist trigger actions fetch data from variables, not directly from
2389 * events. However, for convenience, users are allowed to directly
2390 * specify an event field in an action, which will be automatically
2391 * converted into a variable on their behalf.
2393 * If a user specifies a field on an event that isn't the event the
2394 * histogram currently being defined (the target event histogram), the
2395 * only way that can be accomplished is if a new hist trigger is
2396 * created and the field variable defined on that.
2398 * This function creates a new histogram compatible with the target
2399 * event (meaning a histogram with the same key as the target
2400 * histogram), and creates a variable for the specified field, but
2401 * with 'synthetic_' prepended to the variable name in order to avoid
2402 * collision with normal field variables.
2404 * Return: The variable created for the field.
2406 static struct hist_field *
2407 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2408 char *subsys_name, char *event_name, char *field_name)
2410 struct trace_array *tr = target_hist_data->event_file->tr;
2411 struct hist_field *event_var = ERR_PTR(-EINVAL);
2412 struct hist_trigger_data *hist_data;
2413 unsigned int i, n, first = true;
2414 struct field_var_hist *var_hist;
2415 struct trace_event_file *file;
2416 struct hist_field *key_field;
2421 if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2422 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2423 return ERR_PTR(-EINVAL);
2426 file = event_file(tr, subsys_name, event_name);
2429 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2430 ret = PTR_ERR(file);
2431 return ERR_PTR(ret);
2435 * Look for a histogram compatible with target. We'll use the
2436 * found histogram specification to create a new matching
2437 * histogram with our variable on it. target_hist_data is not
2438 * yet a registered histogram so we can't use that.
2440 hist_data = find_compatible_hist(target_hist_data, file);
2442 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2443 return ERR_PTR(-EINVAL);
2446 /* See if a synthetic field variable has already been created */
2447 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2448 event_name, field_name);
2449 if (!IS_ERR_OR_NULL(event_var))
2452 var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2454 return ERR_PTR(-ENOMEM);
2456 cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2459 return ERR_PTR(-ENOMEM);
2462 /* Use the same keys as the compatible histogram */
2463 strcat(cmd, "keys=");
2465 for_each_hist_key_field(i, hist_data) {
2466 key_field = hist_data->fields[i];
2469 strcat(cmd, key_field->field->name);
2473 /* Create the synthetic field variable specification */
2474 strcat(cmd, ":synthetic_");
2475 strcat(cmd, field_name);
2477 strcat(cmd, field_name);
2479 /* Use the same filter as the compatible histogram */
2480 saved_filter = find_trigger_filter(hist_data, file);
2482 strcat(cmd, " if ");
2483 strcat(cmd, saved_filter);
2486 var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2487 if (!var_hist->cmd) {
2490 return ERR_PTR(-ENOMEM);
2493 /* Save the compatible histogram information */
2494 var_hist->hist_data = hist_data;
2496 /* Create the new histogram with our variable */
2497 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2501 kfree(var_hist->cmd);
2503 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2504 return ERR_PTR(ret);
2509 /* If we can't find the variable, something went wrong */
2510 event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2511 event_name, field_name);
2512 if (IS_ERR_OR_NULL(event_var)) {
2513 kfree(var_hist->cmd);
2515 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2516 return ERR_PTR(-EINVAL);
2519 n = target_hist_data->n_field_var_hists;
2520 target_hist_data->field_var_hists[n] = var_hist;
2521 target_hist_data->n_field_var_hists++;
2526 static struct hist_field *
2527 find_target_event_var(struct hist_trigger_data *hist_data,
2528 char *subsys_name, char *event_name, char *var_name)
2530 struct trace_event_file *file = hist_data->event_file;
2531 struct hist_field *hist_field = NULL;
2534 struct trace_event_call *call;
2539 call = file->event_call;
2541 if (strcmp(subsys_name, call->class->system) != 0)
2544 if (strcmp(event_name, trace_event_name(call)) != 0)
2548 hist_field = find_var_field(hist_data, var_name);
2553 static inline void __update_field_vars(struct tracing_map_elt *elt,
2554 struct ring_buffer_event *rbe,
2556 struct field_var **field_vars,
2557 unsigned int n_field_vars,
2558 unsigned int field_var_str_start)
2560 struct hist_elt_data *elt_data = elt->private_data;
2561 unsigned int i, j, var_idx;
2564 for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2565 struct field_var *field_var = field_vars[i];
2566 struct hist_field *var = field_var->var;
2567 struct hist_field *val = field_var->val;
2569 var_val = val->fn(val, elt, rbe, rec);
2570 var_idx = var->var.idx;
2572 if (val->flags & HIST_FIELD_FL_STRING) {
2573 char *str = elt_data->field_var_str[j++];
2574 char *val_str = (char *)(uintptr_t)var_val;
2576 strscpy(str, val_str, STR_VAR_LEN_MAX);
2577 var_val = (u64)(uintptr_t)str;
2579 tracing_map_set_var(elt, var_idx, var_val);
2583 static void update_field_vars(struct hist_trigger_data *hist_data,
2584 struct tracing_map_elt *elt,
2585 struct ring_buffer_event *rbe,
2588 __update_field_vars(elt, rbe, rec, hist_data->field_vars,
2589 hist_data->n_field_vars, 0);
2592 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2593 struct tracing_map_elt *elt, void *rec,
2594 struct ring_buffer_event *rbe, void *key,
2595 struct action_data *data, u64 *var_ref_vals)
2597 __update_field_vars(elt, rbe, rec, hist_data->save_vars,
2598 hist_data->n_save_vars, hist_data->n_field_var_str);
2601 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2602 struct trace_event_file *file,
2603 char *name, int size, const char *type)
2605 struct hist_field *var;
2608 if (find_var(hist_data, file, name) && !hist_data->remove) {
2609 var = ERR_PTR(-EINVAL);
2613 var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2615 var = ERR_PTR(-ENOMEM);
2619 idx = tracing_map_add_var(hist_data->map);
2622 var = ERR_PTR(-EINVAL);
2627 var->flags = HIST_FIELD_FL_VAR;
2629 var->var.hist_data = var->hist_data = hist_data;
2631 var->var.name = kstrdup(name, GFP_KERNEL);
2632 var->type = kstrdup(type, GFP_KERNEL);
2633 if (!var->var.name || !var->type) {
2634 kfree(var->var.name);
2637 var = ERR_PTR(-ENOMEM);
2643 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2644 struct trace_event_file *file,
2647 struct hist_field *val = NULL, *var = NULL;
2648 unsigned long flags = HIST_FIELD_FL_VAR;
2649 struct trace_array *tr = file->tr;
2650 struct field_var *field_var;
2653 if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2654 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2659 val = parse_atom(hist_data, file, field_name, &flags, NULL);
2661 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2666 var = create_var(hist_data, file, field_name, val->size, val->type);
2668 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2674 field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2682 field_var->var = var;
2683 field_var->val = val;
2687 field_var = ERR_PTR(ret);
2692 * create_target_field_var - Automatically create a variable for a field
2693 * @target_hist_data: The target hist trigger
2694 * @subsys_name: Optional subsystem name
2695 * @event_name: Optional event name
2696 * @var_name: The name of the field (and the resulting variable)
2698 * Hist trigger actions fetch data from variables, not directly from
2699 * events. However, for convenience, users are allowed to directly
2700 * specify an event field in an action, which will be automatically
2701 * converted into a variable on their behalf.
2703 * This function creates a field variable with the name var_name on
2704 * the hist trigger currently being defined on the target event. If
2705 * subsys_name and event_name are specified, this function simply
2706 * verifies that they do in fact match the target event subsystem and
2709 * Return: The variable created for the field.
2711 static struct field_var *
2712 create_target_field_var(struct hist_trigger_data *target_hist_data,
2713 char *subsys_name, char *event_name, char *var_name)
2715 struct trace_event_file *file = target_hist_data->event_file;
2718 struct trace_event_call *call;
2723 call = file->event_call;
2725 if (strcmp(subsys_name, call->class->system) != 0)
2728 if (strcmp(event_name, trace_event_name(call)) != 0)
2732 return create_field_var(target_hist_data, file, var_name);
2735 static bool check_track_val_max(u64 track_val, u64 var_val)
2737 if (var_val <= track_val)
2743 static bool check_track_val_changed(u64 track_val, u64 var_val)
2745 if (var_val == track_val)
2751 static u64 get_track_val(struct hist_trigger_data *hist_data,
2752 struct tracing_map_elt *elt,
2753 struct action_data *data)
2755 unsigned int track_var_idx = data->track_data.track_var->var.idx;
2758 track_val = tracing_map_read_var(elt, track_var_idx);
2763 static void save_track_val(struct hist_trigger_data *hist_data,
2764 struct tracing_map_elt *elt,
2765 struct action_data *data, u64 var_val)
2767 unsigned int track_var_idx = data->track_data.track_var->var.idx;
2769 tracing_map_set_var(elt, track_var_idx, var_val);
2772 static void save_track_data(struct hist_trigger_data *hist_data,
2773 struct tracing_map_elt *elt, void *rec,
2774 struct ring_buffer_event *rbe, void *key,
2775 struct action_data *data, u64 *var_ref_vals)
2777 if (data->track_data.save_data)
2778 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2781 static bool check_track_val(struct tracing_map_elt *elt,
2782 struct action_data *data,
2785 struct hist_trigger_data *hist_data;
2788 hist_data = data->track_data.track_var->hist_data;
2789 track_val = get_track_val(hist_data, elt, data);
2791 return data->track_data.check_val(track_val, var_val);
2794 #ifdef CONFIG_TRACER_SNAPSHOT
2795 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2797 /* called with tr->max_lock held */
2798 struct track_data *track_data = tr->cond_snapshot->cond_data;
2799 struct hist_elt_data *elt_data, *track_elt_data;
2800 struct snapshot_context *context = cond_data;
2801 struct action_data *action;
2807 action = track_data->action_data;
2809 track_val = get_track_val(track_data->hist_data, context->elt,
2810 track_data->action_data);
2812 if (!action->track_data.check_val(track_data->track_val, track_val))
2815 track_data->track_val = track_val;
2816 memcpy(track_data->key, context->key, track_data->key_len);
2818 elt_data = context->elt->private_data;
2819 track_elt_data = track_data->elt.private_data;
2821 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2823 track_data->updated = true;
2828 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2829 struct tracing_map_elt *elt, void *rec,
2830 struct ring_buffer_event *rbe, void *key,
2831 struct action_data *data,
2834 struct trace_event_file *file = hist_data->event_file;
2835 struct snapshot_context context;
2840 tracing_snapshot_cond(file->tr, &context);
2843 static void hist_trigger_print_key(struct seq_file *m,
2844 struct hist_trigger_data *hist_data,
2846 struct tracing_map_elt *elt);
2848 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2852 if (!hist_data->n_actions)
2855 for (i = 0; i < hist_data->n_actions; i++) {
2856 struct action_data *data = hist_data->actions[i];
2858 if (data->action == ACTION_SNAPSHOT)
2865 static void track_data_snapshot_print(struct seq_file *m,
2866 struct hist_trigger_data *hist_data)
2868 struct trace_event_file *file = hist_data->event_file;
2869 struct track_data *track_data;
2870 struct action_data *action;
2872 track_data = tracing_cond_snapshot_data(file->tr);
2876 if (!track_data->updated)
2879 action = snapshot_action(hist_data);
2883 seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n");
2884 seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2885 action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2886 action->track_data.var_str, track_data->track_val);
2888 seq_puts(m, "\ttriggered by event with key: ");
2889 hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2893 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2897 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2898 struct tracing_map_elt *elt, void *rec,
2899 struct ring_buffer_event *rbe, void *key,
2900 struct action_data *data,
2901 u64 *var_ref_vals) {}
2902 static void track_data_snapshot_print(struct seq_file *m,
2903 struct hist_trigger_data *hist_data) {}
2904 #endif /* CONFIG_TRACER_SNAPSHOT */
2906 static void track_data_print(struct seq_file *m,
2907 struct hist_trigger_data *hist_data,
2908 struct tracing_map_elt *elt,
2909 struct action_data *data)
2911 u64 track_val = get_track_val(hist_data, elt, data);
2912 unsigned int i, save_var_idx;
2914 if (data->handler == HANDLER_ONMAX)
2915 seq_printf(m, "\n\tmax: %10llu", track_val);
2916 else if (data->handler == HANDLER_ONCHANGE)
2917 seq_printf(m, "\n\tchanged: %10llu", track_val);
2919 if (data->action == ACTION_SNAPSHOT)
2922 for (i = 0; i < hist_data->n_save_vars; i++) {
2923 struct hist_field *save_val = hist_data->save_vars[i]->val;
2924 struct hist_field *save_var = hist_data->save_vars[i]->var;
2927 save_var_idx = save_var->var.idx;
2929 val = tracing_map_read_var(elt, save_var_idx);
2931 if (save_val->flags & HIST_FIELD_FL_STRING) {
2932 seq_printf(m, " %s: %-32s", save_var->var.name,
2933 (char *)(uintptr_t)(val));
2935 seq_printf(m, " %s: %10llu", save_var->var.name, val);
2939 static void ontrack_action(struct hist_trigger_data *hist_data,
2940 struct tracing_map_elt *elt, void *rec,
2941 struct ring_buffer_event *rbe, void *key,
2942 struct action_data *data, u64 *var_ref_vals)
2944 u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
2946 if (check_track_val(elt, data, var_val)) {
2947 save_track_val(hist_data, elt, data, var_val);
2948 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2952 static void action_data_destroy(struct action_data *data)
2956 lockdep_assert_held(&event_mutex);
2958 kfree(data->action_name);
2960 for (i = 0; i < data->n_params; i++)
2961 kfree(data->params[i]);
2963 if (data->synth_event)
2964 data->synth_event->ref--;
2966 kfree(data->synth_event_name);
2971 static void track_data_destroy(struct hist_trigger_data *hist_data,
2972 struct action_data *data)
2974 struct trace_event_file *file = hist_data->event_file;
2976 destroy_hist_field(data->track_data.track_var, 0);
2978 if (data->action == ACTION_SNAPSHOT) {
2979 struct track_data *track_data;
2981 track_data = tracing_cond_snapshot_data(file->tr);
2982 if (track_data && track_data->hist_data == hist_data) {
2983 tracing_snapshot_cond_disable(file->tr);
2984 track_data_free(track_data);
2988 kfree(data->track_data.var_str);
2990 action_data_destroy(data);
2993 static int action_create(struct hist_trigger_data *hist_data,
2994 struct action_data *data);
2996 static int track_data_create(struct hist_trigger_data *hist_data,
2997 struct action_data *data)
2999 struct hist_field *var_field, *ref_field, *track_var = NULL;
3000 struct trace_event_file *file = hist_data->event_file;
3001 struct trace_array *tr = file->tr;
3002 char *track_data_var_str;
3005 track_data_var_str = data->track_data.var_str;
3006 if (track_data_var_str[0] != '$') {
3007 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3010 track_data_var_str++;
3012 var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3014 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3018 ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3022 data->track_data.var_ref = ref_field;
3024 if (data->handler == HANDLER_ONMAX)
3025 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3026 if (IS_ERR(track_var)) {
3027 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3028 ret = PTR_ERR(track_var);
3032 if (data->handler == HANDLER_ONCHANGE)
3033 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3034 if (IS_ERR(track_var)) {
3035 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3036 ret = PTR_ERR(track_var);
3039 data->track_data.track_var = track_var;
3041 ret = action_create(hist_data, data);
3046 static int parse_action_params(struct trace_array *tr, char *params,
3047 struct action_data *data)
3049 char *param, *saved_param;
3050 bool first_param = true;
3054 if (data->n_params >= SYNTH_FIELDS_MAX) {
3055 hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3059 param = strsep(¶ms, ",");
3061 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3066 param = strstrip(param);
3067 if (strlen(param) < 2) {
3068 hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3073 saved_param = kstrdup(param, GFP_KERNEL);
3079 if (first_param && data->use_trace_keyword) {
3080 data->synth_event_name = saved_param;
3081 first_param = false;
3084 first_param = false;
3086 data->params[data->n_params++] = saved_param;
3092 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3093 enum handler_id handler)
3100 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3105 action_name = strsep(&str, "(");
3106 if (!action_name || !str) {
3107 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3112 if (str_has_prefix(action_name, "save")) {
3113 char *params = strsep(&str, ")");
3116 hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3121 ret = parse_action_params(tr, params, data);
3125 if (handler == HANDLER_ONMAX)
3126 data->track_data.check_val = check_track_val_max;
3127 else if (handler == HANDLER_ONCHANGE)
3128 data->track_data.check_val = check_track_val_changed;
3130 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3135 data->track_data.save_data = save_track_data_vars;
3136 data->fn = ontrack_action;
3137 data->action = ACTION_SAVE;
3138 } else if (str_has_prefix(action_name, "snapshot")) {
3139 char *params = strsep(&str, ")");
3142 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3147 if (handler == HANDLER_ONMAX)
3148 data->track_data.check_val = check_track_val_max;
3149 else if (handler == HANDLER_ONCHANGE)
3150 data->track_data.check_val = check_track_val_changed;
3152 hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3157 data->track_data.save_data = save_track_data_snapshot;
3158 data->fn = ontrack_action;
3159 data->action = ACTION_SNAPSHOT;
3161 char *params = strsep(&str, ")");
3163 if (str_has_prefix(action_name, "trace"))
3164 data->use_trace_keyword = true;
3167 ret = parse_action_params(tr, params, data);
3172 if (handler == HANDLER_ONMAX)
3173 data->track_data.check_val = check_track_val_max;
3174 else if (handler == HANDLER_ONCHANGE)
3175 data->track_data.check_val = check_track_val_changed;
3177 if (handler != HANDLER_ONMATCH) {
3178 data->track_data.save_data = action_trace;
3179 data->fn = ontrack_action;
3181 data->fn = action_trace;
3183 data->action = ACTION_TRACE;
3186 data->action_name = kstrdup(action_name, GFP_KERNEL);
3187 if (!data->action_name) {
3192 data->handler = handler;
3197 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3198 char *str, enum handler_id handler)
3200 struct action_data *data;
3204 data = kzalloc(sizeof(*data), GFP_KERNEL);
3206 return ERR_PTR(-ENOMEM);
3208 var_str = strsep(&str, ")");
3209 if (!var_str || !str) {
3214 data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3215 if (!data->track_data.var_str) {
3220 ret = action_parse(hist_data->event_file->tr, str, data, handler);
3226 track_data_destroy(hist_data, data);
3227 data = ERR_PTR(ret);
3231 static void onmatch_destroy(struct action_data *data)
3233 kfree(data->match_data.event);
3234 kfree(data->match_data.event_system);
3236 action_data_destroy(data);
3239 static void destroy_field_var(struct field_var *field_var)
3244 destroy_hist_field(field_var->var, 0);
3245 destroy_hist_field(field_var->val, 0);
3250 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3254 for (i = 0; i < hist_data->n_field_vars; i++)
3255 destroy_field_var(hist_data->field_vars[i]);
3257 for (i = 0; i < hist_data->n_save_vars; i++)
3258 destroy_field_var(hist_data->save_vars[i]);
3261 static void save_field_var(struct hist_trigger_data *hist_data,
3262 struct field_var *field_var)
3264 hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3266 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3267 hist_data->n_field_var_str++;
3271 static int check_synth_field(struct synth_event *event,
3272 struct hist_field *hist_field,
3273 unsigned int field_pos)
3275 struct synth_field *field;
3277 if (field_pos >= event->n_fields)
3280 field = event->fields[field_pos];
3282 if (strcmp(field->type, hist_field->type) != 0) {
3283 if (field->size != hist_field->size ||
3284 field->is_signed != hist_field->is_signed)
3291 static struct hist_field *
3292 trace_action_find_var(struct hist_trigger_data *hist_data,
3293 struct action_data *data,
3294 char *system, char *event, char *var)
3296 struct trace_array *tr = hist_data->event_file->tr;
3297 struct hist_field *hist_field;
3299 var++; /* skip '$' */
3301 hist_field = find_target_event_var(hist_data, system, event, var);
3303 if (!system && data->handler == HANDLER_ONMATCH) {
3304 system = data->match_data.event_system;
3305 event = data->match_data.event;
3308 hist_field = find_event_var(hist_data, system, event, var);
3312 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3317 static struct hist_field *
3318 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3319 struct action_data *data, char *system,
3320 char *event, char *var)
3322 struct hist_field *hist_field = NULL;
3323 struct field_var *field_var;
3326 * First try to create a field var on the target event (the
3327 * currently being defined). This will create a variable for
3328 * unqualified fields on the target event, or if qualified,
3329 * target fields that have qualified names matching the target.
3331 field_var = create_target_field_var(hist_data, system, event, var);
3333 if (field_var && !IS_ERR(field_var)) {
3334 save_field_var(hist_data, field_var);
3335 hist_field = field_var->var;
3339 * If no explicit system.event is specfied, default to
3340 * looking for fields on the onmatch(system.event.xxx)
3343 if (!system && data->handler == HANDLER_ONMATCH) {
3344 system = data->match_data.event_system;
3345 event = data->match_data.event;
3349 * At this point, we're looking at a field on another
3350 * event. Because we can't modify a hist trigger on
3351 * another event to add a variable for a field, we need
3352 * to create a new trigger on that event and create the
3353 * variable at the same time.
3355 hist_field = create_field_var_hist(hist_data, system, event, var);
3356 if (IS_ERR(hist_field))
3362 destroy_field_var(field_var);
3367 static int trace_action_create(struct hist_trigger_data *hist_data,
3368 struct action_data *data)
3370 struct trace_array *tr = hist_data->event_file->tr;
3371 char *event_name, *param, *system = NULL;
3372 struct hist_field *hist_field, *var_ref;
3374 unsigned int field_pos = 0;
3375 struct synth_event *event;
3376 char *synth_event_name;
3377 int var_ref_idx, ret = 0;
3379 lockdep_assert_held(&event_mutex);
3381 if (data->use_trace_keyword)
3382 synth_event_name = data->synth_event_name;
3384 synth_event_name = data->action_name;
3386 event = find_synth_event(synth_event_name);
3388 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3394 for (i = 0; i < data->n_params; i++) {
3397 p = param = kstrdup(data->params[i], GFP_KERNEL);
3403 system = strsep(¶m, ".");
3405 param = (char *)system;
3406 system = event_name = NULL;
3408 event_name = strsep(¶m, ".");
3416 if (param[0] == '$')
3417 hist_field = trace_action_find_var(hist_data, data,
3421 hist_field = trace_action_create_field_var(hist_data,
3433 if (check_synth_field(event, hist_field, field_pos) == 0) {
3434 var_ref = create_var_ref(hist_data, hist_field,
3435 system, event_name);
3442 var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3443 if (WARN_ON(var_ref_idx < 0)) {
3448 data->var_ref_idx[i] = var_ref_idx;
3455 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3461 if (field_pos != event->n_fields) {
3462 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3467 data->synth_event = event;
3476 static int action_create(struct hist_trigger_data *hist_data,
3477 struct action_data *data)
3479 struct trace_event_file *file = hist_data->event_file;
3480 struct trace_array *tr = file->tr;
3481 struct track_data *track_data;
3482 struct field_var *field_var;
3487 if (data->action == ACTION_TRACE)
3488 return trace_action_create(hist_data, data);
3490 if (data->action == ACTION_SNAPSHOT) {
3491 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3492 if (IS_ERR(track_data)) {
3493 ret = PTR_ERR(track_data);
3497 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3498 cond_snapshot_update);
3500 track_data_free(track_data);
3505 if (data->action == ACTION_SAVE) {
3506 if (hist_data->n_save_vars) {
3508 hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3512 for (i = 0; i < data->n_params; i++) {
3513 param = kstrdup(data->params[i], GFP_KERNEL);
3519 field_var = create_target_field_var(hist_data, NULL, NULL, param);
3520 if (IS_ERR(field_var)) {
3521 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3523 ret = PTR_ERR(field_var);
3528 hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3529 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3530 hist_data->n_save_var_str++;
3538 static int onmatch_create(struct hist_trigger_data *hist_data,
3539 struct action_data *data)
3541 return action_create(hist_data, data);
3544 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3546 char *match_event, *match_event_system;
3547 struct action_data *data;
3550 data = kzalloc(sizeof(*data), GFP_KERNEL);
3552 return ERR_PTR(-ENOMEM);
3554 match_event = strsep(&str, ")");
3555 if (!match_event || !str) {
3556 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3560 match_event_system = strsep(&match_event, ".");
3562 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3566 if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3567 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3571 data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3572 if (!data->match_data.event) {
3577 data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3578 if (!data->match_data.event_system) {
3583 ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3589 onmatch_destroy(data);
3590 data = ERR_PTR(ret);
3594 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3596 hist_data->fields[HITCOUNT_IDX] =
3597 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3598 if (!hist_data->fields[HITCOUNT_IDX])
3601 hist_data->n_vals++;
3602 hist_data->n_fields++;
3604 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3610 static int __create_val_field(struct hist_trigger_data *hist_data,
3611 unsigned int val_idx,
3612 struct trace_event_file *file,
3613 char *var_name, char *field_str,
3614 unsigned long flags)
3616 struct hist_field *hist_field;
3619 hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3620 if (IS_ERR(hist_field)) {
3621 ret = PTR_ERR(hist_field);
3625 hist_data->fields[val_idx] = hist_field;
3627 ++hist_data->n_vals;
3628 ++hist_data->n_fields;
3630 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3636 static int create_val_field(struct hist_trigger_data *hist_data,
3637 unsigned int val_idx,
3638 struct trace_event_file *file,
3641 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3644 return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3647 static int create_var_field(struct hist_trigger_data *hist_data,
3648 unsigned int val_idx,
3649 struct trace_event_file *file,
3650 char *var_name, char *expr_str)
3652 struct trace_array *tr = hist_data->event_file->tr;
3653 unsigned long flags = 0;
3655 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3658 if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3659 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3663 flags |= HIST_FIELD_FL_VAR;
3664 hist_data->n_vars++;
3665 if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3668 return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3671 static int create_val_fields(struct hist_trigger_data *hist_data,
3672 struct trace_event_file *file)
3674 char *fields_str, *field_str;
3675 unsigned int i, j = 1;
3678 ret = create_hitcount_val(hist_data);
3682 fields_str = hist_data->attrs->vals_str;
3686 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3687 j < TRACING_MAP_VALS_MAX; i++) {
3688 field_str = strsep(&fields_str, ",");
3692 if (strcmp(field_str, "hitcount") == 0)
3695 ret = create_val_field(hist_data, j++, file, field_str);
3700 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3706 static int create_key_field(struct hist_trigger_data *hist_data,
3707 unsigned int key_idx,
3708 unsigned int key_offset,
3709 struct trace_event_file *file,
3712 struct trace_array *tr = hist_data->event_file->tr;
3713 struct hist_field *hist_field = NULL;
3714 unsigned long flags = 0;
3715 unsigned int key_size;
3718 if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3721 flags |= HIST_FIELD_FL_KEY;
3723 if (strcmp(field_str, "stacktrace") == 0) {
3724 flags |= HIST_FIELD_FL_STACKTRACE;
3725 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3726 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3728 hist_field = parse_expr(hist_data, file, field_str, flags,
3730 if (IS_ERR(hist_field)) {
3731 ret = PTR_ERR(hist_field);
3735 if (field_has_hist_vars(hist_field, 0)) {
3736 hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3737 destroy_hist_field(hist_field, 0);
3742 key_size = hist_field->size;
3745 hist_data->fields[key_idx] = hist_field;
3747 key_size = ALIGN(key_size, sizeof(u64));
3748 hist_data->fields[key_idx]->size = key_size;
3749 hist_data->fields[key_idx]->offset = key_offset;
3751 hist_data->key_size += key_size;
3753 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3758 hist_data->n_keys++;
3759 hist_data->n_fields++;
3761 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3769 static int create_key_fields(struct hist_trigger_data *hist_data,
3770 struct trace_event_file *file)
3772 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3773 char *fields_str, *field_str;
3776 fields_str = hist_data->attrs->keys_str;
3780 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3781 field_str = strsep(&fields_str, ",");
3784 ret = create_key_field(hist_data, i, key_offset,
3799 static int create_var_fields(struct hist_trigger_data *hist_data,
3800 struct trace_event_file *file)
3802 unsigned int i, j = hist_data->n_vals;
3805 unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3807 for (i = 0; i < n_vars; i++) {
3808 char *var_name = hist_data->attrs->var_defs.name[i];
3809 char *expr = hist_data->attrs->var_defs.expr[i];
3811 ret = create_var_field(hist_data, j++, file, var_name, expr);
3819 static void free_var_defs(struct hist_trigger_data *hist_data)
3823 for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3824 kfree(hist_data->attrs->var_defs.name[i]);
3825 kfree(hist_data->attrs->var_defs.expr[i]);
3828 hist_data->attrs->var_defs.n_vars = 0;
3831 static int parse_var_defs(struct hist_trigger_data *hist_data)
3833 struct trace_array *tr = hist_data->event_file->tr;
3834 char *s, *str, *var_name, *field_str;
3835 unsigned int i, j, n_vars = 0;
3838 for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3839 str = hist_data->attrs->assignment_str[i];
3840 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3841 field_str = strsep(&str, ",");
3845 var_name = strsep(&field_str, "=");
3846 if (!var_name || !field_str) {
3847 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3853 if (n_vars == TRACING_MAP_VARS_MAX) {
3854 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3859 s = kstrdup(var_name, GFP_KERNEL);
3864 hist_data->attrs->var_defs.name[n_vars] = s;
3866 s = kstrdup(field_str, GFP_KERNEL);
3868 kfree(hist_data->attrs->var_defs.name[n_vars]);
3872 hist_data->attrs->var_defs.expr[n_vars++] = s;
3874 hist_data->attrs->var_defs.n_vars = n_vars;
3880 free_var_defs(hist_data);
3885 static int create_hist_fields(struct hist_trigger_data *hist_data,
3886 struct trace_event_file *file)
3890 ret = parse_var_defs(hist_data);
3894 ret = create_val_fields(hist_data, file);
3898 ret = create_var_fields(hist_data, file);
3902 ret = create_key_fields(hist_data, file);
3906 free_var_defs(hist_data);
3911 static int is_descending(struct trace_array *tr, const char *str)
3916 if (strcmp(str, "descending") == 0)
3919 if (strcmp(str, "ascending") == 0)
3922 hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
3927 static int create_sort_keys(struct hist_trigger_data *hist_data)
3929 struct trace_array *tr = hist_data->event_file->tr;
3930 char *fields_str = hist_data->attrs->sort_key_str;
3931 struct tracing_map_sort_key *sort_key;
3932 int descending, ret = 0;
3933 unsigned int i, j, k;
3935 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
3940 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
3941 struct hist_field *hist_field;
3942 char *field_str, *field_name;
3943 const char *test_name;
3945 sort_key = &hist_data->sort_keys[i];
3947 field_str = strsep(&fields_str, ",");
3953 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3957 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
3958 hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
3963 field_name = strsep(&field_str, ".");
3964 if (!field_name || !*field_name) {
3966 hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
3970 if (strcmp(field_name, "hitcount") == 0) {
3971 descending = is_descending(tr, field_str);
3972 if (descending < 0) {
3976 sort_key->descending = descending;
3980 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
3983 hist_field = hist_data->fields[j];
3984 if (hist_field->flags & HIST_FIELD_FL_VAR)
3989 test_name = hist_field_name(hist_field, 0);
3991 if (strcmp(field_name, test_name) == 0) {
3992 sort_key->field_idx = idx;
3993 descending = is_descending(tr, field_str);
3994 if (descending < 0) {
3998 sort_key->descending = descending;
4002 if (j == hist_data->n_fields) {
4004 hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4009 hist_data->n_sort_keys = i;
4014 static void destroy_actions(struct hist_trigger_data *hist_data)
4018 for (i = 0; i < hist_data->n_actions; i++) {
4019 struct action_data *data = hist_data->actions[i];
4021 if (data->handler == HANDLER_ONMATCH)
4022 onmatch_destroy(data);
4023 else if (data->handler == HANDLER_ONMAX ||
4024 data->handler == HANDLER_ONCHANGE)
4025 track_data_destroy(hist_data, data);
4031 static int parse_actions(struct hist_trigger_data *hist_data)
4033 struct trace_array *tr = hist_data->event_file->tr;
4034 struct action_data *data;
4040 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4041 str = hist_data->attrs->action_str[i];
4043 if ((len = str_has_prefix(str, "onmatch("))) {
4044 char *action_str = str + len;
4046 data = onmatch_parse(tr, action_str);
4048 ret = PTR_ERR(data);
4051 } else if ((len = str_has_prefix(str, "onmax("))) {
4052 char *action_str = str + len;
4054 data = track_data_parse(hist_data, action_str,
4057 ret = PTR_ERR(data);
4060 } else if ((len = str_has_prefix(str, "onchange("))) {
4061 char *action_str = str + len;
4063 data = track_data_parse(hist_data, action_str,
4066 ret = PTR_ERR(data);
4074 hist_data->actions[hist_data->n_actions++] = data;
4080 static int create_actions(struct hist_trigger_data *hist_data)
4082 struct action_data *data;
4086 for (i = 0; i < hist_data->attrs->n_actions; i++) {
4087 data = hist_data->actions[i];
4089 if (data->handler == HANDLER_ONMATCH) {
4090 ret = onmatch_create(hist_data, data);
4093 } else if (data->handler == HANDLER_ONMAX ||
4094 data->handler == HANDLER_ONCHANGE) {
4095 ret = track_data_create(hist_data, data);
4107 static void print_actions(struct seq_file *m,
4108 struct hist_trigger_data *hist_data,
4109 struct tracing_map_elt *elt)
4113 for (i = 0; i < hist_data->n_actions; i++) {
4114 struct action_data *data = hist_data->actions[i];
4116 if (data->action == ACTION_SNAPSHOT)
4119 if (data->handler == HANDLER_ONMAX ||
4120 data->handler == HANDLER_ONCHANGE)
4121 track_data_print(m, hist_data, elt, data);
4125 static void print_action_spec(struct seq_file *m,
4126 struct hist_trigger_data *hist_data,
4127 struct action_data *data)
4131 if (data->action == ACTION_SAVE) {
4132 for (i = 0; i < hist_data->n_save_vars; i++) {
4133 seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4134 if (i < hist_data->n_save_vars - 1)
4137 } else if (data->action == ACTION_TRACE) {
4138 if (data->use_trace_keyword)
4139 seq_printf(m, "%s", data->synth_event_name);
4140 for (i = 0; i < data->n_params; i++) {
4141 if (i || data->use_trace_keyword)
4143 seq_printf(m, "%s", data->params[i]);
4148 static void print_track_data_spec(struct seq_file *m,
4149 struct hist_trigger_data *hist_data,
4150 struct action_data *data)
4152 if (data->handler == HANDLER_ONMAX)
4153 seq_puts(m, ":onmax(");
4154 else if (data->handler == HANDLER_ONCHANGE)
4155 seq_puts(m, ":onchange(");
4156 seq_printf(m, "%s", data->track_data.var_str);
4157 seq_printf(m, ").%s(", data->action_name);
4159 print_action_spec(m, hist_data, data);
4164 static void print_onmatch_spec(struct seq_file *m,
4165 struct hist_trigger_data *hist_data,
4166 struct action_data *data)
4168 seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4169 data->match_data.event);
4171 seq_printf(m, "%s(", data->action_name);
4173 print_action_spec(m, hist_data, data);
4178 static bool actions_match(struct hist_trigger_data *hist_data,
4179 struct hist_trigger_data *hist_data_test)
4183 if (hist_data->n_actions != hist_data_test->n_actions)
4186 for (i = 0; i < hist_data->n_actions; i++) {
4187 struct action_data *data = hist_data->actions[i];
4188 struct action_data *data_test = hist_data_test->actions[i];
4189 char *action_name, *action_name_test;
4191 if (data->handler != data_test->handler)
4193 if (data->action != data_test->action)
4196 if (data->n_params != data_test->n_params)
4199 for (j = 0; j < data->n_params; j++) {
4200 if (strcmp(data->params[j], data_test->params[j]) != 0)
4204 if (data->use_trace_keyword)
4205 action_name = data->synth_event_name;
4207 action_name = data->action_name;
4209 if (data_test->use_trace_keyword)
4210 action_name_test = data_test->synth_event_name;
4212 action_name_test = data_test->action_name;
4214 if (strcmp(action_name, action_name_test) != 0)
4217 if (data->handler == HANDLER_ONMATCH) {
4218 if (strcmp(data->match_data.event_system,
4219 data_test->match_data.event_system) != 0)
4221 if (strcmp(data->match_data.event,
4222 data_test->match_data.event) != 0)
4224 } else if (data->handler == HANDLER_ONMAX ||
4225 data->handler == HANDLER_ONCHANGE) {
4226 if (strcmp(data->track_data.var_str,
4227 data_test->track_data.var_str) != 0)
4236 static void print_actions_spec(struct seq_file *m,
4237 struct hist_trigger_data *hist_data)
4241 for (i = 0; i < hist_data->n_actions; i++) {
4242 struct action_data *data = hist_data->actions[i];
4244 if (data->handler == HANDLER_ONMATCH)
4245 print_onmatch_spec(m, hist_data, data);
4246 else if (data->handler == HANDLER_ONMAX ||
4247 data->handler == HANDLER_ONCHANGE)
4248 print_track_data_spec(m, hist_data, data);
4252 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4256 for (i = 0; i < hist_data->n_field_var_hists; i++) {
4257 kfree(hist_data->field_var_hists[i]->cmd);
4258 kfree(hist_data->field_var_hists[i]);
4262 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4267 destroy_hist_trigger_attrs(hist_data->attrs);
4268 destroy_hist_fields(hist_data);
4269 tracing_map_destroy(hist_data->map);
4271 destroy_actions(hist_data);
4272 destroy_field_vars(hist_data);
4273 destroy_field_var_hists(hist_data);
4278 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4280 struct tracing_map *map = hist_data->map;
4281 struct ftrace_event_field *field;
4282 struct hist_field *hist_field;
4285 for_each_hist_field(i, hist_data) {
4286 hist_field = hist_data->fields[i];
4287 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4288 tracing_map_cmp_fn_t cmp_fn;
4290 field = hist_field->field;
4292 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4293 cmp_fn = tracing_map_cmp_none;
4295 cmp_fn = tracing_map_cmp_num(hist_field->size,
4296 hist_field->is_signed);
4297 else if (is_string_field(field))
4298 cmp_fn = tracing_map_cmp_string;
4300 cmp_fn = tracing_map_cmp_num(field->size,
4302 idx = tracing_map_add_key_field(map,
4305 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4306 idx = tracing_map_add_sum_field(map);
4311 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4312 idx = tracing_map_add_var(map);
4315 hist_field->var.idx = idx;
4316 hist_field->var.hist_data = hist_data;
4323 static struct hist_trigger_data *
4324 create_hist_data(unsigned int map_bits,
4325 struct hist_trigger_attrs *attrs,
4326 struct trace_event_file *file,
4329 const struct tracing_map_ops *map_ops = NULL;
4330 struct hist_trigger_data *hist_data;
4333 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4335 return ERR_PTR(-ENOMEM);
4337 hist_data->attrs = attrs;
4338 hist_data->remove = remove;
4339 hist_data->event_file = file;
4341 ret = parse_actions(hist_data);
4345 ret = create_hist_fields(hist_data, file);
4349 ret = create_sort_keys(hist_data);
4353 map_ops = &hist_trigger_elt_data_ops;
4355 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4356 map_ops, hist_data);
4357 if (IS_ERR(hist_data->map)) {
4358 ret = PTR_ERR(hist_data->map);
4359 hist_data->map = NULL;
4363 ret = create_tracing_map_fields(hist_data);
4369 hist_data->attrs = NULL;
4371 destroy_hist_data(hist_data);
4373 hist_data = ERR_PTR(ret);
4378 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4379 struct tracing_map_elt *elt, void *rec,
4380 struct ring_buffer_event *rbe,
4383 struct hist_elt_data *elt_data;
4384 struct hist_field *hist_field;
4385 unsigned int i, var_idx;
4388 elt_data = elt->private_data;
4389 elt_data->var_ref_vals = var_ref_vals;
4391 for_each_hist_val_field(i, hist_data) {
4392 hist_field = hist_data->fields[i];
4393 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4394 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4395 var_idx = hist_field->var.idx;
4396 tracing_map_set_var(elt, var_idx, hist_val);
4399 tracing_map_update_sum(elt, i, hist_val);
4402 for_each_hist_key_field(i, hist_data) {
4403 hist_field = hist_data->fields[i];
4404 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4405 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4406 var_idx = hist_field->var.idx;
4407 tracing_map_set_var(elt, var_idx, hist_val);
4411 update_field_vars(hist_data, elt, rbe, rec);
4414 static inline void add_to_key(char *compound_key, void *key,
4415 struct hist_field *key_field, void *rec)
4417 size_t size = key_field->size;
4419 if (key_field->flags & HIST_FIELD_FL_STRING) {
4420 struct ftrace_event_field *field;
4422 field = key_field->field;
4423 if (field->filter_type == FILTER_DYN_STRING)
4424 size = *(u32 *)(rec + field->offset) >> 16;
4425 else if (field->filter_type == FILTER_PTR_STRING)
4427 else if (field->filter_type == FILTER_STATIC_STRING)
4430 /* ensure NULL-termination */
4431 if (size > key_field->size - 1)
4432 size = key_field->size - 1;
4434 strncpy(compound_key + key_field->offset, (char *)key, size);
4436 memcpy(compound_key + key_field->offset, key, size);
4440 hist_trigger_actions(struct hist_trigger_data *hist_data,
4441 struct tracing_map_elt *elt, void *rec,
4442 struct ring_buffer_event *rbe, void *key,
4445 struct action_data *data;
4448 for (i = 0; i < hist_data->n_actions; i++) {
4449 data = hist_data->actions[i];
4450 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4454 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4455 struct ring_buffer_event *rbe)
4457 struct hist_trigger_data *hist_data = data->private_data;
4458 bool use_compound_key = (hist_data->n_keys > 1);
4459 unsigned long entries[HIST_STACKTRACE_DEPTH];
4460 u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4461 char compound_key[HIST_KEY_SIZE_MAX];
4462 struct tracing_map_elt *elt = NULL;
4463 struct hist_field *key_field;
4468 memset(compound_key, 0, hist_data->key_size);
4470 for_each_hist_key_field(i, hist_data) {
4471 key_field = hist_data->fields[i];
4473 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4474 memset(entries, 0, HIST_STACKTRACE_SIZE);
4475 stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4476 HIST_STACKTRACE_SKIP);
4479 field_contents = key_field->fn(key_field, elt, rbe, rec);
4480 if (key_field->flags & HIST_FIELD_FL_STRING) {
4481 key = (void *)(unsigned long)field_contents;
4482 use_compound_key = true;
4484 key = (void *)&field_contents;
4487 if (use_compound_key)
4488 add_to_key(compound_key, key, key_field, rec);
4491 if (use_compound_key)
4494 if (hist_data->n_var_refs &&
4495 !resolve_var_refs(hist_data, key, var_ref_vals, false))
4498 elt = tracing_map_insert(hist_data->map, key);
4502 hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4504 if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4505 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
4508 static void hist_trigger_stacktrace_print(struct seq_file *m,
4509 unsigned long *stacktrace_entries,
4510 unsigned int max_entries)
4512 char str[KSYM_SYMBOL_LEN];
4513 unsigned int spaces = 8;
4516 for (i = 0; i < max_entries; i++) {
4517 if (!stacktrace_entries[i])
4520 seq_printf(m, "%*c", 1 + spaces, ' ');
4521 sprint_symbol(str, stacktrace_entries[i]);
4522 seq_printf(m, "%s\n", str);
4526 static void hist_trigger_print_key(struct seq_file *m,
4527 struct hist_trigger_data *hist_data,
4529 struct tracing_map_elt *elt)
4531 struct hist_field *key_field;
4532 char str[KSYM_SYMBOL_LEN];
4533 bool multiline = false;
4534 const char *field_name;
4540 for_each_hist_key_field(i, hist_data) {
4541 key_field = hist_data->fields[i];
4543 if (i > hist_data->n_vals)
4546 field_name = hist_field_name(key_field, 0);
4548 if (key_field->flags & HIST_FIELD_FL_HEX) {
4549 uval = *(u64 *)(key + key_field->offset);
4550 seq_printf(m, "%s: %llx", field_name, uval);
4551 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4552 uval = *(u64 *)(key + key_field->offset);
4553 sprint_symbol_no_offset(str, uval);
4554 seq_printf(m, "%s: [%llx] %-45s", field_name,
4556 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4557 uval = *(u64 *)(key + key_field->offset);
4558 sprint_symbol(str, uval);
4559 seq_printf(m, "%s: [%llx] %-55s", field_name,
4561 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4562 struct hist_elt_data *elt_data = elt->private_data;
4565 if (WARN_ON_ONCE(!elt_data))
4568 comm = elt_data->comm;
4570 uval = *(u64 *)(key + key_field->offset);
4571 seq_printf(m, "%s: %-16s[%10llu]", field_name,
4573 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4574 const char *syscall_name;
4576 uval = *(u64 *)(key + key_field->offset);
4577 syscall_name = get_syscall_name(uval);
4579 syscall_name = "unknown_syscall";
4581 seq_printf(m, "%s: %-30s[%3llu]", field_name,
4582 syscall_name, uval);
4583 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4584 seq_puts(m, "stacktrace:\n");
4585 hist_trigger_stacktrace_print(m,
4586 key + key_field->offset,
4587 HIST_STACKTRACE_DEPTH);
4589 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4590 seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4591 *(u64 *)(key + key_field->offset));
4592 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4593 seq_printf(m, "%s: %-50s", field_name,
4594 (char *)(key + key_field->offset));
4596 uval = *(u64 *)(key + key_field->offset);
4597 seq_printf(m, "%s: %10llu", field_name, uval);
4607 static void hist_trigger_entry_print(struct seq_file *m,
4608 struct hist_trigger_data *hist_data,
4610 struct tracing_map_elt *elt)
4612 const char *field_name;
4615 hist_trigger_print_key(m, hist_data, key, elt);
4617 seq_printf(m, " hitcount: %10llu",
4618 tracing_map_read_sum(elt, HITCOUNT_IDX));
4620 for (i = 1; i < hist_data->n_vals; i++) {
4621 field_name = hist_field_name(hist_data->fields[i], 0);
4623 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4624 hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4627 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4628 seq_printf(m, " %s: %10llx", field_name,
4629 tracing_map_read_sum(elt, i));
4631 seq_printf(m, " %s: %10llu", field_name,
4632 tracing_map_read_sum(elt, i));
4636 print_actions(m, hist_data, elt);
4641 static int print_entries(struct seq_file *m,
4642 struct hist_trigger_data *hist_data)
4644 struct tracing_map_sort_entry **sort_entries = NULL;
4645 struct tracing_map *map = hist_data->map;
4648 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4649 hist_data->n_sort_keys,
4654 for (i = 0; i < n_entries; i++)
4655 hist_trigger_entry_print(m, hist_data,
4656 sort_entries[i]->key,
4657 sort_entries[i]->elt);
4659 tracing_map_destroy_sort_entries(sort_entries, n_entries);
4664 static void hist_trigger_show(struct seq_file *m,
4665 struct event_trigger_data *data, int n)
4667 struct hist_trigger_data *hist_data;
4671 seq_puts(m, "\n\n");
4673 seq_puts(m, "# event histogram\n#\n# trigger info: ");
4674 data->ops->print(m, data->ops, data);
4675 seq_puts(m, "#\n\n");
4677 hist_data = data->private_data;
4678 n_entries = print_entries(m, hist_data);
4682 track_data_snapshot_print(m, hist_data);
4684 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
4685 (u64)atomic64_read(&hist_data->map->hits),
4686 n_entries, (u64)atomic64_read(&hist_data->map->drops));
4689 static int hist_show(struct seq_file *m, void *v)
4691 struct event_trigger_data *data;
4692 struct trace_event_file *event_file;
4695 mutex_lock(&event_mutex);
4697 event_file = event_file_data(m->private);
4698 if (unlikely(!event_file)) {
4703 list_for_each_entry(data, &event_file->triggers, list) {
4704 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4705 hist_trigger_show(m, data, n++);
4709 mutex_unlock(&event_mutex);
4714 static int event_hist_open(struct inode *inode, struct file *file)
4718 ret = security_locked_down(LOCKDOWN_TRACEFS);
4722 return single_open(file, hist_show, file);
4725 const struct file_operations event_hist_fops = {
4726 .open = event_hist_open,
4728 .llseek = seq_lseek,
4729 .release = single_release,
4732 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4733 static void hist_field_debug_show_flags(struct seq_file *m,
4734 unsigned long flags)
4736 seq_puts(m, " flags:\n");
4738 if (flags & HIST_FIELD_FL_KEY)
4739 seq_puts(m, " HIST_FIELD_FL_KEY\n");
4740 else if (flags & HIST_FIELD_FL_HITCOUNT)
4741 seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n");
4742 else if (flags & HIST_FIELD_FL_VAR)
4743 seq_puts(m, " HIST_FIELD_FL_VAR\n");
4744 else if (flags & HIST_FIELD_FL_VAR_REF)
4745 seq_puts(m, " HIST_FIELD_FL_VAR_REF\n");
4747 seq_puts(m, " VAL: normal u64 value\n");
4749 if (flags & HIST_FIELD_FL_ALIAS)
4750 seq_puts(m, " HIST_FIELD_FL_ALIAS\n");
4753 static int hist_field_debug_show(struct seq_file *m,
4754 struct hist_field *field, unsigned long flags)
4756 if ((field->flags & flags) != flags) {
4757 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4761 hist_field_debug_show_flags(m, field->flags);
4763 seq_printf(m, " ftrace_event_field name: %s\n",
4764 field->field->name);
4766 if (field->flags & HIST_FIELD_FL_VAR) {
4767 seq_printf(m, " var.name: %s\n", field->var.name);
4768 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
4772 if (field->flags & HIST_FIELD_FL_ALIAS)
4773 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n",
4774 field->var_ref_idx);
4776 if (field->flags & HIST_FIELD_FL_VAR_REF) {
4777 seq_printf(m, " name: %s\n", field->name);
4778 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
4780 seq_printf(m, " var.hist_data: %p\n", field->var.hist_data);
4781 seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n",
4782 field->var_ref_idx);
4784 seq_printf(m, " system: %s\n", field->system);
4785 if (field->event_name)
4786 seq_printf(m, " event_name: %s\n", field->event_name);
4789 seq_printf(m, " type: %s\n", field->type);
4790 seq_printf(m, " size: %u\n", field->size);
4791 seq_printf(m, " is_signed: %u\n", field->is_signed);
4796 static int field_var_debug_show(struct seq_file *m,
4797 struct field_var *field_var, unsigned int i,
4800 const char *vars_name = save_vars ? "save_vars" : "field_vars";
4801 struct hist_field *field;
4804 seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i);
4806 field = field_var->var;
4808 seq_printf(m, "\n %s[%d].var:\n", vars_name, i);
4810 hist_field_debug_show_flags(m, field->flags);
4811 seq_printf(m, " var.name: %s\n", field->var.name);
4812 seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
4815 field = field_var->val;
4817 seq_printf(m, "\n %s[%d].val:\n", vars_name, i);
4819 seq_printf(m, " ftrace_event_field name: %s\n",
4820 field->field->name);
4826 seq_printf(m, " type: %s\n", field->type);
4827 seq_printf(m, " size: %u\n", field->size);
4828 seq_printf(m, " is_signed: %u\n", field->is_signed);
4833 static int hist_action_debug_show(struct seq_file *m,
4834 struct action_data *data, int i)
4838 if (data->handler == HANDLER_ONMAX ||
4839 data->handler == HANDLER_ONCHANGE) {
4840 seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i);
4841 ret = hist_field_debug_show(m, data->track_data.var_ref,
4842 HIST_FIELD_FL_VAR_REF);
4846 seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i);
4847 ret = hist_field_debug_show(m, data->track_data.track_var,
4853 if (data->handler == HANDLER_ONMATCH) {
4854 seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n",
4855 i, data->match_data.event_system);
4856 seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n",
4857 i, data->match_data.event);
4863 static int hist_actions_debug_show(struct seq_file *m,
4864 struct hist_trigger_data *hist_data)
4868 if (hist_data->n_actions)
4869 seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n");
4871 for (i = 0; i < hist_data->n_actions; i++) {
4872 struct action_data *action = hist_data->actions[i];
4874 ret = hist_action_debug_show(m, action, i);
4879 if (hist_data->n_save_vars)
4880 seq_puts(m, "\n save action variables (save() params):\n");
4882 for (i = 0; i < hist_data->n_save_vars; i++) {
4883 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4891 static void hist_trigger_debug_show(struct seq_file *m,
4892 struct event_trigger_data *data, int n)
4894 struct hist_trigger_data *hist_data;
4898 seq_puts(m, "\n\n");
4900 seq_puts(m, "# event histogram\n#\n# trigger info: ");
4901 data->ops->print(m, data->ops, data);
4902 seq_puts(m, "#\n\n");
4904 hist_data = data->private_data;
4906 seq_printf(m, "hist_data: %p\n\n", hist_data);
4907 seq_printf(m, " n_vals: %u\n", hist_data->n_vals);
4908 seq_printf(m, " n_keys: %u\n", hist_data->n_keys);
4909 seq_printf(m, " n_fields: %u\n", hist_data->n_fields);
4911 seq_puts(m, "\n val fields:\n\n");
4913 seq_puts(m, " hist_data->fields[0]:\n");
4914 ret = hist_field_debug_show(m, hist_data->fields[0],
4915 HIST_FIELD_FL_HITCOUNT);
4919 for (i = 1; i < hist_data->n_vals; i++) {
4920 seq_printf(m, "\n hist_data->fields[%d]:\n", i);
4921 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
4926 seq_puts(m, "\n key fields:\n");
4928 for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
4929 seq_printf(m, "\n hist_data->fields[%d]:\n", i);
4930 ret = hist_field_debug_show(m, hist_data->fields[i],
4936 if (hist_data->n_var_refs)
4937 seq_puts(m, "\n variable reference fields:\n");
4939 for (i = 0; i < hist_data->n_var_refs; i++) {
4940 seq_printf(m, "\n hist_data->var_refs[%d]:\n", i);
4941 ret = hist_field_debug_show(m, hist_data->var_refs[i],
4942 HIST_FIELD_FL_VAR_REF);
4947 if (hist_data->n_field_vars)
4948 seq_puts(m, "\n field variables:\n");
4950 for (i = 0; i < hist_data->n_field_vars; i++) {
4951 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
4956 ret = hist_actions_debug_show(m, hist_data);
4961 static int hist_debug_show(struct seq_file *m, void *v)
4963 struct event_trigger_data *data;
4964 struct trace_event_file *event_file;
4967 mutex_lock(&event_mutex);
4969 event_file = event_file_data(m->private);
4970 if (unlikely(!event_file)) {
4975 list_for_each_entry(data, &event_file->triggers, list) {
4976 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4977 hist_trigger_debug_show(m, data, n++);
4981 mutex_unlock(&event_mutex);
4986 static int event_hist_debug_open(struct inode *inode, struct file *file)
4990 ret = security_locked_down(LOCKDOWN_TRACEFS);
4994 return single_open(file, hist_debug_show, file);
4997 const struct file_operations event_hist_debug_fops = {
4998 .open = event_hist_debug_open,
5000 .llseek = seq_lseek,
5001 .release = single_release,
5005 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5007 const char *field_name = hist_field_name(hist_field, 0);
5009 if (hist_field->var.name)
5010 seq_printf(m, "%s=", hist_field->var.name);
5012 if (hist_field->flags & HIST_FIELD_FL_CPU)
5014 else if (field_name) {
5015 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5016 hist_field->flags & HIST_FIELD_FL_ALIAS)
5018 seq_printf(m, "%s", field_name);
5019 } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5020 seq_puts(m, "common_timestamp");
5022 if (hist_field->flags) {
5023 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5024 !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5025 const char *flags = get_hist_field_flags(hist_field);
5028 seq_printf(m, ".%s", flags);
5033 static int event_hist_trigger_print(struct seq_file *m,
5034 struct event_trigger_ops *ops,
5035 struct event_trigger_data *data)
5037 struct hist_trigger_data *hist_data = data->private_data;
5038 struct hist_field *field;
5039 bool have_var = false;
5042 seq_puts(m, "hist:");
5045 seq_printf(m, "%s:", data->name);
5047 seq_puts(m, "keys=");
5049 for_each_hist_key_field(i, hist_data) {
5050 field = hist_data->fields[i];
5052 if (i > hist_data->n_vals)
5055 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5056 seq_puts(m, "stacktrace");
5058 hist_field_print(m, field);
5061 seq_puts(m, ":vals=");
5063 for_each_hist_val_field(i, hist_data) {
5064 field = hist_data->fields[i];
5065 if (field->flags & HIST_FIELD_FL_VAR) {
5070 if (i == HITCOUNT_IDX)
5071 seq_puts(m, "hitcount");
5074 hist_field_print(m, field);
5083 for_each_hist_val_field(i, hist_data) {
5084 field = hist_data->fields[i];
5086 if (field->flags & HIST_FIELD_FL_VAR) {
5089 hist_field_print(m, field);
5094 seq_puts(m, ":sort=");
5096 for (i = 0; i < hist_data->n_sort_keys; i++) {
5097 struct tracing_map_sort_key *sort_key;
5098 unsigned int idx, first_key_idx;
5101 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5103 sort_key = &hist_data->sort_keys[i];
5104 idx = sort_key->field_idx;
5106 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5112 if (idx == HITCOUNT_IDX)
5113 seq_puts(m, "hitcount");
5115 if (idx >= first_key_idx)
5116 idx += hist_data->n_vars;
5117 hist_field_print(m, hist_data->fields[idx]);
5120 if (sort_key->descending)
5121 seq_puts(m, ".descending");
5123 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5124 if (hist_data->enable_timestamps)
5125 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5127 print_actions_spec(m, hist_data);
5129 if (data->filter_str)
5130 seq_printf(m, " if %s", data->filter_str);
5133 seq_puts(m, " [paused]");
5135 seq_puts(m, " [active]");
5142 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5143 struct event_trigger_data *data)
5145 struct hist_trigger_data *hist_data = data->private_data;
5147 if (!data->ref && hist_data->attrs->name)
5148 save_named_trigger(hist_data->attrs->name, data);
5155 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5157 struct trace_event_file *file;
5162 for (i = 0; i < hist_data->n_field_var_hists; i++) {
5163 file = hist_data->field_var_hists[i]->hist_data->event_file;
5164 cmd = hist_data->field_var_hists[i]->cmd;
5165 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5166 "!hist", "hist", cmd);
5170 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5171 struct event_trigger_data *data)
5173 struct hist_trigger_data *hist_data = data->private_data;
5175 if (WARN_ON_ONCE(data->ref <= 0))
5181 del_named_trigger(data);
5183 trigger_data_free(data);
5185 remove_hist_vars(hist_data);
5187 unregister_field_var_hists(hist_data);
5189 destroy_hist_data(hist_data);
5193 static struct event_trigger_ops event_hist_trigger_ops = {
5194 .func = event_hist_trigger,
5195 .print = event_hist_trigger_print,
5196 .init = event_hist_trigger_init,
5197 .free = event_hist_trigger_free,
5200 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5201 struct event_trigger_data *data)
5205 save_named_trigger(data->named_data->name, data);
5207 event_hist_trigger_init(ops, data->named_data);
5212 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5213 struct event_trigger_data *data)
5215 if (WARN_ON_ONCE(data->ref <= 0))
5218 event_hist_trigger_free(ops, data->named_data);
5222 del_named_trigger(data);
5223 trigger_data_free(data);
5227 static struct event_trigger_ops event_hist_trigger_named_ops = {
5228 .func = event_hist_trigger,
5229 .print = event_hist_trigger_print,
5230 .init = event_hist_trigger_named_init,
5231 .free = event_hist_trigger_named_free,
5234 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5237 return &event_hist_trigger_ops;
5240 static void hist_clear(struct event_trigger_data *data)
5242 struct hist_trigger_data *hist_data = data->private_data;
5245 pause_named_trigger(data);
5247 tracepoint_synchronize_unregister();
5249 tracing_map_clear(hist_data->map);
5252 unpause_named_trigger(data);
5255 static bool compatible_field(struct ftrace_event_field *field,
5256 struct ftrace_event_field *test_field)
5258 if (field == test_field)
5260 if (field == NULL || test_field == NULL)
5262 if (strcmp(field->name, test_field->name) != 0)
5264 if (strcmp(field->type, test_field->type) != 0)
5266 if (field->size != test_field->size)
5268 if (field->is_signed != test_field->is_signed)
5274 static bool hist_trigger_match(struct event_trigger_data *data,
5275 struct event_trigger_data *data_test,
5276 struct event_trigger_data *named_data,
5279 struct tracing_map_sort_key *sort_key, *sort_key_test;
5280 struct hist_trigger_data *hist_data, *hist_data_test;
5281 struct hist_field *key_field, *key_field_test;
5284 if (named_data && (named_data != data_test) &&
5285 (named_data != data_test->named_data))
5288 if (!named_data && is_named_trigger(data_test))
5291 hist_data = data->private_data;
5292 hist_data_test = data_test->private_data;
5294 if (hist_data->n_vals != hist_data_test->n_vals ||
5295 hist_data->n_fields != hist_data_test->n_fields ||
5296 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5299 if (!ignore_filter) {
5300 if ((data->filter_str && !data_test->filter_str) ||
5301 (!data->filter_str && data_test->filter_str))
5305 for_each_hist_field(i, hist_data) {
5306 key_field = hist_data->fields[i];
5307 key_field_test = hist_data_test->fields[i];
5309 if (key_field->flags != key_field_test->flags)
5311 if (!compatible_field(key_field->field, key_field_test->field))
5313 if (key_field->offset != key_field_test->offset)
5315 if (key_field->size != key_field_test->size)
5317 if (key_field->is_signed != key_field_test->is_signed)
5319 if (!!key_field->var.name != !!key_field_test->var.name)
5321 if (key_field->var.name &&
5322 strcmp(key_field->var.name, key_field_test->var.name) != 0)
5326 for (i = 0; i < hist_data->n_sort_keys; i++) {
5327 sort_key = &hist_data->sort_keys[i];
5328 sort_key_test = &hist_data_test->sort_keys[i];
5330 if (sort_key->field_idx != sort_key_test->field_idx ||
5331 sort_key->descending != sort_key_test->descending)
5335 if (!ignore_filter && data->filter_str &&
5336 (strcmp(data->filter_str, data_test->filter_str) != 0))
5339 if (!actions_match(hist_data, hist_data_test))
5345 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5346 struct event_trigger_data *data,
5347 struct trace_event_file *file)
5349 struct hist_trigger_data *hist_data = data->private_data;
5350 struct event_trigger_data *test, *named_data = NULL;
5351 struct trace_array *tr = file->tr;
5354 if (hist_data->attrs->name) {
5355 named_data = find_named_trigger(hist_data->attrs->name);
5357 if (!hist_trigger_match(data, named_data, named_data,
5359 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5366 if (hist_data->attrs->name && !named_data)
5369 lockdep_assert_held(&event_mutex);
5371 list_for_each_entry(test, &file->triggers, list) {
5372 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5373 if (!hist_trigger_match(data, test, named_data, false))
5375 if (hist_data->attrs->pause)
5376 test->paused = true;
5377 else if (hist_data->attrs->cont)
5378 test->paused = false;
5379 else if (hist_data->attrs->clear)
5382 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5389 if (hist_data->attrs->cont || hist_data->attrs->clear) {
5390 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5395 if (hist_data->attrs->pause)
5396 data->paused = true;
5399 data->private_data = named_data->private_data;
5400 set_named_trigger_data(data, named_data);
5401 data->ops = &event_hist_trigger_named_ops;
5404 if (data->ops->init) {
5405 ret = data->ops->init(data->ops, data);
5410 if (hist_data->enable_timestamps) {
5411 char *clock = hist_data->attrs->clock;
5413 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5415 hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5419 tracing_set_time_stamp_abs(file->tr, true);
5423 destroy_hist_data(hist_data);
5430 static int hist_trigger_enable(struct event_trigger_data *data,
5431 struct trace_event_file *file)
5435 list_add_tail_rcu(&data->list, &file->triggers);
5437 update_cond_flag(file);
5439 if (trace_event_trigger_enable_disable(file, 1) < 0) {
5440 list_del_rcu(&data->list);
5441 update_cond_flag(file);
5448 static bool have_hist_trigger_match(struct event_trigger_data *data,
5449 struct trace_event_file *file)
5451 struct hist_trigger_data *hist_data = data->private_data;
5452 struct event_trigger_data *test, *named_data = NULL;
5455 lockdep_assert_held(&event_mutex);
5457 if (hist_data->attrs->name)
5458 named_data = find_named_trigger(hist_data->attrs->name);
5460 list_for_each_entry(test, &file->triggers, list) {
5461 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5462 if (hist_trigger_match(data, test, named_data, false)) {
5472 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5473 struct trace_event_file *file)
5475 struct hist_trigger_data *hist_data = data->private_data;
5476 struct event_trigger_data *test, *named_data = NULL;
5478 lockdep_assert_held(&event_mutex);
5480 if (hist_data->attrs->name)
5481 named_data = find_named_trigger(hist_data->attrs->name);
5483 list_for_each_entry(test, &file->triggers, list) {
5484 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5485 if (!hist_trigger_match(data, test, named_data, false))
5487 hist_data = test->private_data;
5488 if (check_var_refs(hist_data))
5497 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5498 struct event_trigger_data *data,
5499 struct trace_event_file *file)
5501 struct hist_trigger_data *hist_data = data->private_data;
5502 struct event_trigger_data *test, *named_data = NULL;
5503 bool unregistered = false;
5505 lockdep_assert_held(&event_mutex);
5507 if (hist_data->attrs->name)
5508 named_data = find_named_trigger(hist_data->attrs->name);
5510 list_for_each_entry(test, &file->triggers, list) {
5511 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5512 if (!hist_trigger_match(data, test, named_data, false))
5514 unregistered = true;
5515 list_del_rcu(&test->list);
5516 trace_event_trigger_enable_disable(file, 0);
5517 update_cond_flag(file);
5522 if (unregistered && test->ops->free)
5523 test->ops->free(test->ops, test);
5525 if (hist_data->enable_timestamps) {
5526 if (!hist_data->remove || unregistered)
5527 tracing_set_time_stamp_abs(file->tr, false);
5531 static bool hist_file_check_refs(struct trace_event_file *file)
5533 struct hist_trigger_data *hist_data;
5534 struct event_trigger_data *test;
5536 lockdep_assert_held(&event_mutex);
5538 list_for_each_entry(test, &file->triggers, list) {
5539 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5540 hist_data = test->private_data;
5541 if (check_var_refs(hist_data))
5549 static void hist_unreg_all(struct trace_event_file *file)
5551 struct event_trigger_data *test, *n;
5552 struct hist_trigger_data *hist_data;
5553 struct synth_event *se;
5554 const char *se_name;
5556 lockdep_assert_held(&event_mutex);
5558 if (hist_file_check_refs(file))
5561 list_for_each_entry_safe(test, n, &file->triggers, list) {
5562 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5563 hist_data = test->private_data;
5564 list_del_rcu(&test->list);
5565 trace_event_trigger_enable_disable(file, 0);
5567 se_name = trace_event_name(file->event_call);
5568 se = find_synth_event(se_name);
5572 update_cond_flag(file);
5573 if (hist_data->enable_timestamps)
5574 tracing_set_time_stamp_abs(file->tr, false);
5575 if (test->ops->free)
5576 test->ops->free(test->ops, test);
5581 static int event_hist_trigger_func(struct event_command *cmd_ops,
5582 struct trace_event_file *file,
5583 char *glob, char *cmd, char *param)
5585 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5586 struct event_trigger_data *trigger_data;
5587 struct hist_trigger_attrs *attrs;
5588 struct event_trigger_ops *trigger_ops;
5589 struct hist_trigger_data *hist_data;
5590 struct synth_event *se;
5591 const char *se_name;
5592 bool remove = false;
5596 lockdep_assert_held(&event_mutex);
5598 if (glob && strlen(glob)) {
5600 last_cmd_set(file, param);
5610 * separate the trigger from the filter (k:v [if filter])
5611 * allowing for whitespace in the trigger
5613 p = trigger = param;
5615 p = strstr(p, "if");
5620 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5624 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5626 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5637 param = strstrip(p);
5638 trigger = strstrip(trigger);
5641 attrs = parse_hist_trigger_attrs(file->tr, trigger);
5643 return PTR_ERR(attrs);
5645 if (attrs->map_bits)
5646 hist_trigger_bits = attrs->map_bits;
5648 hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5649 if (IS_ERR(hist_data)) {
5650 destroy_hist_trigger_attrs(attrs);
5651 return PTR_ERR(hist_data);
5654 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5656 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5657 if (!trigger_data) {
5662 trigger_data->count = -1;
5663 trigger_data->ops = trigger_ops;
5664 trigger_data->cmd_ops = cmd_ops;
5666 INIT_LIST_HEAD(&trigger_data->list);
5667 RCU_INIT_POINTER(trigger_data->filter, NULL);
5669 trigger_data->private_data = hist_data;
5671 /* if param is non-empty, it's supposed to be a filter */
5672 if (param && cmd_ops->set_filter) {
5673 ret = cmd_ops->set_filter(param, trigger_data, file);
5679 if (!have_hist_trigger_match(trigger_data, file))
5682 if (hist_trigger_check_refs(trigger_data, file)) {
5687 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5688 se_name = trace_event_name(file->event_call);
5689 se = find_synth_event(se_name);
5696 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5698 * The above returns on success the # of triggers registered,
5699 * but if it didn't register any it returns zero. Consider no
5700 * triggers registered a failure too.
5703 if (!(attrs->pause || attrs->cont || attrs->clear))
5709 if (get_named_trigger_data(trigger_data))
5712 if (has_hist_vars(hist_data))
5713 save_hist_vars(hist_data);
5715 ret = create_actions(hist_data);
5719 ret = tracing_map_init(hist_data->map);
5723 ret = hist_trigger_enable(trigger_data, file);
5727 se_name = trace_event_name(file->event_call);
5728 se = find_synth_event(se_name);
5731 /* Just return zero, not the number of registered triggers */
5739 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5741 if (cmd_ops->set_filter)
5742 cmd_ops->set_filter(NULL, trigger_data, NULL);
5744 remove_hist_vars(hist_data);
5746 kfree(trigger_data);
5748 destroy_hist_data(hist_data);
5752 static struct event_command trigger_hist_cmd = {
5754 .trigger_type = ETT_EVENT_HIST,
5755 .flags = EVENT_CMD_FL_NEEDS_REC,
5756 .func = event_hist_trigger_func,
5757 .reg = hist_register_trigger,
5758 .unreg = hist_unregister_trigger,
5759 .unreg_all = hist_unreg_all,
5760 .get_trigger_ops = event_hist_get_trigger_ops,
5761 .set_filter = set_trigger_filter,
5764 __init int register_trigger_hist_cmd(void)
5768 ret = register_event_command(&trigger_hist_cmd);
5775 hist_enable_trigger(struct event_trigger_data *data, void *rec,
5776 struct ring_buffer_event *event)
5778 struct enable_trigger_data *enable_data = data->private_data;
5779 struct event_trigger_data *test;
5781 list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5782 lockdep_is_held(&event_mutex)) {
5783 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5784 if (enable_data->enable)
5785 test->paused = false;
5787 test->paused = true;
5793 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5794 struct ring_buffer_event *event)
5799 if (data->count != -1)
5802 hist_enable_trigger(data, rec, event);
5805 static struct event_trigger_ops hist_enable_trigger_ops = {
5806 .func = hist_enable_trigger,
5807 .print = event_enable_trigger_print,
5808 .init = event_trigger_init,
5809 .free = event_enable_trigger_free,
5812 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5813 .func = hist_enable_count_trigger,
5814 .print = event_enable_trigger_print,
5815 .init = event_trigger_init,
5816 .free = event_enable_trigger_free,
5819 static struct event_trigger_ops hist_disable_trigger_ops = {
5820 .func = hist_enable_trigger,
5821 .print = event_enable_trigger_print,
5822 .init = event_trigger_init,
5823 .free = event_enable_trigger_free,
5826 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5827 .func = hist_enable_count_trigger,
5828 .print = event_enable_trigger_print,
5829 .init = event_trigger_init,
5830 .free = event_enable_trigger_free,
5833 static struct event_trigger_ops *
5834 hist_enable_get_trigger_ops(char *cmd, char *param)
5836 struct event_trigger_ops *ops;
5839 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5842 ops = param ? &hist_enable_count_trigger_ops :
5843 &hist_enable_trigger_ops;
5845 ops = param ? &hist_disable_count_trigger_ops :
5846 &hist_disable_trigger_ops;
5851 static void hist_enable_unreg_all(struct trace_event_file *file)
5853 struct event_trigger_data *test, *n;
5855 list_for_each_entry_safe(test, n, &file->triggers, list) {
5856 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5857 list_del_rcu(&test->list);
5858 update_cond_flag(file);
5859 trace_event_trigger_enable_disable(file, 0);
5860 if (test->ops->free)
5861 test->ops->free(test->ops, test);
5866 static struct event_command trigger_hist_enable_cmd = {
5867 .name = ENABLE_HIST_STR,
5868 .trigger_type = ETT_HIST_ENABLE,
5869 .func = event_enable_trigger_func,
5870 .reg = event_enable_register_trigger,
5871 .unreg = event_enable_unregister_trigger,
5872 .unreg_all = hist_enable_unreg_all,
5873 .get_trigger_ops = hist_enable_get_trigger_ops,
5874 .set_filter = set_trigger_filter,
5877 static struct event_command trigger_hist_disable_cmd = {
5878 .name = DISABLE_HIST_STR,
5879 .trigger_type = ETT_HIST_ENABLE,
5880 .func = event_enable_trigger_func,
5881 .reg = event_enable_register_trigger,
5882 .unreg = event_enable_unregister_trigger,
5883 .unreg_all = hist_enable_unreg_all,
5884 .get_trigger_ops = hist_enable_get_trigger_ops,
5885 .set_filter = set_trigger_filter,
5888 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5890 unregister_event_command(&trigger_hist_enable_cmd);
5891 unregister_event_command(&trigger_hist_disable_cmd);
5894 __init int register_trigger_hist_enable_disable_cmds(void)
5898 ret = register_event_command(&trigger_hist_enable_cmd);
5899 if (WARN_ON(ret < 0))
5901 ret = register_event_command(&trigger_hist_disable_cmd);
5902 if (WARN_ON(ret < 0))
5903 unregister_trigger_hist_enable_disable_cmds();