kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"),
  70
  71 #undef C
  72 #define C(a, b)         HIST_ERR_##a
  73
  74 enum { ERRORS };
  75
  76 #undef C
  77 #define C(a, b)         b
  78
  79 static const char *err_text[] = { ERRORS };
  80
  81 struct hist_field;
  82
  83 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  84                                 struct tracing_map_elt *elt,
  85                                 struct ring_buffer_event *rbe,
  86                                 void *event);
  87
  88 #define HIST_FIELD_OPERANDS_MAX 2
  89 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  90 #define HIST_ACTIONS_MAX        8
  91
  92 enum field_op_id {
  93         FIELD_OP_NONE,
  94         FIELD_OP_PLUS,
  95         FIELD_OP_MINUS,
  96         FIELD_OP_UNARY_MINUS,
  97 };
  98
  99 /*
 100  * A hist_var (histogram variable) contains variable information for
 101  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 102  * flag set.  A hist_var has a variable name e.g. ts0, and is
 103  * associated with a given histogram trigger, as specified by
 104  * hist_data.  The hist_var idx is the unique index assigned to the
 105  * variable by the hist trigger's tracing_map.  The idx is what is
 106  * used to set a variable's value and, by a variable reference, to
 107  * retrieve it.
 108  */
 109 struct hist_var {
 110         char                            *name;
 111         struct hist_trigger_data        *hist_data;
 112         unsigned int                    idx;
 113 };
 114
 115 struct hist_field {
 116         struct ftrace_event_field       *field;
 117         unsigned long                   flags;
 118         hist_field_fn_t                 fn;
 119         unsigned int                    ref;
 120         unsigned int                    size;
 121         unsigned int                    offset;
 122         unsigned int                    is_signed;
 123         const char                      *type;
 124         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 125         struct hist_trigger_data        *hist_data;
 126
 127         /*
 128          * Variable fields contain variable-specific info in var.
 129          */
 130         struct hist_var                 var;
 131         enum field_op_id                operator;
 132         char                            *system;
 133         char                            *event_name;
 134
 135         /*
 136          * The name field is used for EXPR and VAR_REF fields.  VAR
 137          * fields contain the variable name in var.name.
 138          */
 139         char                            *name;
 140
 141         /*
 142          * When a histogram trigger is hit, if it has any references
 143          * to variables, the values of those variables are collected
 144          * into a var_ref_vals array by resolve_var_refs().  The
 145          * current value of each variable is read from the tracing_map
 146          * using the hist field's hist_var.idx and entered into the
 147          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 148          */
 149         unsigned int                    var_ref_idx;
 150         bool                            read_once;
 151
 152         unsigned int                    var_str_idx;
 153 };
 154
 155 static u64 hist_field_none(struct hist_field *field,
 156                            struct tracing_map_elt *elt,
 157                            struct ring_buffer_event *rbe,
 158                            void *event)
 159 {
 160         return 0;
 161 }
 162
 163 static u64 hist_field_counter(struct hist_field *field,
 164                               struct tracing_map_elt *elt,
 165                               struct ring_buffer_event *rbe,
 166                               void *event)
 167 {
 168         return 1;
 169 }
 170
 171 static u64 hist_field_string(struct hist_field *hist_field,
 172                              struct tracing_map_elt *elt,
 173                              struct ring_buffer_event *rbe,
 174                              void *event)
 175 {
 176         char *addr = (char *)(event + hist_field->field->offset);
 177
 178         return (u64)(unsigned long)addr;
 179 }
 180
 181 static u64 hist_field_dynstring(struct hist_field *hist_field,
 182                                 struct tracing_map_elt *elt,
 183                                 struct ring_buffer_event *rbe,
 184                                 void *event)
 185 {
 186         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 187         int str_loc = str_item & 0xffff;
 188         char *addr = (char *)(event + str_loc);
 189
 190         return (u64)(unsigned long)addr;
 191 }
 192
 193 static u64 hist_field_pstring(struct hist_field *hist_field,
 194                               struct tracing_map_elt *elt,
 195                               struct ring_buffer_event *rbe,
 196                               void *event)
 197 {
 198         char **addr = (char **)(event + hist_field->field->offset);
 199
 200         return (u64)(unsigned long)*addr;
 201 }
 202
 203 static u64 hist_field_log2(struct hist_field *hist_field,
 204                            struct tracing_map_elt *elt,
 205                            struct ring_buffer_event *rbe,
 206                            void *event)
 207 {
 208         struct hist_field *operand = hist_field->operands[0];
 209
 210         u64 val = operand->fn(operand, elt, rbe, event);
 211
 212         return (u64) ilog2(roundup_pow_of_two(val));
 213 }
 214
 215 static u64 hist_field_plus(struct hist_field *hist_field,
 216                            struct tracing_map_elt *elt,
 217                            struct ring_buffer_event *rbe,
 218                            void *event)
 219 {
 220         struct hist_field *operand1 = hist_field->operands[0];
 221         struct hist_field *operand2 = hist_field->operands[1];
 222
 223         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 224         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 225
 226         return val1 + val2;
 227 }
 228
 229 static u64 hist_field_minus(struct hist_field *hist_field,
 230                             struct tracing_map_elt *elt,
 231                             struct ring_buffer_event *rbe,
 232                             void *event)
 233 {
 234         struct hist_field *operand1 = hist_field->operands[0];
 235         struct hist_field *operand2 = hist_field->operands[1];
 236
 237         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 238         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 239
 240         return val1 - val2;
 241 }
 242
 243 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 244                                   struct tracing_map_elt *elt,
 245                                   struct ring_buffer_event *rbe,
 246                                   void *event)
 247 {
 248         struct hist_field *operand = hist_field->operands[0];
 249
 250         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 251         u64 val = (u64)-sval;
 252
 253         return val;
 254 }
 255
 256 #define DEFINE_HIST_FIELD_FN(type)                                      \
 257         static u64 hist_field_##type(struct hist_field *hist_field,     \
 258                                      struct tracing_map_elt *elt,       \
 259                                      struct ring_buffer_event *rbe,     \
 260                                      void *event)                       \
 261 {                                                                       \
 262         type *addr = (type *)(event + hist_field->field->offset);       \
 263                                                                         \
 264         return (u64)(unsigned long)*addr;                               \
 265 }
 266
 267 DEFINE_HIST_FIELD_FN(s64);
 268 DEFINE_HIST_FIELD_FN(u64);
 269 DEFINE_HIST_FIELD_FN(s32);
 270 DEFINE_HIST_FIELD_FN(u32);
 271 DEFINE_HIST_FIELD_FN(s16);
 272 DEFINE_HIST_FIELD_FN(u16);
 273 DEFINE_HIST_FIELD_FN(s8);
 274 DEFINE_HIST_FIELD_FN(u8);
 275
 276 #define for_each_hist_field(i, hist_data)       \
 277         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 278
 279 #define for_each_hist_val_field(i, hist_data)   \
 280         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 281
 282 #define for_each_hist_key_field(i, hist_data)   \
 283         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 284
 285 #define HIST_STACKTRACE_DEPTH   16
 286 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 287 #define HIST_STACKTRACE_SKIP    5
 288
 289 #define HITCOUNT_IDX            0
 290 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 291
 292 enum hist_field_flags {
 293         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 294         HIST_FIELD_FL_KEY               = 1 << 1,
 295         HIST_FIELD_FL_STRING            = 1 << 2,
 296         HIST_FIELD_FL_HEX               = 1 << 3,
 297         HIST_FIELD_FL_SYM               = 1 << 4,
 298         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 299         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 300         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 301         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 302         HIST_FIELD_FL_LOG2              = 1 << 9,
 303         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 304         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 305         HIST_FIELD_FL_VAR               = 1 << 12,
 306         HIST_FIELD_FL_EXPR              = 1 << 13,
 307         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 308         HIST_FIELD_FL_CPU               = 1 << 15,
 309         HIST_FIELD_FL_ALIAS             = 1 << 16,
 310 };
 311
 312 struct var_defs {
 313         unsigned int    n_vars;
 314         char            *name[TRACING_MAP_VARS_MAX];
 315         char            *expr[TRACING_MAP_VARS_MAX];
 316 };
 317
 318 struct hist_trigger_attrs {
 319         char            *keys_str;
 320         char            *vals_str;
 321         char            *sort_key_str;
 322         char            *name;
 323         char            *clock;
 324         bool            pause;
 325         bool            cont;
 326         bool            clear;
 327         bool            ts_in_usecs;
 328         unsigned int    map_bits;
 329
 330         char            *assignment_str[TRACING_MAP_VARS_MAX];
 331         unsigned int    n_assignments;
 332
 333         char            *action_str[HIST_ACTIONS_MAX];
 334         unsigned int    n_actions;
 335
 336         struct var_defs var_defs;
 337 };
 338
 339 struct field_var {
 340         struct hist_field       *var;
 341         struct hist_field       *val;
 342 };
 343
 344 struct field_var_hist {
 345         struct hist_trigger_data        *hist_data;
 346         char                            *cmd;
 347 };
 348
 349 struct hist_trigger_data {
 350         struct hist_field               *fields[HIST_FIELDS_MAX];
 351         unsigned int                    n_vals;
 352         unsigned int                    n_keys;
 353         unsigned int                    n_fields;
 354         unsigned int                    n_vars;
 355         unsigned int                    n_var_str;
 356         unsigned int                    key_size;
 357         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 358         unsigned int                    n_sort_keys;
 359         struct trace_event_file         *event_file;
 360         struct hist_trigger_attrs       *attrs;
 361         struct tracing_map              *map;
 362         bool                            enable_timestamps;
 363         bool                            remove;
 364         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 365         unsigned int                    n_var_refs;
 366
 367         struct action_data              *actions[HIST_ACTIONS_MAX];
 368         unsigned int                    n_actions;
 369
 370         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 371         unsigned int                    n_field_vars;
 372         unsigned int                    n_field_var_str;
 373         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 374         unsigned int                    n_field_var_hists;
 375
 376         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 377         unsigned int                    n_save_vars;
 378         unsigned int                    n_save_var_str;
 379 };
 380
 381 struct action_data;
 382
 383 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 384                              struct tracing_map_elt *elt, void *rec,
 385                              struct ring_buffer_event *rbe, void *key,
 386                              struct action_data *data, u64 *var_ref_vals);
 387
 388 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 389
 390 enum handler_id {
 391         HANDLER_ONMATCH = 1,
 392         HANDLER_ONMAX,
 393         HANDLER_ONCHANGE,
 394 };
 395
 396 enum action_id {
 397         ACTION_SAVE = 1,
 398         ACTION_TRACE,
 399         ACTION_SNAPSHOT,
 400 };
 401
 402 struct action_data {
 403         enum handler_id         handler;
 404         enum action_id          action;
 405         char                    *action_name;
 406         action_fn_t             fn;
 407
 408         unsigned int            n_params;
 409         char                    *params[SYNTH_FIELDS_MAX];
 410
 411         /*
 412          * When a histogram trigger is hit, the values of any
 413          * references to variables, including variables being passed
 414          * as parameters to synthetic events, are collected into a
 415          * var_ref_vals array.  This var_ref_idx array is an array of
 416          * indices into the var_ref_vals array, one for each synthetic
 417          * event param, and is passed to the synthetic event
 418          * invocation.
 419          */
 420         unsigned int            var_ref_idx[TRACING_MAP_VARS_MAX];
 421         struct synth_event      *synth_event;
 422         bool                    use_trace_keyword;
 423         char                    *synth_event_name;
 424
 425         union {
 426                 struct {
 427                         char                    *event;
 428                         char                    *event_system;
 429                 } match_data;
 430
 431                 struct {
 432                         /*
 433                          * var_str contains the $-unstripped variable
 434                          * name referenced by var_ref, and used when
 435                          * printing the action.  Because var_ref
 436                          * creation is deferred to create_actions(),
 437                          * we need a per-action way to save it until
 438                          * then, thus var_str.
 439                          */
 440                         char                    *var_str;
 441
 442                         /*
 443                          * var_ref refers to the variable being
 444                          * tracked e.g onmax($var).
 445                          */
 446                         struct hist_field       *var_ref;
 447
 448                         /*
 449                          * track_var contains the 'invisible' tracking
 450                          * variable created to keep the current
 451                          * e.g. max value.
 452                          */
 453                         struct hist_field       *track_var;
 454
 455                         check_track_val_fn_t    check_val;
 456                         action_fn_t             save_data;
 457                 } track_data;
 458         };
 459 };
 460
 461 struct track_data {
 462         u64                             track_val;
 463         bool                            updated;
 464
 465         unsigned int                    key_len;
 466         void                            *key;
 467         struct tracing_map_elt          elt;
 468
 469         struct action_data              *action_data;
 470         struct hist_trigger_data        *hist_data;
 471 };
 472
 473 struct hist_elt_data {
 474         char *comm;
 475         u64 *var_ref_vals;
 476         char *field_var_str[SYNTH_FIELDS_MAX];
 477 };
 478
 479 struct snapshot_context {
 480         struct tracing_map_elt  *elt;
 481         void                    *key;
 482 };
 483
 484 static void track_data_free(struct track_data *track_data)
 485 {
 486         struct hist_elt_data *elt_data;
 487
 488         if (!track_data)
 489                 return;
 490
 491         kfree(track_data->key);
 492
 493         elt_data = track_data->elt.private_data;
 494         if (elt_data) {
 495                 kfree(elt_data->comm);
 496                 kfree(elt_data);
 497         }
 498
 499         kfree(track_data);
 500 }
 501
 502 static struct track_data *track_data_alloc(unsigned int key_len,
 503                                            struct action_data *action_data,
 504                                            struct hist_trigger_data *hist_data)
 505 {
 506         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 507         struct hist_elt_data *elt_data;
 508
 509         if (!data)
 510                 return ERR_PTR(-ENOMEM);
 511
 512         data->key = kzalloc(key_len, GFP_KERNEL);
 513         if (!data->key) {
 514                 track_data_free(data);
 515                 return ERR_PTR(-ENOMEM);
 516         }
 517
 518         data->key_len = key_len;
 519         data->action_data = action_data;
 520         data->hist_data = hist_data;
 521
 522         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 523         if (!elt_data) {
 524                 track_data_free(data);
 525                 return ERR_PTR(-ENOMEM);
 526         }
 527
 528         data->elt.private_data = elt_data;
 529
 530         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 531         if (!elt_data->comm) {
 532                 track_data_free(data);
 533                 return ERR_PTR(-ENOMEM);
 534         }
 535
 536         return data;
 537 }
 538
 539 static char last_cmd[MAX_FILTER_STR_VAL];
 540 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 541
 542 static int errpos(char *str)
 543 {
 544         return err_pos(last_cmd, str);
 545 }
 546
 547 static void last_cmd_set(struct trace_event_file *file, char *str)
 548 {
 549         const char *system = NULL, *name = NULL;
 550         struct trace_event_call *call;
 551
 552         if (!str)
 553                 return;
 554
 555         strcpy(last_cmd, "hist:");
 556         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 557
 558         if (file) {
 559                 call = file->event_call;
 560                 system = call->class->system;
 561                 if (system) {
 562                         name = trace_event_name(call);
 563                         if (!name)
 564                                 system = NULL;
 565                 }
 566         }
 567
 568         if (system)
 569                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 570 }
 571
 572 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 573 {
 574         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 575                         err_type, err_pos);
 576 }
 577
 578 static void hist_err_clear(void)
 579 {
 580         last_cmd[0] = '\0';
 581         last_cmd_loc[0] = '\0';
 582 }
 583
 584 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 585                                     unsigned int *var_ref_idx);
 586
 587 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 588                                unsigned int *var_ref_idx)
 589 {
 590         struct tracepoint *tp = event->tp;
 591
 592         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 593                 struct tracepoint_func *probe_func_ptr;
 594                 synth_probe_func_t probe_func;
 595                 void *__data;
 596
 597                 if (!(cpu_online(raw_smp_processor_id())))
 598                         return;
 599
 600                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 601                 if (probe_func_ptr) {
 602                         do {
 603                                 probe_func = probe_func_ptr->func;
 604                                 __data = probe_func_ptr->data;
 605                                 probe_func(__data, var_ref_vals, var_ref_idx);
 606                         } while ((++probe_func_ptr)->func);
 607                 }
 608         }
 609 }
 610
 611 static void action_trace(struct hist_trigger_data *hist_data,
 612                          struct tracing_map_elt *elt, void *rec,
 613                          struct ring_buffer_event *rbe, void *key,
 614                          struct action_data *data, u64 *var_ref_vals)
 615 {
 616         struct synth_event *event = data->synth_event;
 617
 618         trace_synth(event, var_ref_vals, data->var_ref_idx);
 619 }
 620
 621 struct hist_var_data {
 622         struct list_head list;
 623         struct hist_trigger_data *hist_data;
 624 };
 625
 626 static u64 hist_field_timestamp(struct hist_field *hist_field,
 627                                 struct tracing_map_elt *elt,
 628                                 struct ring_buffer_event *rbe,
 629                                 void *event)
 630 {
 631         struct hist_trigger_data *hist_data = hist_field->hist_data;
 632         struct trace_array *tr = hist_data->event_file->tr;
 633
 634         u64 ts = ring_buffer_event_time_stamp(rbe);
 635
 636         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 637                 ts = ns2usecs(ts);
 638
 639         return ts;
 640 }
 641
 642 static u64 hist_field_cpu(struct hist_field *hist_field,
 643                           struct tracing_map_elt *elt,
 644                           struct ring_buffer_event *rbe,
 645                           void *event)
 646 {
 647         int cpu = smp_processor_id();
 648
 649         return cpu;
 650 }
 651
 652 /**
 653  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 654  * @hist_field: The VAR_REF field to check
 655  * @var_data: The hist trigger that owns the variable
 656  * @var_idx: The trigger variable identifier
 657  *
 658  * Check the given VAR_REF field to see whether or not it references
 659  * the given variable associated with the given trigger.
 660  *
 661  * Return: The VAR_REF field if it does reference the variable, NULL if not
 662  */
 663 static struct hist_field *
 664 check_field_for_var_ref(struct hist_field *hist_field,
 665                         struct hist_trigger_data *var_data,
 666                         unsigned int var_idx)
 667 {
 668         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 669
 670         if (hist_field && hist_field->var.idx == var_idx &&
 671             hist_field->var.hist_data == var_data)
 672                 return hist_field;
 673
 674         return NULL;
 675 }
 676
 677 /**
 678  * find_var_ref - Check if a trigger has a reference to a trigger variable
 679  * @hist_data: The hist trigger that might have a reference to the variable
 680  * @var_data: The hist trigger that owns the variable
 681  * @var_idx: The trigger variable identifier
 682  *
 683  * Check the list of var_refs[] on the first hist trigger to see
 684  * whether any of them are references to the variable on the second
 685  * trigger.
 686  *
 687  * Return: The VAR_REF field referencing the variable if so, NULL if not
 688  */
 689 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 690                                        struct hist_trigger_data *var_data,
 691                                        unsigned int var_idx)
 692 {
 693         struct hist_field *hist_field;
 694         unsigned int i;
 695
 696         for (i = 0; i < hist_data->n_var_refs; i++) {
 697                 hist_field = hist_data->var_refs[i];
 698                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 699                         return hist_field;
 700         }
 701
 702         return NULL;
 703 }
 704
 705 /**
 706  * find_any_var_ref - Check if there is a reference to a given trigger variable
 707  * @hist_data: The hist trigger
 708  * @var_idx: The trigger variable identifier
 709  *
 710  * Check to see whether the given variable is currently referenced by
 711  * any other trigger.
 712  *
 713  * The trigger the variable is defined on is explicitly excluded - the
 714  * assumption being that a self-reference doesn't prevent a trigger
 715  * from being removed.
 716  *
 717  * Return: The VAR_REF field referencing the variable if so, NULL if not
 718  */
 719 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 720                                            unsigned int var_idx)
 721 {
 722         struct trace_array *tr = hist_data->event_file->tr;
 723         struct hist_field *found = NULL;
 724         struct hist_var_data *var_data;
 725
 726         list_for_each_entry(var_data, &tr->hist_vars, list) {
 727                 if (var_data->hist_data == hist_data)
 728                         continue;
 729                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 730                 if (found)
 731                         break;
 732         }
 733
 734         return found;
 735 }
 736
 737 /**
 738  * check_var_refs - Check if there is a reference to any of trigger's variables
 739  * @hist_data: The hist trigger
 740  *
 741  * A trigger can define one or more variables.  If any one of them is
 742  * currently referenced by any other trigger, this function will
 743  * determine that.
 744
 745  * Typically used to determine whether or not a trigger can be removed
 746  * - if there are any references to a trigger's variables, it cannot.
 747  *
 748  * Return: True if there is a reference to any of trigger's variables
 749  */
 750 static bool check_var_refs(struct hist_trigger_data *hist_data)
 751 {
 752         struct hist_field *field;
 753         bool found = false;
 754         int i;
 755
 756         for_each_hist_field(i, hist_data) {
 757                 field = hist_data->fields[i];
 758                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 759                         if (find_any_var_ref(hist_data, field->var.idx)) {
 760                                 found = true;
 761                                 break;
 762                         }
 763                 }
 764         }
 765
 766         return found;
 767 }
 768
 769 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 770 {
 771         struct trace_array *tr = hist_data->event_file->tr;
 772         struct hist_var_data *var_data, *found = NULL;
 773
 774         list_for_each_entry(var_data, &tr->hist_vars, list) {
 775                 if (var_data->hist_data == hist_data) {
 776                         found = var_data;
 777                         break;
 778                 }
 779         }
 780
 781         return found;
 782 }
 783
 784 static bool field_has_hist_vars(struct hist_field *hist_field,
 785                                 unsigned int level)
 786 {
 787         int i;
 788
 789         if (level > 3)
 790                 return false;
 791
 792         if (!hist_field)
 793                 return false;
 794
 795         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 796             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 797                 return true;
 798
 799         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 800                 struct hist_field *operand;
 801
 802                 operand = hist_field->operands[i];
 803                 if (field_has_hist_vars(operand, level + 1))
 804                         return true;
 805         }
 806
 807         return false;
 808 }
 809
 810 static bool has_hist_vars(struct hist_trigger_data *hist_data)
 811 {
 812         struct hist_field *hist_field;
 813         int i;
 814
 815         for_each_hist_field(i, hist_data) {
 816                 hist_field = hist_data->fields[i];
 817                 if (field_has_hist_vars(hist_field, 0))
 818                         return true;
 819         }
 820
 821         return false;
 822 }
 823
 824 static int save_hist_vars(struct hist_trigger_data *hist_data)
 825 {
 826         struct trace_array *tr = hist_data->event_file->tr;
 827         struct hist_var_data *var_data;
 828
 829         var_data = find_hist_vars(hist_data);
 830         if (var_data)
 831                 return 0;
 832
 833         if (tracing_check_open_get_tr(tr))
 834                 return -ENODEV;
 835
 836         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
 837         if (!var_data) {
 838                 trace_array_put(tr);
 839                 return -ENOMEM;
 840         }
 841
 842         var_data->hist_data = hist_data;
 843         list_add(&var_data->list, &tr->hist_vars);
 844
 845         return 0;
 846 }
 847
 848 static void remove_hist_vars(struct hist_trigger_data *hist_data)
 849 {
 850         struct trace_array *tr = hist_data->event_file->tr;
 851         struct hist_var_data *var_data;
 852
 853         var_data = find_hist_vars(hist_data);
 854         if (!var_data)
 855                 return;
 856
 857         if (WARN_ON(check_var_refs(hist_data)))
 858                 return;
 859
 860         list_del(&var_data->list);
 861
 862         kfree(var_data);
 863
 864         trace_array_put(tr);
 865 }
 866
 867 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 868                                          const char *var_name)
 869 {
 870         struct hist_field *hist_field, *found = NULL;
 871         int i;
 872
 873         for_each_hist_field(i, hist_data) {
 874                 hist_field = hist_data->fields[i];
 875                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 876                     strcmp(hist_field->var.name, var_name) == 0) {
 877                         found = hist_field;
 878                         break;
 879                 }
 880         }
 881
 882         return found;
 883 }
 884
 885 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 886                                    struct trace_event_file *file,
 887                                    const char *var_name)
 888 {
 889         struct hist_trigger_data *test_data;
 890         struct event_trigger_data *test;
 891         struct hist_field *hist_field;
 892
 893         lockdep_assert_held(&event_mutex);
 894
 895         hist_field = find_var_field(hist_data, var_name);
 896         if (hist_field)
 897                 return hist_field;
 898
 899         list_for_each_entry(test, &file->triggers, list) {
 900                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 901                         test_data = test->private_data;
 902                         hist_field = find_var_field(test_data, var_name);
 903                         if (hist_field)
 904                                 return hist_field;
 905                 }
 906         }
 907
 908         return NULL;
 909 }
 910
 911 static struct trace_event_file *find_var_file(struct trace_array *tr,
 912                                               char *system,
 913                                               char *event_name,
 914                                               char *var_name)
 915 {
 916         struct hist_trigger_data *var_hist_data;
 917         struct hist_var_data *var_data;
 918         struct trace_event_file *file, *found = NULL;
 919
 920         if (system)
 921                 return find_event_file(tr, system, event_name);
 922
 923         list_for_each_entry(var_data, &tr->hist_vars, list) {
 924                 var_hist_data = var_data->hist_data;
 925                 file = var_hist_data->event_file;
 926                 if (file == found)
 927                         continue;
 928
 929                 if (find_var_field(var_hist_data, var_name)) {
 930                         if (found) {
 931                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
 932                                 return NULL;
 933                         }
 934
 935                         found = file;
 936                 }
 937         }
 938
 939         return found;
 940 }
 941
 942 static struct hist_field *find_file_var(struct trace_event_file *file,
 943                                         const char *var_name)
 944 {
 945         struct hist_trigger_data *test_data;
 946         struct event_trigger_data *test;
 947         struct hist_field *hist_field;
 948
 949         lockdep_assert_held(&event_mutex);
 950
 951         list_for_each_entry(test, &file->triggers, list) {
 952                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 953                         test_data = test->private_data;
 954                         hist_field = find_var_field(test_data, var_name);
 955                         if (hist_field)
 956                                 return hist_field;
 957                 }
 958         }
 959
 960         return NULL;
 961 }
 962
 963 static struct hist_field *
 964 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
 965 {
 966         struct trace_array *tr = hist_data->event_file->tr;
 967         struct hist_field *hist_field, *found = NULL;
 968         struct trace_event_file *file;
 969         unsigned int i;
 970
 971         for (i = 0; i < hist_data->n_actions; i++) {
 972                 struct action_data *data = hist_data->actions[i];
 973
 974                 if (data->handler == HANDLER_ONMATCH) {
 975                         char *system = data->match_data.event_system;
 976                         char *event_name = data->match_data.event;
 977
 978                         file = find_var_file(tr, system, event_name, var_name);
 979                         if (!file)
 980                                 continue;
 981                         hist_field = find_file_var(file, var_name);
 982                         if (hist_field) {
 983                                 if (found) {
 984                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
 985                                                  errpos(var_name));
 986                                         return ERR_PTR(-EINVAL);
 987                                 }
 988
 989                                 found = hist_field;
 990                         }
 991                 }
 992         }
 993         return found;
 994 }
 995
 996 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
 997                                          char *system,
 998                                          char *event_name,
 999                                          char *var_name)
1000 {
1001         struct trace_array *tr = hist_data->event_file->tr;
1002         struct hist_field *hist_field = NULL;
1003         struct trace_event_file *file;
1004
1005         if (!system || !event_name) {
1006                 hist_field = find_match_var(hist_data, var_name);
1007                 if (IS_ERR(hist_field))
1008                         return NULL;
1009                 if (hist_field)
1010                         return hist_field;
1011         }
1012
1013         file = find_var_file(tr, system, event_name, var_name);
1014         if (!file)
1015                 return NULL;
1016
1017         hist_field = find_file_var(file, var_name);
1018
1019         return hist_field;
1020 }
1021
1022 static u64 hist_field_var_ref(struct hist_field *hist_field,
1023                               struct tracing_map_elt *elt,
1024                               struct ring_buffer_event *rbe,
1025                               void *event)
1026 {
1027         struct hist_elt_data *elt_data;
1028         u64 var_val = 0;
1029
1030         if (WARN_ON_ONCE(!elt))
1031                 return var_val;
1032
1033         elt_data = elt->private_data;
1034         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1035
1036         return var_val;
1037 }
1038
1039 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1040                              u64 *var_ref_vals, bool self)
1041 {
1042         struct hist_trigger_data *var_data;
1043         struct tracing_map_elt *var_elt;
1044         struct hist_field *hist_field;
1045         unsigned int i, var_idx;
1046         bool resolved = true;
1047         u64 var_val = 0;
1048
1049         for (i = 0; i < hist_data->n_var_refs; i++) {
1050                 hist_field = hist_data->var_refs[i];
1051                 var_idx = hist_field->var.idx;
1052                 var_data = hist_field->var.hist_data;
1053
1054                 if (var_data == NULL) {
1055                         resolved = false;
1056                         break;
1057                 }
1058
1059                 if ((self && var_data != hist_data) ||
1060                     (!self && var_data == hist_data))
1061                         continue;
1062
1063                 var_elt = tracing_map_lookup(var_data->map, key);
1064                 if (!var_elt) {
1065                         resolved = false;
1066                         break;
1067                 }
1068
1069                 if (!tracing_map_var_set(var_elt, var_idx)) {
1070                         resolved = false;
1071                         break;
1072                 }
1073
1074                 if (self || !hist_field->read_once)
1075                         var_val = tracing_map_read_var(var_elt, var_idx);
1076                 else
1077                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1078
1079                 var_ref_vals[i] = var_val;
1080         }
1081
1082         return resolved;
1083 }
1084
1085 static const char *hist_field_name(struct hist_field *field,
1086                                    unsigned int level)
1087 {
1088         const char *field_name = "";
1089
1090         if (level > 1)
1091                 return field_name;
1092
1093         if (field->field)
1094                 field_name = field->field->name;
1095         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1096                  field->flags & HIST_FIELD_FL_ALIAS)
1097                 field_name = hist_field_name(field->operands[0], ++level);
1098         else if (field->flags & HIST_FIELD_FL_CPU)
1099                 field_name = "common_cpu";
1100         else if (field->flags & HIST_FIELD_FL_EXPR ||
1101                  field->flags & HIST_FIELD_FL_VAR_REF) {
1102                 if (field->system) {
1103                         static char full_name[MAX_FILTER_STR_VAL];
1104
1105                         strcat(full_name, field->system);
1106                         strcat(full_name, ".");
1107                         strcat(full_name, field->event_name);
1108                         strcat(full_name, ".");
1109                         strcat(full_name, field->name);
1110                         field_name = full_name;
1111                 } else
1112                         field_name = field->name;
1113         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1114                 field_name = "common_timestamp";
1115
1116         if (field_name == NULL)
1117                 field_name = "";
1118
1119         return field_name;
1120 }
1121
1122 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1123 {
1124         hist_field_fn_t fn = NULL;
1125
1126         switch (field_size) {
1127         case 8:
1128                 if (field_is_signed)
1129                         fn = hist_field_s64;
1130                 else
1131                         fn = hist_field_u64;
1132                 break;
1133         case 4:
1134                 if (field_is_signed)
1135                         fn = hist_field_s32;
1136                 else
1137                         fn = hist_field_u32;
1138                 break;
1139         case 2:
1140                 if (field_is_signed)
1141                         fn = hist_field_s16;
1142                 else
1143                         fn = hist_field_u16;
1144                 break;
1145         case 1:
1146                 if (field_is_signed)
1147                         fn = hist_field_s8;
1148                 else
1149                         fn = hist_field_u8;
1150                 break;
1151         }
1152
1153         return fn;
1154 }
1155
1156 static int parse_map_size(char *str)
1157 {
1158         unsigned long size, map_bits;
1159         int ret;
1160
1161         ret = kstrtoul(str, 0, &size);
1162         if (ret)
1163                 goto out;
1164
1165         map_bits = ilog2(roundup_pow_of_two(size));
1166         if (map_bits < TRACING_MAP_BITS_MIN ||
1167             map_bits > TRACING_MAP_BITS_MAX)
1168                 ret = -EINVAL;
1169         else
1170                 ret = map_bits;
1171  out:
1172         return ret;
1173 }
1174
1175 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1176 {
1177         unsigned int i;
1178
1179         if (!attrs)
1180                 return;
1181
1182         for (i = 0; i < attrs->n_assignments; i++)
1183                 kfree(attrs->assignment_str[i]);
1184
1185         for (i = 0; i < attrs->n_actions; i++)
1186                 kfree(attrs->action_str[i]);
1187
1188         kfree(attrs->name);
1189         kfree(attrs->sort_key_str);
1190         kfree(attrs->keys_str);
1191         kfree(attrs->vals_str);
1192         kfree(attrs->clock);
1193         kfree(attrs);
1194 }
1195
1196 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1197 {
1198         int ret = -EINVAL;
1199
1200         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1201                 return ret;
1202
1203         if ((str_has_prefix(str, "onmatch(")) ||
1204             (str_has_prefix(str, "onmax(")) ||
1205             (str_has_prefix(str, "onchange("))) {
1206                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1207                 if (!attrs->action_str[attrs->n_actions]) {
1208                         ret = -ENOMEM;
1209                         return ret;
1210                 }
1211                 attrs->n_actions++;
1212                 ret = 0;
1213         }
1214         return ret;
1215 }
1216
1217 static int parse_assignment(struct trace_array *tr,
1218                             char *str, struct hist_trigger_attrs *attrs)
1219 {
1220         int len, ret = 0;
1221
1222         if ((len = str_has_prefix(str, "key=")) ||
1223             (len = str_has_prefix(str, "keys="))) {
1224                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1225                 if (!attrs->keys_str) {
1226                         ret = -ENOMEM;
1227                         goto out;
1228                 }
1229         } else if ((len = str_has_prefix(str, "val=")) ||
1230                    (len = str_has_prefix(str, "vals=")) ||
1231                    (len = str_has_prefix(str, "values="))) {
1232                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1233                 if (!attrs->vals_str) {
1234                         ret = -ENOMEM;
1235                         goto out;
1236                 }
1237         } else if ((len = str_has_prefix(str, "sort="))) {
1238                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1239                 if (!attrs->sort_key_str) {
1240                         ret = -ENOMEM;
1241                         goto out;
1242                 }
1243         } else if (str_has_prefix(str, "name=")) {
1244                 attrs->name = kstrdup(str, GFP_KERNEL);
1245                 if (!attrs->name) {
1246                         ret = -ENOMEM;
1247                         goto out;
1248                 }
1249         } else if ((len = str_has_prefix(str, "clock="))) {
1250                 str += len;
1251
1252                 str = strstrip(str);
1253                 attrs->clock = kstrdup(str, GFP_KERNEL);
1254                 if (!attrs->clock) {
1255                         ret = -ENOMEM;
1256                         goto out;
1257                 }
1258         } else if ((len = str_has_prefix(str, "size="))) {
1259                 int map_bits = parse_map_size(str + len);
1260
1261                 if (map_bits < 0) {
1262                         ret = map_bits;
1263                         goto out;
1264                 }
1265                 attrs->map_bits = map_bits;
1266         } else {
1267                 char *assignment;
1268
1269                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1270                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1271                         ret = -EINVAL;
1272                         goto out;
1273                 }
1274
1275                 assignment = kstrdup(str, GFP_KERNEL);
1276                 if (!assignment) {
1277                         ret = -ENOMEM;
1278                         goto out;
1279                 }
1280
1281                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1282         }
1283  out:
1284         return ret;
1285 }
1286
1287 static struct hist_trigger_attrs *
1288 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1289 {
1290         struct hist_trigger_attrs *attrs;
1291         int ret = 0;
1292
1293         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1294         if (!attrs)
1295                 return ERR_PTR(-ENOMEM);
1296
1297         while (trigger_str) {
1298                 char *str = strsep(&trigger_str, ":");
1299                 char *rhs;
1300
1301                 rhs = strchr(str, '=');
1302                 if (rhs) {
1303                         if (!strlen(++rhs)) {
1304                                 ret = -EINVAL;
1305                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1306                                 goto free;
1307                         }
1308                         ret = parse_assignment(tr, str, attrs);
1309                         if (ret)
1310                                 goto free;
1311                 } else if (strcmp(str, "pause") == 0)
1312                         attrs->pause = true;
1313                 else if ((strcmp(str, "cont") == 0) ||
1314                          (strcmp(str, "continue") == 0))
1315                         attrs->cont = true;
1316                 else if (strcmp(str, "clear") == 0)
1317                         attrs->clear = true;
1318                 else {
1319                         ret = parse_action(str, attrs);
1320                         if (ret)
1321                                 goto free;
1322                 }
1323         }
1324
1325         if (!attrs->keys_str) {
1326                 ret = -EINVAL;
1327                 goto free;
1328         }
1329
1330         if (!attrs->clock) {
1331                 attrs->clock = kstrdup("global", GFP_KERNEL);
1332                 if (!attrs->clock) {
1333                         ret = -ENOMEM;
1334                         goto free;
1335                 }
1336         }
1337
1338         return attrs;
1339  free:
1340         destroy_hist_trigger_attrs(attrs);
1341
1342         return ERR_PTR(ret);
1343 }
1344
1345 static inline void save_comm(char *comm, struct task_struct *task)
1346 {
1347         if (!task->pid) {
1348                 strcpy(comm, "<idle>");
1349                 return;
1350         }
1351
1352         if (WARN_ON_ONCE(task->pid < 0)) {
1353                 strcpy(comm, "<XXX>");
1354                 return;
1355         }
1356
1357         strncpy(comm, task->comm, TASK_COMM_LEN);
1358 }
1359
1360 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1361 {
1362         unsigned int i;
1363
1364         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
1365                 kfree(elt_data->field_var_str[i]);
1366
1367         kfree(elt_data->comm);
1368         kfree(elt_data);
1369 }
1370
1371 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1372 {
1373         struct hist_elt_data *elt_data = elt->private_data;
1374
1375         hist_elt_data_free(elt_data);
1376 }
1377
1378 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1379 {
1380         struct hist_trigger_data *hist_data = elt->map->private_data;
1381         unsigned int size = TASK_COMM_LEN;
1382         struct hist_elt_data *elt_data;
1383         struct hist_field *key_field;
1384         unsigned int i, n_str;
1385
1386         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1387         if (!elt_data)
1388                 return -ENOMEM;
1389
1390         for_each_hist_key_field(i, hist_data) {
1391                 key_field = hist_data->fields[i];
1392
1393                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1394                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1395                         if (!elt_data->comm) {
1396                                 kfree(elt_data);
1397                                 return -ENOMEM;
1398                         }
1399                         break;
1400                 }
1401         }
1402
1403         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1404                 hist_data->n_var_str;
1405         if (n_str > SYNTH_FIELDS_MAX) {
1406                 hist_elt_data_free(elt_data);
1407                 return -EINVAL;
1408         }
1409
1410         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1411
1412         size = STR_VAR_LEN_MAX;
1413
1414         for (i = 0; i < n_str; i++) {
1415                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1416                 if (!elt_data->field_var_str[i]) {
1417                         hist_elt_data_free(elt_data);
1418                         return -ENOMEM;
1419                 }
1420         }
1421
1422         elt->private_data = elt_data;
1423
1424         return 0;
1425 }
1426
1427 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1428 {
1429         struct hist_elt_data *elt_data = elt->private_data;
1430
1431         if (elt_data->comm)
1432                 save_comm(elt_data->comm, current);
1433 }
1434
1435 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1436         .elt_alloc      = hist_trigger_elt_data_alloc,
1437         .elt_free       = hist_trigger_elt_data_free,
1438         .elt_init       = hist_trigger_elt_data_init,
1439 };
1440
1441 static const char *get_hist_field_flags(struct hist_field *hist_field)
1442 {
1443         const char *flags_str = NULL;
1444
1445         if (hist_field->flags & HIST_FIELD_FL_HEX)
1446                 flags_str = "hex";
1447         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1448                 flags_str = "sym";
1449         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1450                 flags_str = "sym-offset";
1451         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1452                 flags_str = "execname";
1453         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1454                 flags_str = "syscall";
1455         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1456                 flags_str = "log2";
1457         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1458                 flags_str = "usecs";
1459
1460         return flags_str;
1461 }
1462
1463 static void expr_field_str(struct hist_field *field, char *expr)
1464 {
1465         if (field->flags & HIST_FIELD_FL_VAR_REF)
1466                 strcat(expr, "$");
1467
1468         strcat(expr, hist_field_name(field, 0));
1469
1470         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1471                 const char *flags_str = get_hist_field_flags(field);
1472
1473                 if (flags_str) {
1474                         strcat(expr, ".");
1475                         strcat(expr, flags_str);
1476                 }
1477         }
1478 }
1479
1480 static char *expr_str(struct hist_field *field, unsigned int level)
1481 {
1482         char *expr;
1483
1484         if (level > 1)
1485                 return NULL;
1486
1487         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1488         if (!expr)
1489                 return NULL;
1490
1491         if (!field->operands[0]) {
1492                 expr_field_str(field, expr);
1493                 return expr;
1494         }
1495
1496         if (field->operator == FIELD_OP_UNARY_MINUS) {
1497                 char *subexpr;
1498
1499                 strcat(expr, "-(");
1500                 subexpr = expr_str(field->operands[0], ++level);
1501                 if (!subexpr) {
1502                         kfree(expr);
1503                         return NULL;
1504                 }
1505                 strcat(expr, subexpr);
1506                 strcat(expr, ")");
1507
1508                 kfree(subexpr);
1509
1510                 return expr;
1511         }
1512
1513         expr_field_str(field->operands[0], expr);
1514
1515         switch (field->operator) {
1516         case FIELD_OP_MINUS:
1517                 strcat(expr, "-");
1518                 break;
1519         case FIELD_OP_PLUS:
1520                 strcat(expr, "+");
1521                 break;
1522         default:
1523                 kfree(expr);
1524                 return NULL;
1525         }
1526
1527         expr_field_str(field->operands[1], expr);
1528
1529         return expr;
1530 }
1531
1532 static int contains_operator(char *str)
1533 {
1534         enum field_op_id field_op = FIELD_OP_NONE;
1535         char *op;
1536
1537         op = strpbrk(str, "+-");
1538         if (!op)
1539                 return FIELD_OP_NONE;
1540
1541         switch (*op) {
1542         case '-':
1543                 /*
1544                  * Unfortunately, the modifier ".sym-offset"
1545                  * can confuse things.
1546                  */
1547                 if (op - str >= 4 && !strncmp(op - 4, ".sym-offset", 11))
1548                         return FIELD_OP_NONE;
1549
1550                 if (*str == '-')
1551                         field_op = FIELD_OP_UNARY_MINUS;
1552                 else
1553                         field_op = FIELD_OP_MINUS;
1554                 break;
1555         case '+':
1556                 field_op = FIELD_OP_PLUS;
1557                 break;
1558         default:
1559                 break;
1560         }
1561
1562         return field_op;
1563 }
1564
1565 static void get_hist_field(struct hist_field *hist_field)
1566 {
1567         hist_field->ref++;
1568 }
1569
1570 static void __destroy_hist_field(struct hist_field *hist_field)
1571 {
1572         if (--hist_field->ref > 1)
1573                 return;
1574
1575         kfree(hist_field->var.name);
1576         kfree(hist_field->name);
1577         kfree(hist_field->type);
1578
1579         kfree(hist_field->system);
1580         kfree(hist_field->event_name);
1581
1582         kfree(hist_field);
1583 }
1584
1585 static void destroy_hist_field(struct hist_field *hist_field,
1586                                unsigned int level)
1587 {
1588         unsigned int i;
1589
1590         if (level > 3)
1591                 return;
1592
1593         if (!hist_field)
1594                 return;
1595
1596         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1597                 return; /* var refs will be destroyed separately */
1598
1599         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1600                 destroy_hist_field(hist_field->operands[i], level + 1);
1601
1602         __destroy_hist_field(hist_field);
1603 }
1604
1605 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1606                                             struct ftrace_event_field *field,
1607                                             unsigned long flags,
1608                                             char *var_name)
1609 {
1610         struct hist_field *hist_field;
1611
1612         if (field && is_function_field(field))
1613                 return NULL;
1614
1615         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1616         if (!hist_field)
1617                 return NULL;
1618
1619         hist_field->ref = 1;
1620
1621         hist_field->hist_data = hist_data;
1622
1623         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1624                 goto out; /* caller will populate */
1625
1626         if (flags & HIST_FIELD_FL_VAR_REF) {
1627                 hist_field->fn = hist_field_var_ref;
1628                 goto out;
1629         }
1630
1631         if (flags & HIST_FIELD_FL_HITCOUNT) {
1632                 hist_field->fn = hist_field_counter;
1633                 hist_field->size = sizeof(u64);
1634                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1635                 if (!hist_field->type)
1636                         goto free;
1637                 goto out;
1638         }
1639
1640         if (flags & HIST_FIELD_FL_STACKTRACE) {
1641                 hist_field->fn = hist_field_none;
1642                 goto out;
1643         }
1644
1645         if (flags & HIST_FIELD_FL_LOG2) {
1646                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
1647                 hist_field->fn = hist_field_log2;
1648                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1649                 hist_field->size = hist_field->operands[0]->size;
1650                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
1651                 if (!hist_field->type)
1652                         goto free;
1653                 goto out;
1654         }
1655
1656         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1657                 hist_field->fn = hist_field_timestamp;
1658                 hist_field->size = sizeof(u64);
1659                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1660                 if (!hist_field->type)
1661                         goto free;
1662                 goto out;
1663         }
1664
1665         if (flags & HIST_FIELD_FL_CPU) {
1666                 hist_field->fn = hist_field_cpu;
1667                 hist_field->size = sizeof(int);
1668                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
1669                 if (!hist_field->type)
1670                         goto free;
1671                 goto out;
1672         }
1673
1674         if (WARN_ON_ONCE(!field))
1675                 goto out;
1676
1677         /* Pointers to strings are just pointers and dangerous to dereference */
1678         if (is_string_field(field) &&
1679             (field->filter_type != FILTER_PTR_STRING)) {
1680                 flags |= HIST_FIELD_FL_STRING;
1681
1682                 hist_field->size = MAX_FILTER_STR_VAL;
1683                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1684                 if (!hist_field->type)
1685                         goto free;
1686
1687                 if (field->filter_type == FILTER_STATIC_STRING) {
1688                         hist_field->fn = hist_field_string;
1689                         hist_field->size = field->size;
1690                 } else if (field->filter_type == FILTER_DYN_STRING)
1691                         hist_field->fn = hist_field_dynstring;
1692                 else
1693                         hist_field->fn = hist_field_pstring;
1694         } else {
1695                 hist_field->size = field->size;
1696                 hist_field->is_signed = field->is_signed;
1697                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
1698                 if (!hist_field->type)
1699                         goto free;
1700
1701                 hist_field->fn = select_value_fn(field->size,
1702                                                  field->is_signed);
1703                 if (!hist_field->fn) {
1704                         destroy_hist_field(hist_field, 0);
1705                         return NULL;
1706                 }
1707         }
1708  out:
1709         hist_field->field = field;
1710         hist_field->flags = flags;
1711
1712         if (var_name) {
1713                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1714                 if (!hist_field->var.name)
1715                         goto free;
1716         }
1717
1718         return hist_field;
1719  free:
1720         destroy_hist_field(hist_field, 0);
1721         return NULL;
1722 }
1723
1724 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1725 {
1726         unsigned int i;
1727
1728         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1729                 if (hist_data->fields[i]) {
1730                         destroy_hist_field(hist_data->fields[i], 0);
1731                         hist_data->fields[i] = NULL;
1732                 }
1733         }
1734
1735         for (i = 0; i < hist_data->n_var_refs; i++) {
1736                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1737                 __destroy_hist_field(hist_data->var_refs[i]);
1738                 hist_data->var_refs[i] = NULL;
1739         }
1740 }
1741
1742 static int init_var_ref(struct hist_field *ref_field,
1743                         struct hist_field *var_field,
1744                         char *system, char *event_name)
1745 {
1746         int err = 0;
1747
1748         ref_field->var.idx = var_field->var.idx;
1749         ref_field->var.hist_data = var_field->hist_data;
1750         ref_field->size = var_field->size;
1751         ref_field->is_signed = var_field->is_signed;
1752         ref_field->flags |= var_field->flags &
1753                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1754
1755         if (system) {
1756                 ref_field->system = kstrdup(system, GFP_KERNEL);
1757                 if (!ref_field->system)
1758                         return -ENOMEM;
1759         }
1760
1761         if (event_name) {
1762                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1763                 if (!ref_field->event_name) {
1764                         err = -ENOMEM;
1765                         goto free;
1766                 }
1767         }
1768
1769         if (var_field->var.name) {
1770                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1771                 if (!ref_field->name) {
1772                         err = -ENOMEM;
1773                         goto free;
1774                 }
1775         } else if (var_field->name) {
1776                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1777                 if (!ref_field->name) {
1778                         err = -ENOMEM;
1779                         goto free;
1780                 }
1781         }
1782
1783         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
1784         if (!ref_field->type) {
1785                 err = -ENOMEM;
1786                 goto free;
1787         }
1788  out:
1789         return err;
1790  free:
1791         kfree(ref_field->system);
1792         kfree(ref_field->event_name);
1793         kfree(ref_field->name);
1794
1795         goto out;
1796 }
1797
1798 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1799                             struct hist_field *var_field)
1800 {
1801         struct hist_field *ref_field;
1802         int i;
1803
1804         for (i = 0; i < hist_data->n_var_refs; i++) {
1805                 ref_field = hist_data->var_refs[i];
1806                 if (ref_field->var.idx == var_field->var.idx &&
1807                     ref_field->var.hist_data == var_field->hist_data)
1808                         return i;
1809         }
1810
1811         return -ENOENT;
1812 }
1813
1814 /**
1815  * create_var_ref - Create a variable reference and attach it to trigger
1816  * @hist_data: The trigger that will be referencing the variable
1817  * @var_field: The VAR field to create a reference to
1818  * @system: The optional system string
1819  * @event_name: The optional event_name string
1820  *
1821  * Given a variable hist_field, create a VAR_REF hist_field that
1822  * represents a reference to it.
1823  *
1824  * This function also adds the reference to the trigger that
1825  * now references the variable.
1826  *
1827  * Return: The VAR_REF field if successful, NULL if not
1828  */
1829 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1830                                          struct hist_field *var_field,
1831                                          char *system, char *event_name)
1832 {
1833         unsigned long flags = HIST_FIELD_FL_VAR_REF;
1834         struct hist_field *ref_field;
1835         int i;
1836
1837         /* Check if the variable already exists */
1838         for (i = 0; i < hist_data->n_var_refs; i++) {
1839                 ref_field = hist_data->var_refs[i];
1840                 if (ref_field->var.idx == var_field->var.idx &&
1841                     ref_field->var.hist_data == var_field->hist_data) {
1842                         get_hist_field(ref_field);
1843                         return ref_field;
1844                 }
1845         }
1846
1847         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1848         if (ref_field) {
1849                 if (init_var_ref(ref_field, var_field, system, event_name)) {
1850                         destroy_hist_field(ref_field, 0);
1851                         return NULL;
1852                 }
1853
1854                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1855                 ref_field->var_ref_idx = hist_data->n_var_refs++;
1856         }
1857
1858         return ref_field;
1859 }
1860
1861 static bool is_var_ref(char *var_name)
1862 {
1863         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1864                 return false;
1865
1866         return true;
1867 }
1868
1869 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1870                                  char *var_name)
1871 {
1872         char *name, *field;
1873         unsigned int i;
1874
1875         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1876                 name = hist_data->attrs->var_defs.name[i];
1877
1878                 if (strcmp(var_name, name) == 0) {
1879                         field = hist_data->attrs->var_defs.expr[i];
1880                         if (contains_operator(field) || is_var_ref(field))
1881                                 continue;
1882                         return field;
1883                 }
1884         }
1885
1886         return NULL;
1887 }
1888
1889 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1890                                  char *system, char *event_name,
1891                                  char *var_name)
1892 {
1893         struct trace_event_call *call;
1894
1895         if (system && event_name) {
1896                 call = hist_data->event_file->event_call;
1897
1898                 if (strcmp(system, call->class->system) != 0)
1899                         return NULL;
1900
1901                 if (strcmp(event_name, trace_event_name(call)) != 0)
1902                         return NULL;
1903         }
1904
1905         if (!!system != !!event_name)
1906                 return NULL;
1907
1908         if (!is_var_ref(var_name))
1909                 return NULL;
1910
1911         var_name++;
1912
1913         return field_name_from_var(hist_data, var_name);
1914 }
1915
1916 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1917                                         char *system, char *event_name,
1918                                         char *var_name)
1919 {
1920         struct hist_field *var_field = NULL, *ref_field = NULL;
1921         struct trace_array *tr = hist_data->event_file->tr;
1922
1923         if (!is_var_ref(var_name))
1924                 return NULL;
1925
1926         var_name++;
1927
1928         var_field = find_event_var(hist_data, system, event_name, var_name);
1929         if (var_field)
1930                 ref_field = create_var_ref(hist_data, var_field,
1931                                            system, event_name);
1932
1933         if (!ref_field)
1934                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1935
1936         return ref_field;
1937 }
1938
1939 static struct ftrace_event_field *
1940 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1941             char *field_str, unsigned long *flags)
1942 {
1943         struct ftrace_event_field *field = NULL;
1944         char *field_name, *modifier, *str;
1945         struct trace_array *tr = file->tr;
1946
1947         modifier = str = kstrdup(field_str, GFP_KERNEL);
1948         if (!modifier)
1949                 return ERR_PTR(-ENOMEM);
1950
1951         field_name = strsep(&modifier, ".");
1952         if (modifier) {
1953                 if (strcmp(modifier, "hex") == 0)
1954                         *flags |= HIST_FIELD_FL_HEX;
1955                 else if (strcmp(modifier, "sym") == 0)
1956                         *flags |= HIST_FIELD_FL_SYM;
1957                 else if (strcmp(modifier, "sym-offset") == 0)
1958                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
1959                 else if ((strcmp(modifier, "execname") == 0) &&
1960                          (strcmp(field_name, "common_pid") == 0))
1961                         *flags |= HIST_FIELD_FL_EXECNAME;
1962                 else if (strcmp(modifier, "syscall") == 0)
1963                         *flags |= HIST_FIELD_FL_SYSCALL;
1964                 else if (strcmp(modifier, "log2") == 0)
1965                         *flags |= HIST_FIELD_FL_LOG2;
1966                 else if (strcmp(modifier, "usecs") == 0)
1967                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
1968                 else {
1969                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
1970                         field = ERR_PTR(-EINVAL);
1971                         goto out;
1972                 }
1973         }
1974
1975         if (strcmp(field_name, "common_timestamp") == 0) {
1976                 *flags |= HIST_FIELD_FL_TIMESTAMP;
1977                 hist_data->enable_timestamps = true;
1978                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1979                         hist_data->attrs->ts_in_usecs = true;
1980         } else if (strcmp(field_name, "common_cpu") == 0)
1981                 *flags |= HIST_FIELD_FL_CPU;
1982         else {
1983                 field = trace_find_event_field(file->event_call, field_name);
1984                 if (!field || !field->size) {
1985                         /*
1986                          * For backward compatibility, if field_name
1987                          * was "cpu", then we treat this the same as
1988                          * common_cpu.
1989                          */
1990                         if (strcmp(field_name, "cpu") == 0) {
1991                                 *flags |= HIST_FIELD_FL_CPU;
1992                         } else {
1993                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
1994                                          errpos(field_name));
1995                                 field = ERR_PTR(-EINVAL);
1996                                 goto out;
1997                         }
1998                 }
1999         }
2000  out:
2001         kfree(str);
2002
2003         return field;
2004 }
2005
2006 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2007                                        struct hist_field *var_ref,
2008                                        char *var_name)
2009 {
2010         struct hist_field *alias = NULL;
2011         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2012
2013         alias = create_hist_field(hist_data, NULL, flags, var_name);
2014         if (!alias)
2015                 return NULL;
2016
2017         alias->fn = var_ref->fn;
2018         alias->operands[0] = var_ref;
2019
2020         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2021                 destroy_hist_field(alias, 0);
2022                 return NULL;
2023         }
2024
2025         alias->var_ref_idx = var_ref->var_ref_idx;
2026
2027         return alias;
2028 }
2029
2030 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2031                                      struct trace_event_file *file, char *str,
2032                                      unsigned long *flags, char *var_name)
2033 {
2034         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2035         struct ftrace_event_field *field = NULL;
2036         struct hist_field *hist_field = NULL;
2037         int ret = 0;
2038
2039         s = strchr(str, '.');
2040         if (s) {
2041                 s = strchr(++s, '.');
2042                 if (s) {
2043                         ref_system = strsep(&str, ".");
2044                         if (!str) {
2045                                 ret = -EINVAL;
2046                                 goto out;
2047                         }
2048                         ref_event = strsep(&str, ".");
2049                         if (!str) {
2050                                 ret = -EINVAL;
2051                                 goto out;
2052                         }
2053                         ref_var = str;
2054                 }
2055         }
2056
2057         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2058         if (!s) {
2059                 hist_field = parse_var_ref(hist_data, ref_system,
2060                                            ref_event, ref_var);
2061                 if (hist_field) {
2062                         if (var_name) {
2063                                 hist_field = create_alias(hist_data, hist_field, var_name);
2064                                 if (!hist_field) {
2065                                         ret = -ENOMEM;
2066                                         goto out;
2067                                 }
2068                         }
2069                         return hist_field;
2070                 }
2071         } else
2072                 str = s;
2073
2074         field = parse_field(hist_data, file, str, flags);
2075         if (IS_ERR(field)) {
2076                 ret = PTR_ERR(field);
2077                 goto out;
2078         }
2079
2080         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2081         if (!hist_field) {
2082                 ret = -ENOMEM;
2083                 goto out;
2084         }
2085
2086         return hist_field;
2087  out:
2088         return ERR_PTR(ret);
2089 }
2090
2091 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2092                                      struct trace_event_file *file,
2093                                      char *str, unsigned long flags,
2094                                      char *var_name, unsigned int level);
2095
2096 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2097                                       struct trace_event_file *file,
2098                                       char *str, unsigned long flags,
2099                                       char *var_name, unsigned int level)
2100 {
2101         struct hist_field *operand1, *expr = NULL;
2102         unsigned long operand_flags;
2103         int ret = 0;
2104         char *s;
2105
2106         /* we support only -(xxx) i.e. explicit parens required */
2107
2108         if (level > 3) {
2109                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2110                 ret = -EINVAL;
2111                 goto free;
2112         }
2113
2114         str++; /* skip leading '-' */
2115
2116         s = strchr(str, '(');
2117         if (s)
2118                 str++;
2119         else {
2120                 ret = -EINVAL;
2121                 goto free;
2122         }
2123
2124         s = strrchr(str, ')');
2125         if (s)
2126                 *s = '\0';
2127         else {
2128                 ret = -EINVAL; /* no closing ')' */
2129                 goto free;
2130         }
2131
2132         flags |= HIST_FIELD_FL_EXPR;
2133         expr = create_hist_field(hist_data, NULL, flags, var_name);
2134         if (!expr) {
2135                 ret = -ENOMEM;
2136                 goto free;
2137         }
2138
2139         operand_flags = 0;
2140         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2141         if (IS_ERR(operand1)) {
2142                 ret = PTR_ERR(operand1);
2143                 goto free;
2144         }
2145         if (operand1->flags & HIST_FIELD_FL_STRING) {
2146                 /* String type can not be the operand of unary operator. */
2147                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2148                 destroy_hist_field(operand1, 0);
2149                 ret = -EINVAL;
2150                 goto free;
2151         }
2152
2153         expr->flags |= operand1->flags &
2154                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2155         expr->fn = hist_field_unary_minus;
2156         expr->operands[0] = operand1;
2157         expr->operator = FIELD_OP_UNARY_MINUS;
2158         expr->name = expr_str(expr, 0);
2159         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2160         if (!expr->type) {
2161                 ret = -ENOMEM;
2162                 goto free;
2163         }
2164
2165         return expr;
2166  free:
2167         destroy_hist_field(expr, 0);
2168         return ERR_PTR(ret);
2169 }
2170
2171 static int check_expr_operands(struct trace_array *tr,
2172                                struct hist_field *operand1,
2173                                struct hist_field *operand2)
2174 {
2175         unsigned long operand1_flags = operand1->flags;
2176         unsigned long operand2_flags = operand2->flags;
2177
2178         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2179             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2180                 struct hist_field *var;
2181
2182                 var = find_var_field(operand1->var.hist_data, operand1->name);
2183                 if (!var)
2184                         return -EINVAL;
2185                 operand1_flags = var->flags;
2186         }
2187
2188         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2189             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2190                 struct hist_field *var;
2191
2192                 var = find_var_field(operand2->var.hist_data, operand2->name);
2193                 if (!var)
2194                         return -EINVAL;
2195                 operand2_flags = var->flags;
2196         }
2197
2198         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2199             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2200                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2201                 return -EINVAL;
2202         }
2203
2204         return 0;
2205 }
2206
2207 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2208                                      struct trace_event_file *file,
2209                                      char *str, unsigned long flags,
2210                                      char *var_name, unsigned int level)
2211 {
2212         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2213         unsigned long operand_flags;
2214         int field_op, ret = -EINVAL;
2215         char *sep, *operand1_str;
2216
2217         if (level > 3) {
2218                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2219                 return ERR_PTR(-EINVAL);
2220         }
2221
2222         field_op = contains_operator(str);
2223
2224         if (field_op == FIELD_OP_NONE)
2225                 return parse_atom(hist_data, file, str, &flags, var_name);
2226
2227         if (field_op == FIELD_OP_UNARY_MINUS)
2228                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2229
2230         switch (field_op) {
2231         case FIELD_OP_MINUS:
2232                 sep = "-";
2233                 break;
2234         case FIELD_OP_PLUS:
2235                 sep = "+";
2236                 break;
2237         default:
2238                 goto free;
2239         }
2240
2241         operand1_str = strsep(&str, sep);
2242         if (!operand1_str || !str)
2243                 goto free;
2244
2245         operand_flags = 0;
2246         operand1 = parse_atom(hist_data, file, operand1_str,
2247                               &operand_flags, NULL);
2248         if (IS_ERR(operand1)) {
2249                 ret = PTR_ERR(operand1);
2250                 operand1 = NULL;
2251                 goto free;
2252         }
2253         if (operand1->flags & HIST_FIELD_FL_STRING) {
2254                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2255                 ret = -EINVAL;
2256                 goto free;
2257         }
2258
2259         /* rest of string could be another expression e.g. b+c in a+b+c */
2260         operand_flags = 0;
2261         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2262         if (IS_ERR(operand2)) {
2263                 ret = PTR_ERR(operand2);
2264                 operand2 = NULL;
2265                 goto free;
2266         }
2267         if (operand2->flags & HIST_FIELD_FL_STRING) {
2268                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2269                 ret = -EINVAL;
2270                 goto free;
2271         }
2272
2273         ret = check_expr_operands(file->tr, operand1, operand2);
2274         if (ret)
2275                 goto free;
2276
2277         flags |= HIST_FIELD_FL_EXPR;
2278
2279         flags |= operand1->flags &
2280                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2281
2282         expr = create_hist_field(hist_data, NULL, flags, var_name);
2283         if (!expr) {
2284                 ret = -ENOMEM;
2285                 goto free;
2286         }
2287
2288         operand1->read_once = true;
2289         operand2->read_once = true;
2290
2291         expr->operands[0] = operand1;
2292         expr->operands[1] = operand2;
2293
2294         /* The operand sizes should be the same, so just pick one */
2295         expr->size = operand1->size;
2296
2297         expr->operator = field_op;
2298         expr->name = expr_str(expr, 0);
2299         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2300         if (!expr->type) {
2301                 ret = -ENOMEM;
2302                 goto free;
2303         }
2304
2305         switch (field_op) {
2306         case FIELD_OP_MINUS:
2307                 expr->fn = hist_field_minus;
2308                 break;
2309         case FIELD_OP_PLUS:
2310                 expr->fn = hist_field_plus;
2311                 break;
2312         default:
2313                 ret = -EINVAL;
2314                 goto free;
2315         }
2316
2317         return expr;
2318  free:
2319         destroy_hist_field(operand1, 0);
2320         destroy_hist_field(operand2, 0);
2321         destroy_hist_field(expr, 0);
2322
2323         return ERR_PTR(ret);
2324 }
2325
2326 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2327                                  struct trace_event_file *file)
2328 {
2329         struct event_trigger_data *test;
2330
2331         lockdep_assert_held(&event_mutex);
2332
2333         list_for_each_entry(test, &file->triggers, list) {
2334                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2335                         if (test->private_data == hist_data)
2336                                 return test->filter_str;
2337                 }
2338         }
2339
2340         return NULL;
2341 }
2342
2343 static struct event_command trigger_hist_cmd;
2344 static int event_hist_trigger_func(struct event_command *cmd_ops,
2345                                    struct trace_event_file *file,
2346                                    char *glob, char *cmd, char *param);
2347
2348 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2349                             struct hist_trigger_data *hist_data,
2350                             unsigned int n_keys)
2351 {
2352         struct hist_field *target_hist_field, *hist_field;
2353         unsigned int n, i, j;
2354
2355         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2356                 return false;
2357
2358         i = hist_data->n_vals;
2359         j = target_hist_data->n_vals;
2360
2361         for (n = 0; n < n_keys; n++) {
2362                 hist_field = hist_data->fields[i + n];
2363                 target_hist_field = target_hist_data->fields[j + n];
2364
2365                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2366                         return false;
2367                 if (hist_field->size != target_hist_field->size)
2368                         return false;
2369                 if (hist_field->is_signed != target_hist_field->is_signed)
2370                         return false;
2371         }
2372
2373         return true;
2374 }
2375
2376 static struct hist_trigger_data *
2377 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2378                      struct trace_event_file *file)
2379 {
2380         struct hist_trigger_data *hist_data;
2381         struct event_trigger_data *test;
2382         unsigned int n_keys;
2383
2384         lockdep_assert_held(&event_mutex);
2385
2386         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2387
2388         list_for_each_entry(test, &file->triggers, list) {
2389                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2390                         hist_data = test->private_data;
2391
2392                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2393                                 return hist_data;
2394                 }
2395         }
2396
2397         return NULL;
2398 }
2399
2400 static struct trace_event_file *event_file(struct trace_array *tr,
2401                                            char *system, char *event_name)
2402 {
2403         struct trace_event_file *file;
2404
2405         file = __find_event_file(tr, system, event_name);
2406         if (!file)
2407                 return ERR_PTR(-EINVAL);
2408
2409         return file;
2410 }
2411
2412 static struct hist_field *
2413 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2414                          char *system, char *event_name, char *field_name)
2415 {
2416         struct hist_field *event_var;
2417         char *synthetic_name;
2418
2419         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2420         if (!synthetic_name)
2421                 return ERR_PTR(-ENOMEM);
2422
2423         strcpy(synthetic_name, "synthetic_");
2424         strcat(synthetic_name, field_name);
2425
2426         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2427
2428         kfree(synthetic_name);
2429
2430         return event_var;
2431 }
2432
2433 /**
2434  * create_field_var_hist - Automatically create a histogram and var for a field
2435  * @target_hist_data: The target hist trigger
2436  * @subsys_name: Optional subsystem name
2437  * @event_name: Optional event name
2438  * @field_name: The name of the field (and the resulting variable)
2439  *
2440  * Hist trigger actions fetch data from variables, not directly from
2441  * events.  However, for convenience, users are allowed to directly
2442  * specify an event field in an action, which will be automatically
2443  * converted into a variable on their behalf.
2444
2445  * If a user specifies a field on an event that isn't the event the
2446  * histogram currently being defined (the target event histogram), the
2447  * only way that can be accomplished is if a new hist trigger is
2448  * created and the field variable defined on that.
2449  *
2450  * This function creates a new histogram compatible with the target
2451  * event (meaning a histogram with the same key as the target
2452  * histogram), and creates a variable for the specified field, but
2453  * with 'synthetic_' prepended to the variable name in order to avoid
2454  * collision with normal field variables.
2455  *
2456  * Return: The variable created for the field.
2457  */
2458 static struct hist_field *
2459 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2460                       char *subsys_name, char *event_name, char *field_name)
2461 {
2462         struct trace_array *tr = target_hist_data->event_file->tr;
2463         struct hist_field *event_var = ERR_PTR(-EINVAL);
2464         struct hist_trigger_data *hist_data;
2465         unsigned int i, n, first = true;
2466         struct field_var_hist *var_hist;
2467         struct trace_event_file *file;
2468         struct hist_field *key_field;
2469         char *saved_filter;
2470         char *cmd;
2471         int ret;
2472
2473         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2474                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2475                 return ERR_PTR(-EINVAL);
2476         }
2477
2478         file = event_file(tr, subsys_name, event_name);
2479
2480         if (IS_ERR(file)) {
2481                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2482                 ret = PTR_ERR(file);
2483                 return ERR_PTR(ret);
2484         }
2485
2486         /*
2487          * Look for a histogram compatible with target.  We'll use the
2488          * found histogram specification to create a new matching
2489          * histogram with our variable on it.  target_hist_data is not
2490          * yet a registered histogram so we can't use that.
2491          */
2492         hist_data = find_compatible_hist(target_hist_data, file);
2493         if (!hist_data) {
2494                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2495                 return ERR_PTR(-EINVAL);
2496         }
2497
2498         /* See if a synthetic field variable has already been created */
2499         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2500                                              event_name, field_name);
2501         if (!IS_ERR_OR_NULL(event_var))
2502                 return event_var;
2503
2504         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2505         if (!var_hist)
2506                 return ERR_PTR(-ENOMEM);
2507
2508         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2509         if (!cmd) {
2510                 kfree(var_hist);
2511                 return ERR_PTR(-ENOMEM);
2512         }
2513
2514         /* Use the same keys as the compatible histogram */
2515         strcat(cmd, "keys=");
2516
2517         for_each_hist_key_field(i, hist_data) {
2518                 key_field = hist_data->fields[i];
2519                 if (!first)
2520                         strcat(cmd, ",");
2521                 strcat(cmd, key_field->field->name);
2522                 first = false;
2523         }
2524
2525         /* Create the synthetic field variable specification */
2526         strcat(cmd, ":synthetic_");
2527         strcat(cmd, field_name);
2528         strcat(cmd, "=");
2529         strcat(cmd, field_name);
2530
2531         /* Use the same filter as the compatible histogram */
2532         saved_filter = find_trigger_filter(hist_data, file);
2533         if (saved_filter) {
2534                 strcat(cmd, " if ");
2535                 strcat(cmd, saved_filter);
2536         }
2537
2538         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2539         if (!var_hist->cmd) {
2540                 kfree(cmd);
2541                 kfree(var_hist);
2542                 return ERR_PTR(-ENOMEM);
2543         }
2544
2545         /* Save the compatible histogram information */
2546         var_hist->hist_data = hist_data;
2547
2548         /* Create the new histogram with our variable */
2549         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2550                                       "", "hist", cmd);
2551         if (ret) {
2552                 kfree(cmd);
2553                 kfree(var_hist->cmd);
2554                 kfree(var_hist);
2555                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2556                 return ERR_PTR(ret);
2557         }
2558
2559         kfree(cmd);
2560
2561         /* If we can't find the variable, something went wrong */
2562         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2563                                              event_name, field_name);
2564         if (IS_ERR_OR_NULL(event_var)) {
2565                 kfree(var_hist->cmd);
2566                 kfree(var_hist);
2567                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2568                 return ERR_PTR(-EINVAL);
2569         }
2570
2571         n = target_hist_data->n_field_var_hists;
2572         target_hist_data->field_var_hists[n] = var_hist;
2573         target_hist_data->n_field_var_hists++;
2574
2575         return event_var;
2576 }
2577
2578 static struct hist_field *
2579 find_target_event_var(struct hist_trigger_data *hist_data,
2580                       char *subsys_name, char *event_name, char *var_name)
2581 {
2582         struct trace_event_file *file = hist_data->event_file;
2583         struct hist_field *hist_field = NULL;
2584
2585         if (subsys_name) {
2586                 struct trace_event_call *call;
2587
2588                 if (!event_name)
2589                         return NULL;
2590
2591                 call = file->event_call;
2592
2593                 if (strcmp(subsys_name, call->class->system) != 0)
2594                         return NULL;
2595
2596                 if (strcmp(event_name, trace_event_name(call)) != 0)
2597                         return NULL;
2598         }
2599
2600         hist_field = find_var_field(hist_data, var_name);
2601
2602         return hist_field;
2603 }
2604
2605 static inline void __update_field_vars(struct tracing_map_elt *elt,
2606                                        struct ring_buffer_event *rbe,
2607                                        void *rec,
2608                                        struct field_var **field_vars,
2609                                        unsigned int n_field_vars,
2610                                        unsigned int field_var_str_start)
2611 {
2612         struct hist_elt_data *elt_data = elt->private_data;
2613         unsigned int i, j, var_idx;
2614         u64 var_val;
2615
2616         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2617                 struct field_var *field_var = field_vars[i];
2618                 struct hist_field *var = field_var->var;
2619                 struct hist_field *val = field_var->val;
2620
2621                 var_val = val->fn(val, elt, rbe, rec);
2622                 var_idx = var->var.idx;
2623
2624                 if (val->flags & HIST_FIELD_FL_STRING) {
2625                         char *str = elt_data->field_var_str[j++];
2626                         char *val_str = (char *)(uintptr_t)var_val;
2627
2628                         strscpy(str, val_str, val->size);
2629                         var_val = (u64)(uintptr_t)str;
2630                 }
2631                 tracing_map_set_var(elt, var_idx, var_val);
2632         }
2633 }
2634
2635 static void update_field_vars(struct hist_trigger_data *hist_data,
2636                               struct tracing_map_elt *elt,
2637                               struct ring_buffer_event *rbe,
2638                               void *rec)
2639 {
2640         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
2641                             hist_data->n_field_vars, 0);
2642 }
2643
2644 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2645                                  struct tracing_map_elt *elt, void *rec,
2646                                  struct ring_buffer_event *rbe, void *key,
2647                                  struct action_data *data, u64 *var_ref_vals)
2648 {
2649         __update_field_vars(elt, rbe, rec, hist_data->save_vars,
2650                             hist_data->n_save_vars, hist_data->n_field_var_str);
2651 }
2652
2653 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2654                                      struct trace_event_file *file,
2655                                      char *name, int size, const char *type)
2656 {
2657         struct hist_field *var;
2658         int idx;
2659
2660         if (find_var(hist_data, file, name) && !hist_data->remove) {
2661                 var = ERR_PTR(-EINVAL);
2662                 goto out;
2663         }
2664
2665         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2666         if (!var) {
2667                 var = ERR_PTR(-ENOMEM);
2668                 goto out;
2669         }
2670
2671         idx = tracing_map_add_var(hist_data->map);
2672         if (idx < 0) {
2673                 kfree(var);
2674                 var = ERR_PTR(-EINVAL);
2675                 goto out;
2676         }
2677
2678         var->ref = 1;
2679         var->flags = HIST_FIELD_FL_VAR;
2680         var->var.idx = idx;
2681         var->var.hist_data = var->hist_data = hist_data;
2682         var->size = size;
2683         var->var.name = kstrdup(name, GFP_KERNEL);
2684         var->type = kstrdup(type, GFP_KERNEL);
2685         if (!var->var.name || !var->type) {
2686                 kfree(var->var.name);
2687                 kfree(var->type);
2688                 kfree(var);
2689                 var = ERR_PTR(-ENOMEM);
2690         }
2691  out:
2692         return var;
2693 }
2694
2695 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2696                                           struct trace_event_file *file,
2697                                           char *field_name)
2698 {
2699         struct hist_field *val = NULL, *var = NULL;
2700         unsigned long flags = HIST_FIELD_FL_VAR;
2701         struct trace_array *tr = file->tr;
2702         struct field_var *field_var;
2703         int ret = 0;
2704
2705         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2706                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2707                 ret = -EINVAL;
2708                 goto err;
2709         }
2710
2711         val = parse_atom(hist_data, file, field_name, &flags, NULL);
2712         if (IS_ERR(val)) {
2713                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2714                 ret = PTR_ERR(val);
2715                 goto err;
2716         }
2717
2718         var = create_var(hist_data, file, field_name, val->size, val->type);
2719         if (IS_ERR(var)) {
2720                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2721                 kfree(val);
2722                 ret = PTR_ERR(var);
2723                 goto err;
2724         }
2725
2726         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2727         if (!field_var) {
2728                 kfree(val);
2729                 kfree(var);
2730                 ret =  -ENOMEM;
2731                 goto err;
2732         }
2733
2734         field_var->var = var;
2735         field_var->val = val;
2736  out:
2737         return field_var;
2738  err:
2739         field_var = ERR_PTR(ret);
2740         goto out;
2741 }
2742
2743 /**
2744  * create_target_field_var - Automatically create a variable for a field
2745  * @target_hist_data: The target hist trigger
2746  * @subsys_name: Optional subsystem name
2747  * @event_name: Optional event name
2748  * @var_name: The name of the field (and the resulting variable)
2749  *
2750  * Hist trigger actions fetch data from variables, not directly from
2751  * events.  However, for convenience, users are allowed to directly
2752  * specify an event field in an action, which will be automatically
2753  * converted into a variable on their behalf.
2754
2755  * This function creates a field variable with the name var_name on
2756  * the hist trigger currently being defined on the target event.  If
2757  * subsys_name and event_name are specified, this function simply
2758  * verifies that they do in fact match the target event subsystem and
2759  * event name.
2760  *
2761  * Return: The variable created for the field.
2762  */
2763 static struct field_var *
2764 create_target_field_var(struct hist_trigger_data *target_hist_data,
2765                         char *subsys_name, char *event_name, char *var_name)
2766 {
2767         struct trace_event_file *file = target_hist_data->event_file;
2768
2769         if (subsys_name) {
2770                 struct trace_event_call *call;
2771
2772                 if (!event_name)
2773                         return NULL;
2774
2775                 call = file->event_call;
2776
2777                 if (strcmp(subsys_name, call->class->system) != 0)
2778                         return NULL;
2779
2780                 if (strcmp(event_name, trace_event_name(call)) != 0)
2781                         return NULL;
2782         }
2783
2784         return create_field_var(target_hist_data, file, var_name);
2785 }
2786
2787 static bool check_track_val_max(u64 track_val, u64 var_val)
2788 {
2789         if (var_val <= track_val)
2790                 return false;
2791
2792         return true;
2793 }
2794
2795 static bool check_track_val_changed(u64 track_val, u64 var_val)
2796 {
2797         if (var_val == track_val)
2798                 return false;
2799
2800         return true;
2801 }
2802
2803 static u64 get_track_val(struct hist_trigger_data *hist_data,
2804                          struct tracing_map_elt *elt,
2805                          struct action_data *data)
2806 {
2807         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2808         u64 track_val;
2809
2810         track_val = tracing_map_read_var(elt, track_var_idx);
2811
2812         return track_val;
2813 }
2814
2815 static void save_track_val(struct hist_trigger_data *hist_data,
2816                            struct tracing_map_elt *elt,
2817                            struct action_data *data, u64 var_val)
2818 {
2819         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2820
2821         tracing_map_set_var(elt, track_var_idx, var_val);
2822 }
2823
2824 static void save_track_data(struct hist_trigger_data *hist_data,
2825                             struct tracing_map_elt *elt, void *rec,
2826                             struct ring_buffer_event *rbe, void *key,
2827                             struct action_data *data, u64 *var_ref_vals)
2828 {
2829         if (data->track_data.save_data)
2830                 data->track_data.save_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
2831 }
2832
2833 static bool check_track_val(struct tracing_map_elt *elt,
2834                             struct action_data *data,
2835                             u64 var_val)
2836 {
2837         struct hist_trigger_data *hist_data;
2838         u64 track_val;
2839
2840         hist_data = data->track_data.track_var->hist_data;
2841         track_val = get_track_val(hist_data, elt, data);
2842
2843         return data->track_data.check_val(track_val, var_val);
2844 }
2845
2846 #ifdef CONFIG_TRACER_SNAPSHOT
2847 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2848 {
2849         /* called with tr->max_lock held */
2850         struct track_data *track_data = tr->cond_snapshot->cond_data;
2851         struct hist_elt_data *elt_data, *track_elt_data;
2852         struct snapshot_context *context = cond_data;
2853         struct action_data *action;
2854         u64 track_val;
2855
2856         if (!track_data)
2857                 return false;
2858
2859         action = track_data->action_data;
2860
2861         track_val = get_track_val(track_data->hist_data, context->elt,
2862                                   track_data->action_data);
2863
2864         if (!action->track_data.check_val(track_data->track_val, track_val))
2865                 return false;
2866
2867         track_data->track_val = track_val;
2868         memcpy(track_data->key, context->key, track_data->key_len);
2869
2870         elt_data = context->elt->private_data;
2871         track_elt_data = track_data->elt.private_data;
2872         if (elt_data->comm)
2873                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2874
2875         track_data->updated = true;
2876
2877         return true;
2878 }
2879
2880 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2881                                      struct tracing_map_elt *elt, void *rec,
2882                                      struct ring_buffer_event *rbe, void *key,
2883                                      struct action_data *data,
2884                                      u64 *var_ref_vals)
2885 {
2886         struct trace_event_file *file = hist_data->event_file;
2887         struct snapshot_context context;
2888
2889         context.elt = elt;
2890         context.key = key;
2891
2892         tracing_snapshot_cond(file->tr, &context);
2893 }
2894
2895 static void hist_trigger_print_key(struct seq_file *m,
2896                                    struct hist_trigger_data *hist_data,
2897                                    void *key,
2898                                    struct tracing_map_elt *elt);
2899
2900 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2901 {
2902         unsigned int i;
2903
2904         if (!hist_data->n_actions)
2905                 return NULL;
2906
2907         for (i = 0; i < hist_data->n_actions; i++) {
2908                 struct action_data *data = hist_data->actions[i];
2909
2910                 if (data->action == ACTION_SNAPSHOT)
2911                         return data;
2912         }
2913
2914         return NULL;
2915 }
2916
2917 static void track_data_snapshot_print(struct seq_file *m,
2918                                       struct hist_trigger_data *hist_data)
2919 {
2920         struct trace_event_file *file = hist_data->event_file;
2921         struct track_data *track_data;
2922         struct action_data *action;
2923
2924         track_data = tracing_cond_snapshot_data(file->tr);
2925         if (!track_data)
2926                 return;
2927
2928         if (!track_data->updated)
2929                 return;
2930
2931         action = snapshot_action(hist_data);
2932         if (!action)
2933                 return;
2934
2935         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
2936         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
2937                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
2938                    action->track_data.var_str, track_data->track_val);
2939
2940         seq_puts(m, "\ttriggered by event with key: ");
2941         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
2942         seq_putc(m, '\n');
2943 }
2944 #else
2945 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2946 {
2947         return false;
2948 }
2949 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2950                                      struct tracing_map_elt *elt, void *rec,
2951                                      struct ring_buffer_event *rbe, void *key,
2952                                      struct action_data *data,
2953                                      u64 *var_ref_vals) {}
2954 static void track_data_snapshot_print(struct seq_file *m,
2955                                       struct hist_trigger_data *hist_data) {}
2956 #endif /* CONFIG_TRACER_SNAPSHOT */
2957
2958 static void track_data_print(struct seq_file *m,
2959                              struct hist_trigger_data *hist_data,
2960                              struct tracing_map_elt *elt,
2961                              struct action_data *data)
2962 {
2963         u64 track_val = get_track_val(hist_data, elt, data);
2964         unsigned int i, save_var_idx;
2965
2966         if (data->handler == HANDLER_ONMAX)
2967                 seq_printf(m, "\n\tmax: %10llu", track_val);
2968         else if (data->handler == HANDLER_ONCHANGE)
2969                 seq_printf(m, "\n\tchanged: %10llu", track_val);
2970
2971         if (data->action == ACTION_SNAPSHOT)
2972                 return;
2973
2974         for (i = 0; i < hist_data->n_save_vars; i++) {
2975                 struct hist_field *save_val = hist_data->save_vars[i]->val;
2976                 struct hist_field *save_var = hist_data->save_vars[i]->var;
2977                 u64 val;
2978
2979                 save_var_idx = save_var->var.idx;
2980
2981                 val = tracing_map_read_var(elt, save_var_idx);
2982
2983                 if (save_val->flags & HIST_FIELD_FL_STRING) {
2984                         seq_printf(m, "  %s: %-32s", save_var->var.name,
2985                                    (char *)(uintptr_t)(val));
2986                 } else
2987                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
2988         }
2989 }
2990
2991 static void ontrack_action(struct hist_trigger_data *hist_data,
2992                            struct tracing_map_elt *elt, void *rec,
2993                            struct ring_buffer_event *rbe, void *key,
2994                            struct action_data *data, u64 *var_ref_vals)
2995 {
2996         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
2997
2998         if (check_track_val(elt, data, var_val)) {
2999                 save_track_val(hist_data, elt, data, var_val);
3000                 save_track_data(hist_data, elt, rec, rbe, key, data, var_ref_vals);
3001         }
3002 }
3003
3004 static void action_data_destroy(struct action_data *data)
3005 {
3006         unsigned int i;
3007
3008         lockdep_assert_held(&event_mutex);
3009
3010         kfree(data->action_name);
3011
3012         for (i = 0; i < data->n_params; i++)
3013                 kfree(data->params[i]);
3014
3015         if (data->synth_event)
3016                 data->synth_event->ref--;
3017
3018         kfree(data->synth_event_name);
3019
3020         kfree(data);
3021 }
3022
3023 static void track_data_destroy(struct hist_trigger_data *hist_data,
3024                                struct action_data *data)
3025 {
3026         struct trace_event_file *file = hist_data->event_file;
3027
3028         destroy_hist_field(data->track_data.track_var, 0);
3029
3030         if (data->action == ACTION_SNAPSHOT) {
3031                 struct track_data *track_data;
3032
3033                 track_data = tracing_cond_snapshot_data(file->tr);
3034                 if (track_data && track_data->hist_data == hist_data) {
3035                         tracing_snapshot_cond_disable(file->tr);
3036                         track_data_free(track_data);
3037                 }
3038         }
3039
3040         kfree(data->track_data.var_str);
3041
3042         action_data_destroy(data);
3043 }
3044
3045 static int action_create(struct hist_trigger_data *hist_data,
3046                          struct action_data *data);
3047
3048 static int track_data_create(struct hist_trigger_data *hist_data,
3049                              struct action_data *data)
3050 {
3051         struct hist_field *var_field, *ref_field, *track_var = NULL;
3052         struct trace_event_file *file = hist_data->event_file;
3053         struct trace_array *tr = file->tr;
3054         char *track_data_var_str;
3055         int ret = 0;
3056
3057         track_data_var_str = data->track_data.var_str;
3058         if (track_data_var_str[0] != '$') {
3059                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3060                 return -EINVAL;
3061         }
3062         track_data_var_str++;
3063
3064         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3065         if (!var_field) {
3066                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3067                 return -EINVAL;
3068         }
3069
3070         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3071         if (!ref_field)
3072                 return -ENOMEM;
3073
3074         data->track_data.var_ref = ref_field;
3075
3076         if (data->handler == HANDLER_ONMAX)
3077                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3078         if (IS_ERR(track_var)) {
3079                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3080                 ret = PTR_ERR(track_var);
3081                 goto out;
3082         }
3083
3084         if (data->handler == HANDLER_ONCHANGE)
3085                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3086         if (IS_ERR(track_var)) {
3087                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3088                 ret = PTR_ERR(track_var);
3089                 goto out;
3090         }
3091         data->track_data.track_var = track_var;
3092
3093         ret = action_create(hist_data, data);
3094  out:
3095         return ret;
3096 }
3097
3098 static int parse_action_params(struct trace_array *tr, char *params,
3099                                struct action_data *data)
3100 {
3101         char *param, *saved_param;
3102         bool first_param = true;
3103         int ret = 0;
3104
3105         while (params) {
3106                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3107                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3108                         goto out;
3109                 }
3110
3111                 param = strsep(&params, ",");
3112                 if (!param) {
3113                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3114                         ret = -EINVAL;
3115                         goto out;
3116                 }
3117
3118                 param = strstrip(param);
3119                 if (strlen(param) < 2) {
3120                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3121                         ret = -EINVAL;
3122                         goto out;
3123                 }
3124
3125                 saved_param = kstrdup(param, GFP_KERNEL);
3126                 if (!saved_param) {
3127                         ret = -ENOMEM;
3128                         goto out;
3129                 }
3130
3131                 if (first_param && data->use_trace_keyword) {
3132                         data->synth_event_name = saved_param;
3133                         first_param = false;
3134                         continue;
3135                 }
3136                 first_param = false;
3137
3138                 data->params[data->n_params++] = saved_param;
3139         }
3140  out:
3141         return ret;
3142 }
3143
3144 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3145                         enum handler_id handler)
3146 {
3147         char *action_name;
3148         int ret = 0;
3149
3150         strsep(&str, ".");
3151         if (!str) {
3152                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3153                 ret = -EINVAL;
3154                 goto out;
3155         }
3156
3157         action_name = strsep(&str, "(");
3158         if (!action_name || !str) {
3159                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3160                 ret = -EINVAL;
3161                 goto out;
3162         }
3163
3164         if (str_has_prefix(action_name, "save")) {
3165                 char *params = strsep(&str, ")");
3166
3167                 if (!params) {
3168                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3169                         ret = -EINVAL;
3170                         goto out;
3171                 }
3172
3173                 ret = parse_action_params(tr, params, data);
3174                 if (ret)
3175                         goto out;
3176
3177                 if (handler == HANDLER_ONMAX)
3178                         data->track_data.check_val = check_track_val_max;
3179                 else if (handler == HANDLER_ONCHANGE)
3180                         data->track_data.check_val = check_track_val_changed;
3181                 else {
3182                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3183                         ret = -EINVAL;
3184                         goto out;
3185                 }
3186
3187                 data->track_data.save_data = save_track_data_vars;
3188                 data->fn = ontrack_action;
3189                 data->action = ACTION_SAVE;
3190         } else if (str_has_prefix(action_name, "snapshot")) {
3191                 char *params = strsep(&str, ")");
3192
3193                 if (!str) {
3194                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3195                         ret = -EINVAL;
3196                         goto out;
3197                 }
3198
3199                 if (handler == HANDLER_ONMAX)
3200                         data->track_data.check_val = check_track_val_max;
3201                 else if (handler == HANDLER_ONCHANGE)
3202                         data->track_data.check_val = check_track_val_changed;
3203                 else {
3204                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3205                         ret = -EINVAL;
3206                         goto out;
3207                 }
3208
3209                 data->track_data.save_data = save_track_data_snapshot;
3210                 data->fn = ontrack_action;
3211                 data->action = ACTION_SNAPSHOT;
3212         } else {
3213                 char *params = strsep(&str, ")");
3214
3215                 if (str_has_prefix(action_name, "trace"))
3216                         data->use_trace_keyword = true;
3217
3218                 if (params) {
3219                         ret = parse_action_params(tr, params, data);
3220                         if (ret)
3221                                 goto out;
3222                 }
3223
3224                 if (handler == HANDLER_ONMAX)
3225                         data->track_data.check_val = check_track_val_max;
3226                 else if (handler == HANDLER_ONCHANGE)
3227                         data->track_data.check_val = check_track_val_changed;
3228
3229                 if (handler != HANDLER_ONMATCH) {
3230                         data->track_data.save_data = action_trace;
3231                         data->fn = ontrack_action;
3232                 } else
3233                         data->fn = action_trace;
3234
3235                 data->action = ACTION_TRACE;
3236         }
3237
3238         data->action_name = kstrdup(action_name, GFP_KERNEL);
3239         if (!data->action_name) {
3240                 ret = -ENOMEM;
3241                 goto out;
3242         }
3243
3244         data->handler = handler;
3245  out:
3246         return ret;
3247 }
3248
3249 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3250                                             char *str, enum handler_id handler)
3251 {
3252         struct action_data *data;
3253         int ret = -EINVAL;
3254         char *var_str;
3255
3256         data = kzalloc(sizeof(*data), GFP_KERNEL);
3257         if (!data)
3258                 return ERR_PTR(-ENOMEM);
3259
3260         var_str = strsep(&str, ")");
3261         if (!var_str || !str) {
3262                 ret = -EINVAL;
3263                 goto free;
3264         }
3265
3266         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3267         if (!data->track_data.var_str) {
3268                 ret = -ENOMEM;
3269                 goto free;
3270         }
3271
3272         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3273         if (ret)
3274                 goto free;
3275  out:
3276         return data;
3277  free:
3278         track_data_destroy(hist_data, data);
3279         data = ERR_PTR(ret);
3280         goto out;
3281 }
3282
3283 static void onmatch_destroy(struct action_data *data)
3284 {
3285         kfree(data->match_data.event);
3286         kfree(data->match_data.event_system);
3287
3288         action_data_destroy(data);
3289 }
3290
3291 static void destroy_field_var(struct field_var *field_var)
3292 {
3293         if (!field_var)
3294                 return;
3295
3296         destroy_hist_field(field_var->var, 0);
3297         destroy_hist_field(field_var->val, 0);
3298
3299         kfree(field_var);
3300 }
3301
3302 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3303 {
3304         unsigned int i;
3305
3306         for (i = 0; i < hist_data->n_field_vars; i++)
3307                 destroy_field_var(hist_data->field_vars[i]);
3308
3309         for (i = 0; i < hist_data->n_save_vars; i++)
3310                 destroy_field_var(hist_data->save_vars[i]);
3311 }
3312
3313 static void save_field_var(struct hist_trigger_data *hist_data,
3314                            struct field_var *field_var)
3315 {
3316         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3317
3318         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3319                 hist_data->n_field_var_str++;
3320 }
3321
3322
3323 static int check_synth_field(struct synth_event *event,
3324                              struct hist_field *hist_field,
3325                              unsigned int field_pos)
3326 {
3327         struct synth_field *field;
3328
3329         if (field_pos >= event->n_fields)
3330                 return -EINVAL;
3331
3332         field = event->fields[field_pos];
3333
3334         /*
3335          * A dynamic string synth field can accept static or
3336          * dynamic. A static string synth field can only accept a
3337          * same-sized static string, which is checked for later.
3338          */
3339         if (strstr(hist_field->type, "char[") && field->is_string
3340             && field->is_dynamic)
3341                 return 0;
3342
3343         if (strcmp(field->type, hist_field->type) != 0) {
3344                 if (field->size != hist_field->size ||
3345                     field->is_signed != hist_field->is_signed)
3346                         return -EINVAL;
3347         }
3348
3349         return 0;
3350 }
3351
3352 static struct hist_field *
3353 trace_action_find_var(struct hist_trigger_data *hist_data,
3354                       struct action_data *data,
3355                       char *system, char *event, char *var)
3356 {
3357         struct trace_array *tr = hist_data->event_file->tr;
3358         struct hist_field *hist_field;
3359
3360         var++; /* skip '$' */
3361
3362         hist_field = find_target_event_var(hist_data, system, event, var);
3363         if (!hist_field) {
3364                 if (!system && data->handler == HANDLER_ONMATCH) {
3365                         system = data->match_data.event_system;
3366                         event = data->match_data.event;
3367                 }
3368
3369                 hist_field = find_event_var(hist_data, system, event, var);
3370         }
3371
3372         if (!hist_field)
3373                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3374
3375         return hist_field;
3376 }
3377
3378 static struct hist_field *
3379 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3380                               struct action_data *data, char *system,
3381                               char *event, char *var)
3382 {
3383         struct hist_field *hist_field = NULL;
3384         struct field_var *field_var;
3385
3386         /*
3387          * First try to create a field var on the target event (the
3388          * currently being defined).  This will create a variable for
3389          * unqualified fields on the target event, or if qualified,
3390          * target fields that have qualified names matching the target.
3391          */
3392         field_var = create_target_field_var(hist_data, system, event, var);
3393
3394         if (field_var && !IS_ERR(field_var)) {
3395                 save_field_var(hist_data, field_var);
3396                 hist_field = field_var->var;
3397         } else {
3398                 field_var = NULL;
3399                 /*
3400                  * If no explicit system.event is specfied, default to
3401                  * looking for fields on the onmatch(system.event.xxx)
3402                  * event.
3403                  */
3404                 if (!system && data->handler == HANDLER_ONMATCH) {
3405                         system = data->match_data.event_system;
3406                         event = data->match_data.event;
3407                 }
3408
3409                 if (!event)
3410                         goto free;
3411                 /*
3412                  * At this point, we're looking at a field on another
3413                  * event.  Because we can't modify a hist trigger on
3414                  * another event to add a variable for a field, we need
3415                  * to create a new trigger on that event and create the
3416                  * variable at the same time.
3417                  */
3418                 hist_field = create_field_var_hist(hist_data, system, event, var);
3419                 if (IS_ERR(hist_field))
3420                         goto free;
3421         }
3422  out:
3423         return hist_field;
3424  free:
3425         destroy_field_var(field_var);
3426         hist_field = NULL;
3427         goto out;
3428 }
3429
3430 static int trace_action_create(struct hist_trigger_data *hist_data,
3431                                struct action_data *data)
3432 {
3433         struct trace_array *tr = hist_data->event_file->tr;
3434         char *event_name, *param, *system = NULL;
3435         struct hist_field *hist_field, *var_ref;
3436         unsigned int i;
3437         unsigned int field_pos = 0;
3438         struct synth_event *event;
3439         char *synth_event_name;
3440         int var_ref_idx, ret = 0;
3441
3442         lockdep_assert_held(&event_mutex);
3443
3444         if (data->use_trace_keyword)
3445                 synth_event_name = data->synth_event_name;
3446         else
3447                 synth_event_name = data->action_name;
3448
3449         event = find_synth_event(synth_event_name);
3450         if (!event) {
3451                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3452                 return -EINVAL;
3453         }
3454
3455         event->ref++;
3456
3457         for (i = 0; i < data->n_params; i++) {
3458                 char *p;
3459
3460                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3461                 if (!param) {
3462                         ret = -ENOMEM;
3463                         goto err;
3464                 }
3465
3466                 system = strsep(&param, ".");
3467                 if (!param) {
3468                         param = (char *)system;
3469                         system = event_name = NULL;
3470                 } else {
3471                         event_name = strsep(&param, ".");
3472                         if (!param) {
3473                                 kfree(p);
3474                                 ret = -EINVAL;
3475                                 goto err;
3476                         }
3477                 }
3478
3479                 if (param[0] == '$')
3480                         hist_field = trace_action_find_var(hist_data, data,
3481                                                            system, event_name,
3482                                                            param);
3483                 else
3484                         hist_field = trace_action_create_field_var(hist_data,
3485                                                                    data,
3486                                                                    system,
3487                                                                    event_name,
3488                                                                    param);
3489
3490                 if (!hist_field) {
3491                         kfree(p);
3492                         ret = -EINVAL;
3493                         goto err;
3494                 }
3495
3496                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3497                         var_ref = create_var_ref(hist_data, hist_field,
3498                                                  system, event_name);
3499                         if (!var_ref) {
3500                                 kfree(p);
3501                                 ret = -ENOMEM;
3502                                 goto err;
3503                         }
3504
3505                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3506                         if (WARN_ON(var_ref_idx < 0)) {
3507                                 ret = var_ref_idx;
3508                                 goto err;
3509                         }
3510
3511                         data->var_ref_idx[i] = var_ref_idx;
3512
3513                         field_pos++;
3514                         kfree(p);
3515                         continue;
3516                 }
3517
3518                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3519                 kfree(p);
3520                 ret = -EINVAL;
3521                 goto err;
3522         }
3523
3524         if (field_pos != event->n_fields) {
3525                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3526                 ret = -EINVAL;
3527                 goto err;
3528         }
3529
3530         data->synth_event = event;
3531  out:
3532         return ret;
3533  err:
3534         event->ref--;
3535
3536         goto out;
3537 }
3538
3539 static int action_create(struct hist_trigger_data *hist_data,
3540                          struct action_data *data)
3541 {
3542         struct trace_event_file *file = hist_data->event_file;
3543         struct trace_array *tr = file->tr;
3544         struct track_data *track_data;
3545         struct field_var *field_var;
3546         unsigned int i;
3547         char *param;
3548         int ret = 0;
3549
3550         if (data->action == ACTION_TRACE)
3551                 return trace_action_create(hist_data, data);
3552
3553         if (data->action == ACTION_SNAPSHOT) {
3554                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3555                 if (IS_ERR(track_data)) {
3556                         ret = PTR_ERR(track_data);
3557                         goto out;
3558                 }
3559
3560                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3561                                                    cond_snapshot_update);
3562                 if (ret)
3563                         track_data_free(track_data);
3564
3565                 goto out;
3566         }
3567
3568         if (data->action == ACTION_SAVE) {
3569                 if (hist_data->n_save_vars) {
3570                         ret = -EEXIST;
3571                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3572                         goto out;
3573                 }
3574
3575                 for (i = 0; i < data->n_params; i++) {
3576                         param = kstrdup(data->params[i], GFP_KERNEL);
3577                         if (!param) {
3578                                 ret = -ENOMEM;
3579                                 goto out;
3580                         }
3581
3582                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3583                         if (IS_ERR(field_var)) {
3584                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3585                                          errpos(param));
3586                                 ret = PTR_ERR(field_var);
3587                                 kfree(param);
3588                                 goto out;
3589                         }
3590
3591                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3592                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3593                                 hist_data->n_save_var_str++;
3594                         kfree(param);
3595                 }
3596         }
3597  out:
3598         return ret;
3599 }
3600
3601 static int onmatch_create(struct hist_trigger_data *hist_data,
3602                           struct action_data *data)
3603 {
3604         return action_create(hist_data, data);
3605 }
3606
3607 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3608 {
3609         char *match_event, *match_event_system;
3610         struct action_data *data;
3611         int ret = -EINVAL;
3612
3613         data = kzalloc(sizeof(*data), GFP_KERNEL);
3614         if (!data)
3615                 return ERR_PTR(-ENOMEM);
3616
3617         match_event = strsep(&str, ")");
3618         if (!match_event || !str) {
3619                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3620                 goto free;
3621         }
3622
3623         match_event_system = strsep(&match_event, ".");
3624         if (!match_event) {
3625                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3626                 goto free;
3627         }
3628
3629         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3630                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3631                 goto free;
3632         }
3633
3634         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3635         if (!data->match_data.event) {
3636                 ret = -ENOMEM;
3637                 goto free;
3638         }
3639
3640         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3641         if (!data->match_data.event_system) {
3642                 ret = -ENOMEM;
3643                 goto free;
3644         }
3645
3646         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3647         if (ret)
3648                 goto free;
3649  out:
3650         return data;
3651  free:
3652         onmatch_destroy(data);
3653         data = ERR_PTR(ret);
3654         goto out;
3655 }
3656
3657 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3658 {
3659         hist_data->fields[HITCOUNT_IDX] =
3660                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3661         if (!hist_data->fields[HITCOUNT_IDX])
3662                 return -ENOMEM;
3663
3664         hist_data->n_vals++;
3665         hist_data->n_fields++;
3666
3667         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3668                 return -EINVAL;
3669
3670         return 0;
3671 }
3672
3673 static int __create_val_field(struct hist_trigger_data *hist_data,
3674                               unsigned int val_idx,
3675                               struct trace_event_file *file,
3676                               char *var_name, char *field_str,
3677                               unsigned long flags)
3678 {
3679         struct hist_field *hist_field;
3680         int ret = 0;
3681
3682         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3683         if (IS_ERR(hist_field)) {
3684                 ret = PTR_ERR(hist_field);
3685                 goto out;
3686         }
3687
3688         hist_data->fields[val_idx] = hist_field;
3689
3690         ++hist_data->n_vals;
3691         ++hist_data->n_fields;
3692
3693         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3694                 ret = -EINVAL;
3695  out:
3696         return ret;
3697 }
3698
3699 static int create_val_field(struct hist_trigger_data *hist_data,
3700                             unsigned int val_idx,
3701                             struct trace_event_file *file,
3702                             char *field_str)
3703 {
3704         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3705                 return -EINVAL;
3706
3707         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3708 }
3709
3710 static int create_var_field(struct hist_trigger_data *hist_data,
3711                             unsigned int val_idx,
3712                             struct trace_event_file *file,
3713                             char *var_name, char *expr_str)
3714 {
3715         struct trace_array *tr = hist_data->event_file->tr;
3716         unsigned long flags = 0;
3717         int ret;
3718
3719         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3720                 return -EINVAL;
3721
3722         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3723                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3724                 return -EINVAL;
3725         }
3726
3727         flags |= HIST_FIELD_FL_VAR;
3728         hist_data->n_vars++;
3729         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3730                 return -EINVAL;
3731
3732         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3733
3734         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
3735                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
3736
3737         return ret;
3738 }
3739
3740 static int create_val_fields(struct hist_trigger_data *hist_data,
3741                              struct trace_event_file *file)
3742 {
3743         char *fields_str, *field_str;
3744         unsigned int i, j = 1;
3745         int ret;
3746
3747         ret = create_hitcount_val(hist_data);
3748         if (ret)
3749                 goto out;
3750
3751         fields_str = hist_data->attrs->vals_str;
3752         if (!fields_str)
3753                 goto out;
3754
3755         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3756                      j < TRACING_MAP_VALS_MAX; i++) {
3757                 field_str = strsep(&fields_str, ",");
3758                 if (!field_str)
3759                         break;
3760
3761                 if (strcmp(field_str, "hitcount") == 0)
3762                         continue;
3763
3764                 ret = create_val_field(hist_data, j++, file, field_str);
3765                 if (ret)
3766                         goto out;
3767         }
3768
3769         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3770                 ret = -EINVAL;
3771  out:
3772         return ret;
3773 }
3774
3775 static int create_key_field(struct hist_trigger_data *hist_data,
3776                             unsigned int key_idx,
3777                             unsigned int key_offset,
3778                             struct trace_event_file *file,
3779                             char *field_str)
3780 {
3781         struct trace_array *tr = hist_data->event_file->tr;
3782         struct hist_field *hist_field = NULL;
3783         unsigned long flags = 0;
3784         unsigned int key_size;
3785         int ret = 0;
3786
3787         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3788                 return -EINVAL;
3789
3790         flags |= HIST_FIELD_FL_KEY;
3791
3792         if (strcmp(field_str, "stacktrace") == 0) {
3793                 flags |= HIST_FIELD_FL_STACKTRACE;
3794                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3795                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3796         } else {
3797                 hist_field = parse_expr(hist_data, file, field_str, flags,
3798                                         NULL, 0);
3799                 if (IS_ERR(hist_field)) {
3800                         ret = PTR_ERR(hist_field);
3801                         goto out;
3802                 }
3803
3804                 if (field_has_hist_vars(hist_field, 0)) {
3805                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3806                         destroy_hist_field(hist_field, 0);
3807                         ret = -EINVAL;
3808                         goto out;
3809                 }
3810
3811                 key_size = hist_field->size;
3812         }
3813
3814         hist_data->fields[key_idx] = hist_field;
3815
3816         key_size = ALIGN(key_size, sizeof(u64));
3817         hist_data->fields[key_idx]->size = key_size;
3818         hist_data->fields[key_idx]->offset = key_offset;
3819
3820         hist_data->key_size += key_size;
3821
3822         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3823                 ret = -EINVAL;
3824                 goto out;
3825         }
3826
3827         hist_data->n_keys++;
3828         hist_data->n_fields++;
3829
3830         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3831                 return -EINVAL;
3832
3833         ret = key_size;
3834  out:
3835         return ret;
3836 }
3837
3838 static int create_key_fields(struct hist_trigger_data *hist_data,
3839                              struct trace_event_file *file)
3840 {
3841         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3842         char *fields_str, *field_str;
3843         int ret = -EINVAL;
3844
3845         fields_str = hist_data->attrs->keys_str;
3846         if (!fields_str)
3847                 goto out;
3848
3849         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3850                 field_str = strsep(&fields_str, ",");
3851                 if (!field_str)
3852                         break;
3853                 ret = create_key_field(hist_data, i, key_offset,
3854                                        file, field_str);
3855                 if (ret < 0)
3856                         goto out;
3857                 key_offset += ret;
3858         }
3859         if (fields_str) {
3860                 ret = -EINVAL;
3861                 goto out;
3862         }
3863         ret = 0;
3864  out:
3865         return ret;
3866 }
3867
3868 static int create_var_fields(struct hist_trigger_data *hist_data,
3869                              struct trace_event_file *file)
3870 {
3871         unsigned int i, j = hist_data->n_vals;
3872         int ret = 0;
3873
3874         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
3875
3876         for (i = 0; i < n_vars; i++) {
3877                 char *var_name = hist_data->attrs->var_defs.name[i];
3878                 char *expr = hist_data->attrs->var_defs.expr[i];
3879
3880                 ret = create_var_field(hist_data, j++, file, var_name, expr);
3881                 if (ret)
3882                         goto out;
3883         }
3884  out:
3885         return ret;
3886 }
3887
3888 static void free_var_defs(struct hist_trigger_data *hist_data)
3889 {
3890         unsigned int i;
3891
3892         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
3893                 kfree(hist_data->attrs->var_defs.name[i]);
3894                 kfree(hist_data->attrs->var_defs.expr[i]);
3895         }
3896
3897         hist_data->attrs->var_defs.n_vars = 0;
3898 }
3899
3900 static int parse_var_defs(struct hist_trigger_data *hist_data)
3901 {
3902         struct trace_array *tr = hist_data->event_file->tr;
3903         char *s, *str, *var_name, *field_str;
3904         unsigned int i, j, n_vars = 0;
3905         int ret = 0;
3906
3907         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
3908                 str = hist_data->attrs->assignment_str[i];
3909                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
3910                         field_str = strsep(&str, ",");
3911                         if (!field_str)
3912                                 break;
3913
3914                         var_name = strsep(&field_str, "=");
3915                         if (!var_name || !field_str) {
3916                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
3917                                          errpos(var_name));
3918                                 ret = -EINVAL;
3919                                 goto free;
3920                         }
3921
3922                         if (n_vars == TRACING_MAP_VARS_MAX) {
3923                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
3924                                 ret = -EINVAL;
3925                                 goto free;
3926                         }
3927
3928                         s = kstrdup(var_name, GFP_KERNEL);
3929                         if (!s) {
3930                                 ret = -ENOMEM;
3931                                 goto free;
3932                         }
3933                         hist_data->attrs->var_defs.name[n_vars] = s;
3934
3935                         s = kstrdup(field_str, GFP_KERNEL);
3936                         if (!s) {
3937                                 ret = -ENOMEM;
3938                                 goto free;
3939                         }
3940                         hist_data->attrs->var_defs.expr[n_vars++] = s;
3941
3942                         hist_data->attrs->var_defs.n_vars = n_vars;
3943                 }
3944         }
3945
3946         return ret;
3947  free:
3948         free_var_defs(hist_data);
3949
3950         return ret;
3951 }
3952
3953 static int create_hist_fields(struct hist_trigger_data *hist_data,
3954                               struct trace_event_file *file)
3955 {
3956         int ret;
3957
3958         ret = parse_var_defs(hist_data);
3959         if (ret)
3960                 goto out;
3961
3962         ret = create_val_fields(hist_data, file);
3963         if (ret)
3964                 goto out;
3965
3966         ret = create_var_fields(hist_data, file);
3967         if (ret)
3968                 goto out;
3969
3970         ret = create_key_fields(hist_data, file);
3971         if (ret)
3972                 goto out;
3973  out:
3974         free_var_defs(hist_data);
3975
3976         return ret;
3977 }
3978
3979 static int is_descending(struct trace_array *tr, const char *str)
3980 {
3981         if (!str)
3982                 return 0;
3983
3984         if (strcmp(str, "descending") == 0)
3985                 return 1;
3986
3987         if (strcmp(str, "ascending") == 0)
3988                 return 0;
3989
3990         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
3991
3992         return -EINVAL;
3993 }
3994
3995 static int create_sort_keys(struct hist_trigger_data *hist_data)
3996 {
3997         struct trace_array *tr = hist_data->event_file->tr;
3998         char *fields_str = hist_data->attrs->sort_key_str;
3999         struct tracing_map_sort_key *sort_key;
4000         int descending, ret = 0;
4001         unsigned int i, j, k;
4002
4003         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4004
4005         if (!fields_str)
4006                 goto out;
4007
4008         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4009                 struct hist_field *hist_field;
4010                 char *field_str, *field_name;
4011                 const char *test_name;
4012
4013                 sort_key = &hist_data->sort_keys[i];
4014
4015                 field_str = strsep(&fields_str, ",");
4016                 if (!field_str)
4017                         break;
4018
4019                 if (!*field_str) {
4020                         ret = -EINVAL;
4021                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4022                         break;
4023                 }
4024
4025                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4026                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4027                         ret = -EINVAL;
4028                         break;
4029                 }
4030
4031                 field_name = strsep(&field_str, ".");
4032                 if (!field_name || !*field_name) {
4033                         ret = -EINVAL;
4034                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4035                         break;
4036                 }
4037
4038                 if (strcmp(field_name, "hitcount") == 0) {
4039                         descending = is_descending(tr, field_str);
4040                         if (descending < 0) {
4041                                 ret = descending;
4042                                 break;
4043                         }
4044                         sort_key->descending = descending;
4045                         continue;
4046                 }
4047
4048                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4049                         unsigned int idx;
4050
4051                         hist_field = hist_data->fields[j];
4052                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4053                                 continue;
4054
4055                         idx = k++;
4056
4057                         test_name = hist_field_name(hist_field, 0);
4058
4059                         if (strcmp(field_name, test_name) == 0) {
4060                                 sort_key->field_idx = idx;
4061                                 descending = is_descending(tr, field_str);
4062                                 if (descending < 0) {
4063                                         ret = descending;
4064                                         goto out;
4065                                 }
4066                                 sort_key->descending = descending;
4067                                 break;
4068                         }
4069                 }
4070                 if (j == hist_data->n_fields) {
4071                         ret = -EINVAL;
4072                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4073                         break;
4074                 }
4075         }
4076
4077         hist_data->n_sort_keys = i;
4078  out:
4079         return ret;
4080 }
4081
4082 static void destroy_actions(struct hist_trigger_data *hist_data)
4083 {
4084         unsigned int i;
4085
4086         for (i = 0; i < hist_data->n_actions; i++) {
4087                 struct action_data *data = hist_data->actions[i];
4088
4089                 if (data->handler == HANDLER_ONMATCH)
4090                         onmatch_destroy(data);
4091                 else if (data->handler == HANDLER_ONMAX ||
4092                          data->handler == HANDLER_ONCHANGE)
4093                         track_data_destroy(hist_data, data);
4094                 else
4095                         kfree(data);
4096         }
4097 }
4098
4099 static int parse_actions(struct hist_trigger_data *hist_data)
4100 {
4101         struct trace_array *tr = hist_data->event_file->tr;
4102         struct action_data *data;
4103         unsigned int i;
4104         int ret = 0;
4105         char *str;
4106         int len;
4107
4108         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4109                 str = hist_data->attrs->action_str[i];
4110
4111                 if ((len = str_has_prefix(str, "onmatch("))) {
4112                         char *action_str = str + len;
4113
4114                         data = onmatch_parse(tr, action_str);
4115                         if (IS_ERR(data)) {
4116                                 ret = PTR_ERR(data);
4117                                 break;
4118                         }
4119                 } else if ((len = str_has_prefix(str, "onmax("))) {
4120                         char *action_str = str + len;
4121
4122                         data = track_data_parse(hist_data, action_str,
4123                                                 HANDLER_ONMAX);
4124                         if (IS_ERR(data)) {
4125                                 ret = PTR_ERR(data);
4126                                 break;
4127                         }
4128                 } else if ((len = str_has_prefix(str, "onchange("))) {
4129                         char *action_str = str + len;
4130
4131                         data = track_data_parse(hist_data, action_str,
4132                                                 HANDLER_ONCHANGE);
4133                         if (IS_ERR(data)) {
4134                                 ret = PTR_ERR(data);
4135                                 break;
4136                         }
4137                 } else {
4138                         ret = -EINVAL;
4139                         break;
4140                 }
4141
4142                 hist_data->actions[hist_data->n_actions++] = data;
4143         }
4144
4145         return ret;
4146 }
4147
4148 static int create_actions(struct hist_trigger_data *hist_data)
4149 {
4150         struct action_data *data;
4151         unsigned int i;
4152         int ret = 0;
4153
4154         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4155                 data = hist_data->actions[i];
4156
4157                 if (data->handler == HANDLER_ONMATCH) {
4158                         ret = onmatch_create(hist_data, data);
4159                         if (ret)
4160                                 break;
4161                 } else if (data->handler == HANDLER_ONMAX ||
4162                            data->handler == HANDLER_ONCHANGE) {
4163                         ret = track_data_create(hist_data, data);
4164                         if (ret)
4165                                 break;
4166                 } else {
4167                         ret = -EINVAL;
4168                         break;
4169                 }
4170         }
4171
4172         return ret;
4173 }
4174
4175 static void print_actions(struct seq_file *m,
4176                           struct hist_trigger_data *hist_data,
4177                           struct tracing_map_elt *elt)
4178 {
4179         unsigned int i;
4180
4181         for (i = 0; i < hist_data->n_actions; i++) {
4182                 struct action_data *data = hist_data->actions[i];
4183
4184                 if (data->action == ACTION_SNAPSHOT)
4185                         continue;
4186
4187                 if (data->handler == HANDLER_ONMAX ||
4188                     data->handler == HANDLER_ONCHANGE)
4189                         track_data_print(m, hist_data, elt, data);
4190         }
4191 }
4192
4193 static void print_action_spec(struct seq_file *m,
4194                               struct hist_trigger_data *hist_data,
4195                               struct action_data *data)
4196 {
4197         unsigned int i;
4198
4199         if (data->action == ACTION_SAVE) {
4200                 for (i = 0; i < hist_data->n_save_vars; i++) {
4201                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4202                         if (i < hist_data->n_save_vars - 1)
4203                                 seq_puts(m, ",");
4204                 }
4205         } else if (data->action == ACTION_TRACE) {
4206                 if (data->use_trace_keyword)
4207                         seq_printf(m, "%s", data->synth_event_name);
4208                 for (i = 0; i < data->n_params; i++) {
4209                         if (i || data->use_trace_keyword)
4210                                 seq_puts(m, ",");
4211                         seq_printf(m, "%s", data->params[i]);
4212                 }
4213         }
4214 }
4215
4216 static void print_track_data_spec(struct seq_file *m,
4217                                   struct hist_trigger_data *hist_data,
4218                                   struct action_data *data)
4219 {
4220         if (data->handler == HANDLER_ONMAX)
4221                 seq_puts(m, ":onmax(");
4222         else if (data->handler == HANDLER_ONCHANGE)
4223                 seq_puts(m, ":onchange(");
4224         seq_printf(m, "%s", data->track_data.var_str);
4225         seq_printf(m, ").%s(", data->action_name);
4226
4227         print_action_spec(m, hist_data, data);
4228
4229         seq_puts(m, ")");
4230 }
4231
4232 static void print_onmatch_spec(struct seq_file *m,
4233                                struct hist_trigger_data *hist_data,
4234                                struct action_data *data)
4235 {
4236         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4237                    data->match_data.event);
4238
4239         seq_printf(m, "%s(", data->action_name);
4240
4241         print_action_spec(m, hist_data, data);
4242
4243         seq_puts(m, ")");
4244 }
4245
4246 static bool actions_match(struct hist_trigger_data *hist_data,
4247                           struct hist_trigger_data *hist_data_test)
4248 {
4249         unsigned int i, j;
4250
4251         if (hist_data->n_actions != hist_data_test->n_actions)
4252                 return false;
4253
4254         for (i = 0; i < hist_data->n_actions; i++) {
4255                 struct action_data *data = hist_data->actions[i];
4256                 struct action_data *data_test = hist_data_test->actions[i];
4257                 char *action_name, *action_name_test;
4258
4259                 if (data->handler != data_test->handler)
4260                         return false;
4261                 if (data->action != data_test->action)
4262                         return false;
4263
4264                 if (data->n_params != data_test->n_params)
4265                         return false;
4266
4267                 for (j = 0; j < data->n_params; j++) {
4268                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4269                                 return false;
4270                 }
4271
4272                 if (data->use_trace_keyword)
4273                         action_name = data->synth_event_name;
4274                 else
4275                         action_name = data->action_name;
4276
4277                 if (data_test->use_trace_keyword)
4278                         action_name_test = data_test->synth_event_name;
4279                 else
4280                         action_name_test = data_test->action_name;
4281
4282                 if (strcmp(action_name, action_name_test) != 0)
4283                         return false;
4284
4285                 if (data->handler == HANDLER_ONMATCH) {
4286                         if (strcmp(data->match_data.event_system,
4287                                    data_test->match_data.event_system) != 0)
4288                                 return false;
4289                         if (strcmp(data->match_data.event,
4290                                    data_test->match_data.event) != 0)
4291                                 return false;
4292                 } else if (data->handler == HANDLER_ONMAX ||
4293                            data->handler == HANDLER_ONCHANGE) {
4294                         if (strcmp(data->track_data.var_str,
4295                                    data_test->track_data.var_str) != 0)
4296                                 return false;
4297                 }
4298         }
4299
4300         return true;
4301 }
4302
4303
4304 static void print_actions_spec(struct seq_file *m,
4305                                struct hist_trigger_data *hist_data)
4306 {
4307         unsigned int i;
4308
4309         for (i = 0; i < hist_data->n_actions; i++) {
4310                 struct action_data *data = hist_data->actions[i];
4311
4312                 if (data->handler == HANDLER_ONMATCH)
4313                         print_onmatch_spec(m, hist_data, data);
4314                 else if (data->handler == HANDLER_ONMAX ||
4315                          data->handler == HANDLER_ONCHANGE)
4316                         print_track_data_spec(m, hist_data, data);
4317         }
4318 }
4319
4320 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4321 {
4322         unsigned int i;
4323
4324         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4325                 kfree(hist_data->field_var_hists[i]->cmd);
4326                 kfree(hist_data->field_var_hists[i]);
4327         }
4328 }
4329
4330 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4331 {
4332         if (!hist_data)
4333                 return;
4334
4335         destroy_hist_trigger_attrs(hist_data->attrs);
4336         destroy_hist_fields(hist_data);
4337         tracing_map_destroy(hist_data->map);
4338
4339         destroy_actions(hist_data);
4340         destroy_field_vars(hist_data);
4341         destroy_field_var_hists(hist_data);
4342
4343         kfree(hist_data);
4344 }
4345
4346 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4347 {
4348         struct tracing_map *map = hist_data->map;
4349         struct ftrace_event_field *field;
4350         struct hist_field *hist_field;
4351         int i, idx = 0;
4352
4353         for_each_hist_field(i, hist_data) {
4354                 hist_field = hist_data->fields[i];
4355                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4356                         tracing_map_cmp_fn_t cmp_fn;
4357
4358                         field = hist_field->field;
4359
4360                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4361                                 cmp_fn = tracing_map_cmp_none;
4362                         else if (!field)
4363                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4364                                                              hist_field->is_signed);
4365                         else if (is_string_field(field))
4366                                 cmp_fn = tracing_map_cmp_string;
4367                         else
4368                                 cmp_fn = tracing_map_cmp_num(field->size,
4369                                                              field->is_signed);
4370                         idx = tracing_map_add_key_field(map,
4371                                                         hist_field->offset,
4372                                                         cmp_fn);
4373                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4374                         idx = tracing_map_add_sum_field(map);
4375
4376                 if (idx < 0)
4377                         return idx;
4378
4379                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4380                         idx = tracing_map_add_var(map);
4381                         if (idx < 0)
4382                                 return idx;
4383                         hist_field->var.idx = idx;
4384                         hist_field->var.hist_data = hist_data;
4385                 }
4386         }
4387
4388         return 0;
4389 }
4390
4391 static struct hist_trigger_data *
4392 create_hist_data(unsigned int map_bits,
4393                  struct hist_trigger_attrs *attrs,
4394                  struct trace_event_file *file,
4395                  bool remove)
4396 {
4397         const struct tracing_map_ops *map_ops = NULL;
4398         struct hist_trigger_data *hist_data;
4399         int ret = 0;
4400
4401         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4402         if (!hist_data)
4403                 return ERR_PTR(-ENOMEM);
4404
4405         hist_data->attrs = attrs;
4406         hist_data->remove = remove;
4407         hist_data->event_file = file;
4408
4409         ret = parse_actions(hist_data);
4410         if (ret)
4411                 goto free;
4412
4413         ret = create_hist_fields(hist_data, file);
4414         if (ret)
4415                 goto free;
4416
4417         ret = create_sort_keys(hist_data);
4418         if (ret)
4419                 goto free;
4420
4421         map_ops = &hist_trigger_elt_data_ops;
4422
4423         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4424                                             map_ops, hist_data);
4425         if (IS_ERR(hist_data->map)) {
4426                 ret = PTR_ERR(hist_data->map);
4427                 hist_data->map = NULL;
4428                 goto free;
4429         }
4430
4431         ret = create_tracing_map_fields(hist_data);
4432         if (ret)
4433                 goto free;
4434  out:
4435         return hist_data;
4436  free:
4437         hist_data->attrs = NULL;
4438
4439         destroy_hist_data(hist_data);
4440
4441         hist_data = ERR_PTR(ret);
4442
4443         goto out;
4444 }
4445
4446 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4447                                     struct tracing_map_elt *elt, void *rec,
4448                                     struct ring_buffer_event *rbe,
4449                                     u64 *var_ref_vals)
4450 {
4451         struct hist_elt_data *elt_data;
4452         struct hist_field *hist_field;
4453         unsigned int i, var_idx;
4454         u64 hist_val;
4455
4456         elt_data = elt->private_data;
4457         elt_data->var_ref_vals = var_ref_vals;
4458
4459         for_each_hist_val_field(i, hist_data) {
4460                 hist_field = hist_data->fields[i];
4461                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4462                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4463                         var_idx = hist_field->var.idx;
4464
4465                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4466                                 unsigned int str_start, var_str_idx, idx;
4467                                 char *str, *val_str;
4468
4469                                 str_start = hist_data->n_field_var_str +
4470                                         hist_data->n_save_var_str;
4471                                 var_str_idx = hist_field->var_str_idx;
4472                                 idx = str_start + var_str_idx;
4473
4474                                 str = elt_data->field_var_str[idx];
4475                                 val_str = (char *)(uintptr_t)hist_val;
4476                                 strscpy(str, val_str, hist_field->size);
4477
4478                                 hist_val = (u64)(uintptr_t)str;
4479                         }
4480                         tracing_map_set_var(elt, var_idx, hist_val);
4481                         continue;
4482                 }
4483                 tracing_map_update_sum(elt, i, hist_val);
4484         }
4485
4486         for_each_hist_key_field(i, hist_data) {
4487                 hist_field = hist_data->fields[i];
4488                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4489                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4490                         var_idx = hist_field->var.idx;
4491                         tracing_map_set_var(elt, var_idx, hist_val);
4492                 }
4493         }
4494
4495         update_field_vars(hist_data, elt, rbe, rec);
4496 }
4497
4498 static inline void add_to_key(char *compound_key, void *key,
4499                               struct hist_field *key_field, void *rec)
4500 {
4501         size_t size = key_field->size;
4502
4503         if (key_field->flags & HIST_FIELD_FL_STRING) {
4504                 struct ftrace_event_field *field;
4505
4506                 field = key_field->field;
4507                 if (field->filter_type == FILTER_DYN_STRING)
4508                         size = *(u32 *)(rec + field->offset) >> 16;
4509                 else if (field->filter_type == FILTER_STATIC_STRING)
4510                         size = field->size;
4511
4512                 /* ensure NULL-termination */
4513                 if (size > key_field->size - 1)
4514                         size = key_field->size - 1;
4515
4516                 strncpy(compound_key + key_field->offset, (char *)key, size);
4517         } else
4518                 memcpy(compound_key + key_field->offset, key, size);
4519 }
4520
4521 static void
4522 hist_trigger_actions(struct hist_trigger_data *hist_data,
4523                      struct tracing_map_elt *elt, void *rec,
4524                      struct ring_buffer_event *rbe, void *key,
4525                      u64 *var_ref_vals)
4526 {
4527         struct action_data *data;
4528         unsigned int i;
4529
4530         for (i = 0; i < hist_data->n_actions; i++) {
4531                 data = hist_data->actions[i];
4532                 data->fn(hist_data, elt, rec, rbe, key, data, var_ref_vals);
4533         }
4534 }
4535
4536 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4537                                struct ring_buffer_event *rbe)
4538 {
4539         struct hist_trigger_data *hist_data = data->private_data;
4540         bool use_compound_key = (hist_data->n_keys > 1);
4541         unsigned long entries[HIST_STACKTRACE_DEPTH];
4542         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4543         char compound_key[HIST_KEY_SIZE_MAX];
4544         struct tracing_map_elt *elt = NULL;
4545         struct hist_field *key_field;
4546         u64 field_contents;
4547         void *key = NULL;
4548         unsigned int i;
4549
4550         memset(compound_key, 0, hist_data->key_size);
4551
4552         for_each_hist_key_field(i, hist_data) {
4553                 key_field = hist_data->fields[i];
4554
4555                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4556                         memset(entries, 0, HIST_STACKTRACE_SIZE);
4557                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4558                                          HIST_STACKTRACE_SKIP);
4559                         key = entries;
4560                 } else {
4561                         field_contents = key_field->fn(key_field, elt, rbe, rec);
4562                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4563                                 key = (void *)(unsigned long)field_contents;
4564                                 use_compound_key = true;
4565                         } else
4566                                 key = (void *)&field_contents;
4567                 }
4568
4569                 if (use_compound_key)
4570                         add_to_key(compound_key, key, key_field, rec);
4571         }
4572
4573         if (use_compound_key)
4574                 key = compound_key;
4575
4576         if (hist_data->n_var_refs &&
4577             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4578                 return;
4579
4580         elt = tracing_map_insert(hist_data->map, key);
4581         if (!elt)
4582                 return;
4583
4584         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4585
4586         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4587                 hist_trigger_actions(hist_data, elt, rec, rbe, key, var_ref_vals);
4588 }
4589
4590 static void hist_trigger_stacktrace_print(struct seq_file *m,
4591                                           unsigned long *stacktrace_entries,
4592                                           unsigned int max_entries)
4593 {
4594         char str[KSYM_SYMBOL_LEN];
4595         unsigned int spaces = 8;
4596         unsigned int i;
4597
4598         for (i = 0; i < max_entries; i++) {
4599                 if (!stacktrace_entries[i])
4600                         return;
4601
4602                 seq_printf(m, "%*c", 1 + spaces, ' ');
4603                 sprint_symbol(str, stacktrace_entries[i]);
4604                 seq_printf(m, "%s\n", str);
4605         }
4606 }
4607
4608 static void hist_trigger_print_key(struct seq_file *m,
4609                                    struct hist_trigger_data *hist_data,
4610                                    void *key,
4611                                    struct tracing_map_elt *elt)
4612 {
4613         struct hist_field *key_field;
4614         char str[KSYM_SYMBOL_LEN];
4615         bool multiline = false;
4616         const char *field_name;
4617         unsigned int i;
4618         u64 uval;
4619
4620         seq_puts(m, "{ ");
4621
4622         for_each_hist_key_field(i, hist_data) {
4623                 key_field = hist_data->fields[i];
4624
4625                 if (i > hist_data->n_vals)
4626                         seq_puts(m, ", ");
4627
4628                 field_name = hist_field_name(key_field, 0);
4629
4630                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4631                         uval = *(u64 *)(key + key_field->offset);
4632                         seq_printf(m, "%s: %llx", field_name, uval);
4633                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4634                         uval = *(u64 *)(key + key_field->offset);
4635                         sprint_symbol_no_offset(str, uval);
4636                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4637                                    uval, str);
4638                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4639                         uval = *(u64 *)(key + key_field->offset);
4640                         sprint_symbol(str, uval);
4641                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4642                                    uval, str);
4643                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4644                         struct hist_elt_data *elt_data = elt->private_data;
4645                         char *comm;
4646
4647                         if (WARN_ON_ONCE(!elt_data))
4648                                 return;
4649
4650                         comm = elt_data->comm;
4651
4652                         uval = *(u64 *)(key + key_field->offset);
4653                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4654                                    comm, uval);
4655                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4656                         const char *syscall_name;
4657
4658                         uval = *(u64 *)(key + key_field->offset);
4659                         syscall_name = get_syscall_name(uval);
4660                         if (!syscall_name)
4661                                 syscall_name = "unknown_syscall";
4662
4663                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4664                                    syscall_name, uval);
4665                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4666                         seq_puts(m, "stacktrace:\n");
4667                         hist_trigger_stacktrace_print(m,
4668                                                       key + key_field->offset,
4669                                                       HIST_STACKTRACE_DEPTH);
4670                         multiline = true;
4671                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4672                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4673                                    *(u64 *)(key + key_field->offset));
4674                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4675                         seq_printf(m, "%s: %-50s", field_name,
4676                                    (char *)(key + key_field->offset));
4677                 } else {
4678                         uval = *(u64 *)(key + key_field->offset);
4679                         seq_printf(m, "%s: %10llu", field_name, uval);
4680                 }
4681         }
4682
4683         if (!multiline)
4684                 seq_puts(m, " ");
4685
4686         seq_puts(m, "}");
4687 }
4688
4689 static void hist_trigger_entry_print(struct seq_file *m,
4690                                      struct hist_trigger_data *hist_data,
4691                                      void *key,
4692                                      struct tracing_map_elt *elt)
4693 {
4694         const char *field_name;
4695         unsigned int i;
4696
4697         hist_trigger_print_key(m, hist_data, key, elt);
4698
4699         seq_printf(m, " hitcount: %10llu",
4700                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4701
4702         for (i = 1; i < hist_data->n_vals; i++) {
4703                 field_name = hist_field_name(hist_data->fields[i], 0);
4704
4705                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4706                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4707                         continue;
4708
4709                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4710                         seq_printf(m, "  %s: %10llx", field_name,
4711                                    tracing_map_read_sum(elt, i));
4712                 } else {
4713                         seq_printf(m, "  %s: %10llu", field_name,
4714                                    tracing_map_read_sum(elt, i));
4715                 }
4716         }
4717
4718         print_actions(m, hist_data, elt);
4719
4720         seq_puts(m, "\n");
4721 }
4722
4723 static int print_entries(struct seq_file *m,
4724                          struct hist_trigger_data *hist_data)
4725 {
4726         struct tracing_map_sort_entry **sort_entries = NULL;
4727         struct tracing_map *map = hist_data->map;
4728         int i, n_entries;
4729
4730         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4731                                              hist_data->n_sort_keys,
4732                                              &sort_entries);
4733         if (n_entries < 0)
4734                 return n_entries;
4735
4736         for (i = 0; i < n_entries; i++)
4737                 hist_trigger_entry_print(m, hist_data,
4738                                          sort_entries[i]->key,
4739                                          sort_entries[i]->elt);
4740
4741         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4742
4743         return n_entries;
4744 }
4745
4746 static void hist_trigger_show(struct seq_file *m,
4747                               struct event_trigger_data *data, int n)
4748 {
4749         struct hist_trigger_data *hist_data;
4750         int n_entries;
4751
4752         if (n > 0)
4753                 seq_puts(m, "\n\n");
4754
4755         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4756         data->ops->print(m, data->ops, data);
4757         seq_puts(m, "#\n\n");
4758
4759         hist_data = data->private_data;
4760         n_entries = print_entries(m, hist_data);
4761         if (n_entries < 0)
4762                 n_entries = 0;
4763
4764         track_data_snapshot_print(m, hist_data);
4765
4766         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4767                    (u64)atomic64_read(&hist_data->map->hits),
4768                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4769 }
4770
4771 static int hist_show(struct seq_file *m, void *v)
4772 {
4773         struct event_trigger_data *data;
4774         struct trace_event_file *event_file;
4775         int n = 0, ret = 0;
4776
4777         mutex_lock(&event_mutex);
4778
4779         event_file = event_file_data(m->private);
4780         if (unlikely(!event_file)) {
4781                 ret = -ENODEV;
4782                 goto out_unlock;
4783         }
4784
4785         list_for_each_entry(data, &event_file->triggers, list) {
4786                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4787                         hist_trigger_show(m, data, n++);
4788         }
4789
4790  out_unlock:
4791         mutex_unlock(&event_mutex);
4792
4793         return ret;
4794 }
4795
4796 static int event_hist_open(struct inode *inode, struct file *file)
4797 {
4798         int ret;
4799
4800         ret = security_locked_down(LOCKDOWN_TRACEFS);
4801         if (ret)
4802                 return ret;
4803
4804         return single_open(file, hist_show, file);
4805 }
4806
4807 const struct file_operations event_hist_fops = {
4808         .open = event_hist_open,
4809         .read = seq_read,
4810         .llseek = seq_lseek,
4811         .release = single_release,
4812 };
4813
4814 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4815 static void hist_field_debug_show_flags(struct seq_file *m,
4816                                         unsigned long flags)
4817 {
4818         seq_puts(m, "      flags:\n");
4819
4820         if (flags & HIST_FIELD_FL_KEY)
4821                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4822         else if (flags & HIST_FIELD_FL_HITCOUNT)
4823                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4824         else if (flags & HIST_FIELD_FL_VAR)
4825                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4826         else if (flags & HIST_FIELD_FL_VAR_REF)
4827                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4828         else
4829                 seq_puts(m, "        VAL: normal u64 value\n");
4830
4831         if (flags & HIST_FIELD_FL_ALIAS)
4832                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4833 }
4834
4835 static int hist_field_debug_show(struct seq_file *m,
4836                                  struct hist_field *field, unsigned long flags)
4837 {
4838         if ((field->flags & flags) != flags) {
4839                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4840                 return -EINVAL;
4841         }
4842
4843         hist_field_debug_show_flags(m, field->flags);
4844         if (field->field)
4845                 seq_printf(m, "      ftrace_event_field name: %s\n",
4846                            field->field->name);
4847
4848         if (field->flags & HIST_FIELD_FL_VAR) {
4849                 seq_printf(m, "      var.name: %s\n", field->var.name);
4850                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4851                            field->var.idx);
4852         }
4853
4854         if (field->flags & HIST_FIELD_FL_ALIAS)
4855                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4856                            field->var_ref_idx);
4857
4858         if (field->flags & HIST_FIELD_FL_VAR_REF) {
4859                 seq_printf(m, "      name: %s\n", field->name);
4860                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4861                            field->var.idx);
4862                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
4863                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
4864                            field->var_ref_idx);
4865                 if (field->system)
4866                         seq_printf(m, "      system: %s\n", field->system);
4867                 if (field->event_name)
4868                         seq_printf(m, "      event_name: %s\n", field->event_name);
4869         }
4870
4871         seq_printf(m, "      type: %s\n", field->type);
4872         seq_printf(m, "      size: %u\n", field->size);
4873         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4874
4875         return 0;
4876 }
4877
4878 static int field_var_debug_show(struct seq_file *m,
4879                                 struct field_var *field_var, unsigned int i,
4880                                 bool save_vars)
4881 {
4882         const char *vars_name = save_vars ? "save_vars" : "field_vars";
4883         struct hist_field *field;
4884         int ret = 0;
4885
4886         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
4887
4888         field = field_var->var;
4889
4890         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
4891
4892         hist_field_debug_show_flags(m, field->flags);
4893         seq_printf(m, "      var.name: %s\n", field->var.name);
4894         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
4895                    field->var.idx);
4896
4897         field = field_var->val;
4898
4899         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
4900         if (field->field)
4901                 seq_printf(m, "      ftrace_event_field name: %s\n",
4902                            field->field->name);
4903         else {
4904                 ret = -EINVAL;
4905                 goto out;
4906         }
4907
4908         seq_printf(m, "      type: %s\n", field->type);
4909         seq_printf(m, "      size: %u\n", field->size);
4910         seq_printf(m, "      is_signed: %u\n", field->is_signed);
4911 out:
4912         return ret;
4913 }
4914
4915 static int hist_action_debug_show(struct seq_file *m,
4916                                   struct action_data *data, int i)
4917 {
4918         int ret = 0;
4919
4920         if (data->handler == HANDLER_ONMAX ||
4921             data->handler == HANDLER_ONCHANGE) {
4922                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
4923                 ret = hist_field_debug_show(m, data->track_data.var_ref,
4924                                             HIST_FIELD_FL_VAR_REF);
4925                 if (ret)
4926                         goto out;
4927
4928                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
4929                 ret = hist_field_debug_show(m, data->track_data.track_var,
4930                                             HIST_FIELD_FL_VAR);
4931                 if (ret)
4932                         goto out;
4933         }
4934
4935         if (data->handler == HANDLER_ONMATCH) {
4936                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
4937                            i, data->match_data.event_system);
4938                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
4939                            i, data->match_data.event);
4940         }
4941 out:
4942         return ret;
4943 }
4944
4945 static int hist_actions_debug_show(struct seq_file *m,
4946                                    struct hist_trigger_data *hist_data)
4947 {
4948         int i, ret = 0;
4949
4950         if (hist_data->n_actions)
4951                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
4952
4953         for (i = 0; i < hist_data->n_actions; i++) {
4954                 struct action_data *action = hist_data->actions[i];
4955
4956                 ret = hist_action_debug_show(m, action, i);
4957                 if (ret)
4958                         goto out;
4959         }
4960
4961         if (hist_data->n_save_vars)
4962                 seq_puts(m, "\n  save action variables (save() params):\n");
4963
4964         for (i = 0; i < hist_data->n_save_vars; i++) {
4965                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
4966                 if (ret)
4967                         goto out;
4968         }
4969 out:
4970         return ret;
4971 }
4972
4973 static void hist_trigger_debug_show(struct seq_file *m,
4974                                     struct event_trigger_data *data, int n)
4975 {
4976         struct hist_trigger_data *hist_data;
4977         int i, ret;
4978
4979         if (n > 0)
4980                 seq_puts(m, "\n\n");
4981
4982         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4983         data->ops->print(m, data->ops, data);
4984         seq_puts(m, "#\n\n");
4985
4986         hist_data = data->private_data;
4987
4988         seq_printf(m, "hist_data: %p\n\n", hist_data);
4989         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
4990         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
4991         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
4992
4993         seq_puts(m, "\n  val fields:\n\n");
4994
4995         seq_puts(m, "    hist_data->fields[0]:\n");
4996         ret = hist_field_debug_show(m, hist_data->fields[0],
4997                                     HIST_FIELD_FL_HITCOUNT);
4998         if (ret)
4999                 return;
5000
5001         for (i = 1; i < hist_data->n_vals; i++) {
5002                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5003                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5004                 if (ret)
5005                         return;
5006         }
5007
5008         seq_puts(m, "\n  key fields:\n");
5009
5010         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5011                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5012                 ret = hist_field_debug_show(m, hist_data->fields[i],
5013                                             HIST_FIELD_FL_KEY);
5014                 if (ret)
5015                         return;
5016         }
5017
5018         if (hist_data->n_var_refs)
5019                 seq_puts(m, "\n  variable reference fields:\n");
5020
5021         for (i = 0; i < hist_data->n_var_refs; i++) {
5022                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5023                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5024                                             HIST_FIELD_FL_VAR_REF);
5025                 if (ret)
5026                         return;
5027         }
5028
5029         if (hist_data->n_field_vars)
5030                 seq_puts(m, "\n  field variables:\n");
5031
5032         for (i = 0; i < hist_data->n_field_vars; i++) {
5033                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5034                 if (ret)
5035                         return;
5036         }
5037
5038         ret = hist_actions_debug_show(m, hist_data);
5039         if (ret)
5040                 return;
5041 }
5042
5043 static int hist_debug_show(struct seq_file *m, void *v)
5044 {
5045         struct event_trigger_data *data;
5046         struct trace_event_file *event_file;
5047         int n = 0, ret = 0;
5048
5049         mutex_lock(&event_mutex);
5050
5051         event_file = event_file_data(m->private);
5052         if (unlikely(!event_file)) {
5053                 ret = -ENODEV;
5054                 goto out_unlock;
5055         }
5056
5057         list_for_each_entry(data, &event_file->triggers, list) {
5058                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5059                         hist_trigger_debug_show(m, data, n++);
5060         }
5061
5062  out_unlock:
5063         mutex_unlock(&event_mutex);
5064
5065         return ret;
5066 }
5067
5068 static int event_hist_debug_open(struct inode *inode, struct file *file)
5069 {
5070         int ret;
5071
5072         ret = security_locked_down(LOCKDOWN_TRACEFS);
5073         if (ret)
5074                 return ret;
5075
5076         return single_open(file, hist_debug_show, file);
5077 }
5078
5079 const struct file_operations event_hist_debug_fops = {
5080         .open = event_hist_debug_open,
5081         .read = seq_read,
5082         .llseek = seq_lseek,
5083         .release = single_release,
5084 };
5085 #endif
5086
5087 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5088 {
5089         const char *field_name = hist_field_name(hist_field, 0);
5090
5091         if (hist_field->var.name)
5092                 seq_printf(m, "%s=", hist_field->var.name);
5093
5094         if (hist_field->flags & HIST_FIELD_FL_CPU)
5095                 seq_puts(m, "common_cpu");
5096         else if (field_name) {
5097                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5098                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5099                         seq_putc(m, '$');
5100                 seq_printf(m, "%s", field_name);
5101         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5102                 seq_puts(m, "common_timestamp");
5103
5104         if (hist_field->flags) {
5105                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5106                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5107                         const char *flags = get_hist_field_flags(hist_field);
5108
5109                         if (flags)
5110                                 seq_printf(m, ".%s", flags);
5111                 }
5112         }
5113 }
5114
5115 static int event_hist_trigger_print(struct seq_file *m,
5116                                     struct event_trigger_ops *ops,
5117                                     struct event_trigger_data *data)
5118 {
5119         struct hist_trigger_data *hist_data = data->private_data;
5120         struct hist_field *field;
5121         bool have_var = false;
5122         unsigned int i;
5123
5124         seq_puts(m, "hist:");
5125
5126         if (data->name)
5127                 seq_printf(m, "%s:", data->name);
5128
5129         seq_puts(m, "keys=");
5130
5131         for_each_hist_key_field(i, hist_data) {
5132                 field = hist_data->fields[i];
5133
5134                 if (i > hist_data->n_vals)
5135                         seq_puts(m, ",");
5136
5137                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5138                         seq_puts(m, "stacktrace");
5139                 else
5140                         hist_field_print(m, field);
5141         }
5142
5143         seq_puts(m, ":vals=");
5144
5145         for_each_hist_val_field(i, hist_data) {
5146                 field = hist_data->fields[i];
5147                 if (field->flags & HIST_FIELD_FL_VAR) {
5148                         have_var = true;
5149                         continue;
5150                 }
5151
5152                 if (i == HITCOUNT_IDX)
5153                         seq_puts(m, "hitcount");
5154                 else {
5155                         seq_puts(m, ",");
5156                         hist_field_print(m, field);
5157                 }
5158         }
5159
5160         if (have_var) {
5161                 unsigned int n = 0;
5162
5163                 seq_puts(m, ":");
5164
5165                 for_each_hist_val_field(i, hist_data) {
5166                         field = hist_data->fields[i];
5167
5168                         if (field->flags & HIST_FIELD_FL_VAR) {
5169                                 if (n++)
5170                                         seq_puts(m, ",");
5171                                 hist_field_print(m, field);
5172                         }
5173                 }
5174         }
5175
5176         seq_puts(m, ":sort=");
5177
5178         for (i = 0; i < hist_data->n_sort_keys; i++) {
5179                 struct tracing_map_sort_key *sort_key;
5180                 unsigned int idx, first_key_idx;
5181
5182                 /* skip VAR vals */
5183                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5184
5185                 sort_key = &hist_data->sort_keys[i];
5186                 idx = sort_key->field_idx;
5187
5188                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5189                         return -EINVAL;
5190
5191                 if (i > 0)
5192                         seq_puts(m, ",");
5193
5194                 if (idx == HITCOUNT_IDX)
5195                         seq_puts(m, "hitcount");
5196                 else {
5197                         if (idx >= first_key_idx)
5198                                 idx += hist_data->n_vars;
5199                         hist_field_print(m, hist_data->fields[idx]);
5200                 }
5201
5202                 if (sort_key->descending)
5203                         seq_puts(m, ".descending");
5204         }
5205         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5206         if (hist_data->enable_timestamps)
5207                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5208
5209         print_actions_spec(m, hist_data);
5210
5211         if (data->filter_str)
5212                 seq_printf(m, " if %s", data->filter_str);
5213
5214         if (data->paused)
5215                 seq_puts(m, " [paused]");
5216         else
5217                 seq_puts(m, " [active]");
5218
5219         seq_putc(m, '\n');
5220
5221         return 0;
5222 }
5223
5224 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5225                                    struct event_trigger_data *data)
5226 {
5227         struct hist_trigger_data *hist_data = data->private_data;
5228
5229         if (!data->ref && hist_data->attrs->name)
5230                 save_named_trigger(hist_data->attrs->name, data);
5231
5232         data->ref++;
5233
5234         return 0;
5235 }
5236
5237 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5238 {
5239         struct trace_event_file *file;
5240         unsigned int i;
5241         char *cmd;
5242         int ret;
5243
5244         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5245                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5246                 cmd = hist_data->field_var_hists[i]->cmd;
5247                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5248                                               "!hist", "hist", cmd);
5249         }
5250 }
5251
5252 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5253                                     struct event_trigger_data *data)
5254 {
5255         struct hist_trigger_data *hist_data = data->private_data;
5256
5257         if (WARN_ON_ONCE(data->ref <= 0))
5258                 return;
5259
5260         data->ref--;
5261         if (!data->ref) {
5262                 if (data->name)
5263                         del_named_trigger(data);
5264
5265                 trigger_data_free(data);
5266
5267                 remove_hist_vars(hist_data);
5268
5269                 unregister_field_var_hists(hist_data);
5270
5271                 destroy_hist_data(hist_data);
5272         }
5273 }
5274
5275 static struct event_trigger_ops event_hist_trigger_ops = {
5276         .func                   = event_hist_trigger,
5277         .print                  = event_hist_trigger_print,
5278         .init                   = event_hist_trigger_init,
5279         .free                   = event_hist_trigger_free,
5280 };
5281
5282 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5283                                          struct event_trigger_data *data)
5284 {
5285         data->ref++;
5286
5287         save_named_trigger(data->named_data->name, data);
5288
5289         event_hist_trigger_init(ops, data->named_data);
5290
5291         return 0;
5292 }
5293
5294 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5295                                           struct event_trigger_data *data)
5296 {
5297         if (WARN_ON_ONCE(data->ref <= 0))
5298                 return;
5299
5300         event_hist_trigger_free(ops, data->named_data);
5301
5302         data->ref--;
5303         if (!data->ref) {
5304                 del_named_trigger(data);
5305                 trigger_data_free(data);
5306         }
5307 }
5308
5309 static struct event_trigger_ops event_hist_trigger_named_ops = {
5310         .func                   = event_hist_trigger,
5311         .print                  = event_hist_trigger_print,
5312         .init                   = event_hist_trigger_named_init,
5313         .free                   = event_hist_trigger_named_free,
5314 };
5315
5316 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5317                                                             char *param)
5318 {
5319         return &event_hist_trigger_ops;
5320 }
5321
5322 static void hist_clear(struct event_trigger_data *data)
5323 {
5324         struct hist_trigger_data *hist_data = data->private_data;
5325
5326         if (data->name)
5327                 pause_named_trigger(data);
5328
5329         tracepoint_synchronize_unregister();
5330
5331         tracing_map_clear(hist_data->map);
5332
5333         if (data->name)
5334                 unpause_named_trigger(data);
5335 }
5336
5337 static bool compatible_field(struct ftrace_event_field *field,
5338                              struct ftrace_event_field *test_field)
5339 {
5340         if (field == test_field)
5341                 return true;
5342         if (field == NULL || test_field == NULL)
5343                 return false;
5344         if (strcmp(field->name, test_field->name) != 0)
5345                 return false;
5346         if (strcmp(field->type, test_field->type) != 0)
5347                 return false;
5348         if (field->size != test_field->size)
5349                 return false;
5350         if (field->is_signed != test_field->is_signed)
5351                 return false;
5352
5353         return true;
5354 }
5355
5356 static bool hist_trigger_match(struct event_trigger_data *data,
5357                                struct event_trigger_data *data_test,
5358                                struct event_trigger_data *named_data,
5359                                bool ignore_filter)
5360 {
5361         struct tracing_map_sort_key *sort_key, *sort_key_test;
5362         struct hist_trigger_data *hist_data, *hist_data_test;
5363         struct hist_field *key_field, *key_field_test;
5364         unsigned int i;
5365
5366         if (named_data && (named_data != data_test) &&
5367             (named_data != data_test->named_data))
5368                 return false;
5369
5370         if (!named_data && is_named_trigger(data_test))
5371                 return false;
5372
5373         hist_data = data->private_data;
5374         hist_data_test = data_test->private_data;
5375
5376         if (hist_data->n_vals != hist_data_test->n_vals ||
5377             hist_data->n_fields != hist_data_test->n_fields ||
5378             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5379                 return false;
5380
5381         if (!ignore_filter) {
5382                 if ((data->filter_str && !data_test->filter_str) ||
5383                    (!data->filter_str && data_test->filter_str))
5384                         return false;
5385         }
5386
5387         for_each_hist_field(i, hist_data) {
5388                 key_field = hist_data->fields[i];
5389                 key_field_test = hist_data_test->fields[i];
5390
5391                 if (key_field->flags != key_field_test->flags)
5392                         return false;
5393                 if (!compatible_field(key_field->field, key_field_test->field))
5394                         return false;
5395                 if (key_field->offset != key_field_test->offset)
5396                         return false;
5397                 if (key_field->size != key_field_test->size)
5398                         return false;
5399                 if (key_field->is_signed != key_field_test->is_signed)
5400                         return false;
5401                 if (!!key_field->var.name != !!key_field_test->var.name)
5402                         return false;
5403                 if (key_field->var.name &&
5404                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5405                         return false;
5406         }
5407
5408         for (i = 0; i < hist_data->n_sort_keys; i++) {
5409                 sort_key = &hist_data->sort_keys[i];
5410                 sort_key_test = &hist_data_test->sort_keys[i];
5411
5412                 if (sort_key->field_idx != sort_key_test->field_idx ||
5413                     sort_key->descending != sort_key_test->descending)
5414                         return false;
5415         }
5416
5417         if (!ignore_filter && data->filter_str &&
5418             (strcmp(data->filter_str, data_test->filter_str) != 0))
5419                 return false;
5420
5421         if (!actions_match(hist_data, hist_data_test))
5422                 return false;
5423
5424         return true;
5425 }
5426
5427 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5428                                  struct event_trigger_data *data,
5429                                  struct trace_event_file *file)
5430 {
5431         struct hist_trigger_data *hist_data = data->private_data;
5432         struct event_trigger_data *test, *named_data = NULL;
5433         struct trace_array *tr = file->tr;
5434         int ret = 0;
5435
5436         if (hist_data->attrs->name) {
5437                 named_data = find_named_trigger(hist_data->attrs->name);
5438                 if (named_data) {
5439                         if (!hist_trigger_match(data, named_data, named_data,
5440                                                 true)) {
5441                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5442                                 ret = -EINVAL;
5443                                 goto out;
5444                         }
5445                 }
5446         }
5447
5448         if (hist_data->attrs->name && !named_data)
5449                 goto new;
5450
5451         lockdep_assert_held(&event_mutex);
5452
5453         list_for_each_entry(test, &file->triggers, list) {
5454                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5455                         if (!hist_trigger_match(data, test, named_data, false))
5456                                 continue;
5457                         if (hist_data->attrs->pause)
5458                                 test->paused = true;
5459                         else if (hist_data->attrs->cont)
5460                                 test->paused = false;
5461                         else if (hist_data->attrs->clear)
5462                                 hist_clear(test);
5463                         else {
5464                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5465                                 ret = -EEXIST;
5466                         }
5467                         goto out;
5468                 }
5469         }
5470  new:
5471         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5472                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5473                 ret = -ENOENT;
5474                 goto out;
5475         }
5476
5477         if (hist_data->attrs->pause)
5478                 data->paused = true;
5479
5480         if (named_data) {
5481                 data->private_data = named_data->private_data;
5482                 set_named_trigger_data(data, named_data);
5483                 data->ops = &event_hist_trigger_named_ops;
5484         }
5485
5486         if (data->ops->init) {
5487                 ret = data->ops->init(data->ops, data);
5488                 if (ret < 0)
5489                         goto out;
5490         }
5491
5492         if (hist_data->enable_timestamps) {
5493                 char *clock = hist_data->attrs->clock;
5494
5495                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5496                 if (ret) {
5497                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5498                         goto out;
5499                 }
5500
5501                 tracing_set_time_stamp_abs(file->tr, true);
5502         }
5503
5504         if (named_data)
5505                 destroy_hist_data(hist_data);
5506
5507         ret++;
5508  out:
5509         return ret;
5510 }
5511
5512 static int hist_trigger_enable(struct event_trigger_data *data,
5513                                struct trace_event_file *file)
5514 {
5515         int ret = 0;
5516
5517         list_add_tail_rcu(&data->list, &file->triggers);
5518
5519         update_cond_flag(file);
5520
5521         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5522                 list_del_rcu(&data->list);
5523                 update_cond_flag(file);
5524                 ret--;
5525         }
5526
5527         return ret;
5528 }
5529
5530 static bool have_hist_trigger_match(struct event_trigger_data *data,
5531                                     struct trace_event_file *file)
5532 {
5533         struct hist_trigger_data *hist_data = data->private_data;
5534         struct event_trigger_data *test, *named_data = NULL;
5535         bool match = false;
5536
5537         lockdep_assert_held(&event_mutex);
5538
5539         if (hist_data->attrs->name)
5540                 named_data = find_named_trigger(hist_data->attrs->name);
5541
5542         list_for_each_entry(test, &file->triggers, list) {
5543                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5544                         if (hist_trigger_match(data, test, named_data, false)) {
5545                                 match = true;
5546                                 break;
5547                         }
5548                 }
5549         }
5550
5551         return match;
5552 }
5553
5554 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5555                                     struct trace_event_file *file)
5556 {
5557         struct hist_trigger_data *hist_data = data->private_data;
5558         struct event_trigger_data *test, *named_data = NULL;
5559
5560         lockdep_assert_held(&event_mutex);
5561
5562         if (hist_data->attrs->name)
5563                 named_data = find_named_trigger(hist_data->attrs->name);
5564
5565         list_for_each_entry(test, &file->triggers, list) {
5566                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5567                         if (!hist_trigger_match(data, test, named_data, false))
5568                                 continue;
5569                         hist_data = test->private_data;
5570                         if (check_var_refs(hist_data))
5571                                 return true;
5572                         break;
5573                 }
5574         }
5575
5576         return false;
5577 }
5578
5579 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5580                                     struct event_trigger_data *data,
5581                                     struct trace_event_file *file)
5582 {
5583         struct hist_trigger_data *hist_data = data->private_data;
5584         struct event_trigger_data *test, *named_data = NULL;
5585         bool unregistered = false;
5586
5587         lockdep_assert_held(&event_mutex);
5588
5589         if (hist_data->attrs->name)
5590                 named_data = find_named_trigger(hist_data->attrs->name);
5591
5592         list_for_each_entry(test, &file->triggers, list) {
5593                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5594                         if (!hist_trigger_match(data, test, named_data, false))
5595                                 continue;
5596                         unregistered = true;
5597                         list_del_rcu(&test->list);
5598                         trace_event_trigger_enable_disable(file, 0);
5599                         update_cond_flag(file);
5600                         break;
5601                 }
5602         }
5603
5604         if (unregistered && test->ops->free)
5605                 test->ops->free(test->ops, test);
5606
5607         if (hist_data->enable_timestamps) {
5608                 if (!hist_data->remove || unregistered)
5609                         tracing_set_time_stamp_abs(file->tr, false);
5610         }
5611 }
5612
5613 static bool hist_file_check_refs(struct trace_event_file *file)
5614 {
5615         struct hist_trigger_data *hist_data;
5616         struct event_trigger_data *test;
5617
5618         lockdep_assert_held(&event_mutex);
5619
5620         list_for_each_entry(test, &file->triggers, list) {
5621                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5622                         hist_data = test->private_data;
5623                         if (check_var_refs(hist_data))
5624                                 return true;
5625                 }
5626         }
5627
5628         return false;
5629 }
5630
5631 static void hist_unreg_all(struct trace_event_file *file)
5632 {
5633         struct event_trigger_data *test, *n;
5634         struct hist_trigger_data *hist_data;
5635         struct synth_event *se;
5636         const char *se_name;
5637
5638         lockdep_assert_held(&event_mutex);
5639
5640         if (hist_file_check_refs(file))
5641                 return;
5642
5643         list_for_each_entry_safe(test, n, &file->triggers, list) {
5644                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5645                         hist_data = test->private_data;
5646                         list_del_rcu(&test->list);
5647                         trace_event_trigger_enable_disable(file, 0);
5648
5649                         se_name = trace_event_name(file->event_call);
5650                         se = find_synth_event(se_name);
5651                         if (se)
5652                                 se->ref--;
5653
5654                         update_cond_flag(file);
5655                         if (hist_data->enable_timestamps)
5656                                 tracing_set_time_stamp_abs(file->tr, false);
5657                         if (test->ops->free)
5658                                 test->ops->free(test->ops, test);
5659                 }
5660         }
5661 }
5662
5663 static int event_hist_trigger_func(struct event_command *cmd_ops,
5664                                    struct trace_event_file *file,
5665                                    char *glob, char *cmd, char *param)
5666 {
5667         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5668         struct event_trigger_data *trigger_data;
5669         struct hist_trigger_attrs *attrs;
5670         struct event_trigger_ops *trigger_ops;
5671         struct hist_trigger_data *hist_data;
5672         struct synth_event *se;
5673         const char *se_name;
5674         bool remove = false;
5675         char *trigger, *p;
5676         int ret = 0;
5677
5678         lockdep_assert_held(&event_mutex);
5679
5680         if (glob && strlen(glob)) {
5681                 hist_err_clear();
5682                 last_cmd_set(file, param);
5683         }
5684
5685         if (!param)
5686                 return -EINVAL;
5687
5688         if (glob[0] == '!')
5689                 remove = true;
5690
5691         /*
5692          * separate the trigger from the filter (k:v [if filter])
5693          * allowing for whitespace in the trigger
5694          */
5695         p = trigger = param;
5696         do {
5697                 p = strstr(p, "if");
5698                 if (!p)
5699                         break;
5700                 if (p == param)
5701                         return -EINVAL;
5702                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5703                         p++;
5704                         continue;
5705                 }
5706                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5707                         return -EINVAL;
5708                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5709                         p++;
5710                         continue;
5711                 }
5712                 break;
5713         } while (p);
5714
5715         if (!p)
5716                 param = NULL;
5717         else {
5718                 *(p - 1) = '\0';
5719                 param = strstrip(p);
5720                 trigger = strstrip(trigger);
5721         }
5722
5723         attrs = parse_hist_trigger_attrs(file->tr, trigger);
5724         if (IS_ERR(attrs))
5725                 return PTR_ERR(attrs);
5726
5727         if (attrs->map_bits)
5728                 hist_trigger_bits = attrs->map_bits;
5729
5730         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5731         if (IS_ERR(hist_data)) {
5732                 destroy_hist_trigger_attrs(attrs);
5733                 return PTR_ERR(hist_data);
5734         }
5735
5736         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5737
5738         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5739         if (!trigger_data) {
5740                 ret = -ENOMEM;
5741                 goto out_free;
5742         }
5743
5744         trigger_data->count = -1;
5745         trigger_data->ops = trigger_ops;
5746         trigger_data->cmd_ops = cmd_ops;
5747
5748         INIT_LIST_HEAD(&trigger_data->list);
5749         RCU_INIT_POINTER(trigger_data->filter, NULL);
5750
5751         trigger_data->private_data = hist_data;
5752
5753         /* if param is non-empty, it's supposed to be a filter */
5754         if (param && cmd_ops->set_filter) {
5755                 ret = cmd_ops->set_filter(param, trigger_data, file);
5756                 if (ret < 0)
5757                         goto out_free;
5758         }
5759
5760         if (remove) {
5761                 if (!have_hist_trigger_match(trigger_data, file))
5762                         goto out_free;
5763
5764                 if (hist_trigger_check_refs(trigger_data, file)) {
5765                         ret = -EBUSY;
5766                         goto out_free;
5767                 }
5768
5769                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5770                 se_name = trace_event_name(file->event_call);
5771                 se = find_synth_event(se_name);
5772                 if (se)
5773                         se->ref--;
5774                 ret = 0;
5775                 goto out_free;
5776         }
5777
5778         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5779         /*
5780          * The above returns on success the # of triggers registered,
5781          * but if it didn't register any it returns zero.  Consider no
5782          * triggers registered a failure too.
5783          */
5784         if (!ret) {
5785                 if (!(attrs->pause || attrs->cont || attrs->clear))
5786                         ret = -ENOENT;
5787                 goto out_free;
5788         } else if (ret < 0)
5789                 goto out_free;
5790
5791         if (get_named_trigger_data(trigger_data))
5792                 goto enable;
5793
5794         if (has_hist_vars(hist_data))
5795                 save_hist_vars(hist_data);
5796
5797         ret = create_actions(hist_data);
5798         if (ret)
5799                 goto out_unreg;
5800
5801         ret = tracing_map_init(hist_data->map);
5802         if (ret)
5803                 goto out_unreg;
5804 enable:
5805         ret = hist_trigger_enable(trigger_data, file);
5806         if (ret)
5807                 goto out_unreg;
5808
5809         se_name = trace_event_name(file->event_call);
5810         se = find_synth_event(se_name);
5811         if (se)
5812                 se->ref++;
5813         /* Just return zero, not the number of registered triggers */
5814         ret = 0;
5815  out:
5816         if (ret == 0)
5817                 hist_err_clear();
5818
5819         return ret;
5820  out_unreg:
5821         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5822  out_free:
5823         if (cmd_ops->set_filter)
5824                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5825
5826         remove_hist_vars(hist_data);
5827
5828         kfree(trigger_data);
5829
5830         destroy_hist_data(hist_data);
5831         goto out;
5832 }
5833
5834 static struct event_command trigger_hist_cmd = {
5835         .name                   = "hist",
5836         .trigger_type           = ETT_EVENT_HIST,
5837         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5838         .func                   = event_hist_trigger_func,
5839         .reg                    = hist_register_trigger,
5840         .unreg                  = hist_unregister_trigger,
5841         .unreg_all              = hist_unreg_all,
5842         .get_trigger_ops        = event_hist_get_trigger_ops,
5843         .set_filter             = set_trigger_filter,
5844 };
5845
5846 __init int register_trigger_hist_cmd(void)
5847 {
5848         int ret;
5849
5850         ret = register_event_command(&trigger_hist_cmd);
5851         WARN_ON(ret < 0);
5852
5853         return ret;
5854 }
5855
5856 static void
5857 hist_enable_trigger(struct event_trigger_data *data, void *rec,
5858                     struct ring_buffer_event *event)
5859 {
5860         struct enable_trigger_data *enable_data = data->private_data;
5861         struct event_trigger_data *test;
5862
5863         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
5864                                 lockdep_is_held(&event_mutex)) {
5865                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5866                         if (enable_data->enable)
5867                                 test->paused = false;
5868                         else
5869                                 test->paused = true;
5870                 }
5871         }
5872 }
5873
5874 static void
5875 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5876                           struct ring_buffer_event *event)
5877 {
5878         if (!data->count)
5879                 return;
5880
5881         if (data->count != -1)
5882                 (data->count)--;
5883
5884         hist_enable_trigger(data, rec, event);
5885 }
5886
5887 static struct event_trigger_ops hist_enable_trigger_ops = {
5888         .func                   = hist_enable_trigger,
5889         .print                  = event_enable_trigger_print,
5890         .init                   = event_trigger_init,
5891         .free                   = event_enable_trigger_free,
5892 };
5893
5894 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5895         .func                   = hist_enable_count_trigger,
5896         .print                  = event_enable_trigger_print,
5897         .init                   = event_trigger_init,
5898         .free                   = event_enable_trigger_free,
5899 };
5900
5901 static struct event_trigger_ops hist_disable_trigger_ops = {
5902         .func                   = hist_enable_trigger,
5903         .print                  = event_enable_trigger_print,
5904         .init                   = event_trigger_init,
5905         .free                   = event_enable_trigger_free,
5906 };
5907
5908 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5909         .func                   = hist_enable_count_trigger,
5910         .print                  = event_enable_trigger_print,
5911         .init                   = event_trigger_init,
5912         .free                   = event_enable_trigger_free,
5913 };
5914
5915 static struct event_trigger_ops *
5916 hist_enable_get_trigger_ops(char *cmd, char *param)
5917 {
5918         struct event_trigger_ops *ops;
5919         bool enable;
5920
5921         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5922
5923         if (enable)
5924                 ops = param ? &hist_enable_count_trigger_ops :
5925                         &hist_enable_trigger_ops;
5926         else
5927                 ops = param ? &hist_disable_count_trigger_ops :
5928                         &hist_disable_trigger_ops;
5929
5930         return ops;
5931 }
5932
5933 static void hist_enable_unreg_all(struct trace_event_file *file)
5934 {
5935         struct event_trigger_data *test, *n;
5936
5937         list_for_each_entry_safe(test, n, &file->triggers, list) {
5938                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5939                         list_del_rcu(&test->list);
5940                         update_cond_flag(file);
5941                         trace_event_trigger_enable_disable(file, 0);
5942                         if (test->ops->free)
5943                                 test->ops->free(test->ops, test);
5944                 }
5945         }
5946 }
5947
5948 static struct event_command trigger_hist_enable_cmd = {
5949         .name                   = ENABLE_HIST_STR,
5950         .trigger_type           = ETT_HIST_ENABLE,
5951         .func                   = event_enable_trigger_func,
5952         .reg                    = event_enable_register_trigger,
5953         .unreg                  = event_enable_unregister_trigger,
5954         .unreg_all              = hist_enable_unreg_all,
5955         .get_trigger_ops        = hist_enable_get_trigger_ops,
5956         .set_filter             = set_trigger_filter,
5957 };
5958
5959 static struct event_command trigger_hist_disable_cmd = {
5960         .name                   = DISABLE_HIST_STR,
5961         .trigger_type           = ETT_HIST_ENABLE,
5962         .func                   = event_enable_trigger_func,
5963         .reg                    = event_enable_register_trigger,
5964         .unreg                  = event_enable_unregister_trigger,
5965         .unreg_all              = hist_enable_unreg_all,
5966         .get_trigger_ops        = hist_enable_get_trigger_ops,
5967         .set_filter             = set_trigger_filter,
5968 };
5969
5970 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5971 {
5972         unregister_event_command(&trigger_hist_enable_cmd);
5973         unregister_event_command(&trigger_hist_disable_cmd);
5974 }
5975
5976 __init int register_trigger_hist_enable_disable_cmds(void)
5977 {
5978         int ret;
5979
5980         ret = register_event_command(&trigger_hist_enable_cmd);
5981         if (WARN_ON(ret < 0))
5982                 return ret;
5983         ret = register_event_command(&trigger_hist_disable_cmd);
5984         if (WARN_ON(ret < 0))
5985                 unregister_trigger_hist_enable_disable_cmds();
5986
5987         return ret;
5988 }