kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),
  73
  74 #undef C
  75 #define C(a, b)         HIST_ERR_##a
  76
  77 enum { ERRORS };
  78
  79 #undef C
  80 #define C(a, b)         b
  81
  82 static const char *err_text[] = { ERRORS };
  83
  84 struct hist_field;
  85
  86 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  87                                 struct tracing_map_elt *elt,
  88                                 struct trace_buffer *buffer,
  89                                 struct ring_buffer_event *rbe,
  90                                 void *event);
  91
  92 #define HIST_FIELD_OPERANDS_MAX 2
  93 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  94 #define HIST_ACTIONS_MAX        8
  95 #define HIST_CONST_DIGITS_MAX   21
  96 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  97
  98 enum field_op_id {
  99         FIELD_OP_NONE,
 100         FIELD_OP_PLUS,
 101         FIELD_OP_MINUS,
 102         FIELD_OP_UNARY_MINUS,
 103         FIELD_OP_DIV,
 104         FIELD_OP_MULT,
 105 };
 106
 107 enum hist_field_fn {
 108         HIST_FIELD_FN_NOP,
 109         HIST_FIELD_FN_VAR_REF,
 110         HIST_FIELD_FN_COUNTER,
 111         HIST_FIELD_FN_CONST,
 112         HIST_FIELD_FN_LOG2,
 113         HIST_FIELD_FN_BUCKET,
 114         HIST_FIELD_FN_TIMESTAMP,
 115         HIST_FIELD_FN_CPU,
 116         HIST_FIELD_FN_STRING,
 117         HIST_FIELD_FN_DYNSTRING,
 118         HIST_FIELD_FN_RELDYNSTRING,
 119         HIST_FIELD_FN_PSTRING,
 120         HIST_FIELD_FN_S64,
 121         HIST_FIELD_FN_U64,
 122         HIST_FIELD_FN_S32,
 123         HIST_FIELD_FN_U32,
 124         HIST_FIELD_FN_S16,
 125         HIST_FIELD_FN_U16,
 126         HIST_FIELD_FN_S8,
 127         HIST_FIELD_FN_U8,
 128         HIST_FIELD_FN_UMINUS,
 129         HIST_FIELD_FN_MINUS,
 130         HIST_FIELD_FN_PLUS,
 131         HIST_FIELD_FN_DIV,
 132         HIST_FIELD_FN_MULT,
 133         HIST_FIELD_FN_DIV_POWER2,
 134         HIST_FIELD_FN_DIV_NOT_POWER2,
 135         HIST_FIELD_FN_DIV_MULT_SHIFT,
 136         HIST_FIELD_FN_EXECNAME,
 137 };
 138
 139 /*
 140  * A hist_var (histogram variable) contains variable information for
 141  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 142  * flag set.  A hist_var has a variable name e.g. ts0, and is
 143  * associated with a given histogram trigger, as specified by
 144  * hist_data.  The hist_var idx is the unique index assigned to the
 145  * variable by the hist trigger's tracing_map.  The idx is what is
 146  * used to set a variable's value and, by a variable reference, to
 147  * retrieve it.
 148  */
 149 struct hist_var {
 150         char                            *name;
 151         struct hist_trigger_data        *hist_data;
 152         unsigned int                    idx;
 153 };
 154
 155 struct hist_field {
 156         struct ftrace_event_field       *field;
 157         unsigned long                   flags;
 158         unsigned long                   buckets;
 159         const char                      *type;
 160         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 161         struct hist_trigger_data        *hist_data;
 162         enum hist_field_fn              fn_num;
 163         unsigned int                    ref;
 164         unsigned int                    size;
 165         unsigned int                    offset;
 166         unsigned int                    is_signed;
 167
 168         /*
 169          * Variable fields contain variable-specific info in var.
 170          */
 171         struct hist_var                 var;
 172         enum field_op_id                operator;
 173         char                            *system;
 174         char                            *event_name;
 175
 176         /*
 177          * The name field is used for EXPR and VAR_REF fields.  VAR
 178          * fields contain the variable name in var.name.
 179          */
 180         char                            *name;
 181
 182         /*
 183          * When a histogram trigger is hit, if it has any references
 184          * to variables, the values of those variables are collected
 185          * into a var_ref_vals array by resolve_var_refs().  The
 186          * current value of each variable is read from the tracing_map
 187          * using the hist field's hist_var.idx and entered into the
 188          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 189          */
 190         unsigned int                    var_ref_idx;
 191         bool                            read_once;
 192
 193         unsigned int                    var_str_idx;
 194
 195         /* Numeric literals are represented as u64 */
 196         u64                             constant;
 197         /* Used to optimize division by constants */
 198         u64                             div_multiplier;
 199 };
 200
 201 static u64 hist_fn_call(struct hist_field *hist_field,
 202                         struct tracing_map_elt *elt,
 203                         struct trace_buffer *buffer,
 204                         struct ring_buffer_event *rbe,
 205                         void *event);
 206
 207 static u64 hist_field_const(struct hist_field *field,
 208                            struct tracing_map_elt *elt,
 209                            struct trace_buffer *buffer,
 210                            struct ring_buffer_event *rbe,
 211                            void *event)
 212 {
 213         return field->constant;
 214 }
 215
 216 static u64 hist_field_counter(struct hist_field *field,
 217                               struct tracing_map_elt *elt,
 218                               struct trace_buffer *buffer,
 219                               struct ring_buffer_event *rbe,
 220                               void *event)
 221 {
 222         return 1;
 223 }
 224
 225 static u64 hist_field_string(struct hist_field *hist_field,
 226                              struct tracing_map_elt *elt,
 227                              struct trace_buffer *buffer,
 228                              struct ring_buffer_event *rbe,
 229                              void *event)
 230 {
 231         char *addr = (char *)(event + hist_field->field->offset);
 232
 233         return (u64)(unsigned long)addr;
 234 }
 235
 236 static u64 hist_field_dynstring(struct hist_field *hist_field,
 237                                 struct tracing_map_elt *elt,
 238                                 struct trace_buffer *buffer,
 239                                 struct ring_buffer_event *rbe,
 240                                 void *event)
 241 {
 242         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 243         int str_loc = str_item & 0xffff;
 244         char *addr = (char *)(event + str_loc);
 245
 246         return (u64)(unsigned long)addr;
 247 }
 248
 249 static u64 hist_field_reldynstring(struct hist_field *hist_field,
 250                                    struct tracing_map_elt *elt,
 251                                    struct trace_buffer *buffer,
 252                                    struct ring_buffer_event *rbe,
 253                                    void *event)
 254 {
 255         u32 *item = event + hist_field->field->offset;
 256         u32 str_item = *item;
 257         int str_loc = str_item & 0xffff;
 258         char *addr = (char *)&item[1] + str_loc;
 259
 260         return (u64)(unsigned long)addr;
 261 }
 262
 263 static u64 hist_field_pstring(struct hist_field *hist_field,
 264                               struct tracing_map_elt *elt,
 265                               struct trace_buffer *buffer,
 266                               struct ring_buffer_event *rbe,
 267                               void *event)
 268 {
 269         char **addr = (char **)(event + hist_field->field->offset);
 270
 271         return (u64)(unsigned long)*addr;
 272 }
 273
 274 static u64 hist_field_log2(struct hist_field *hist_field,
 275                            struct tracing_map_elt *elt,
 276                            struct trace_buffer *buffer,
 277                            struct ring_buffer_event *rbe,
 278                            void *event)
 279 {
 280         struct hist_field *operand = hist_field->operands[0];
 281
 282         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 283
 284         return (u64) ilog2(roundup_pow_of_two(val));
 285 }
 286
 287 static u64 hist_field_bucket(struct hist_field *hist_field,
 288                              struct tracing_map_elt *elt,
 289                              struct trace_buffer *buffer,
 290                              struct ring_buffer_event *rbe,
 291                              void *event)
 292 {
 293         struct hist_field *operand = hist_field->operands[0];
 294         unsigned long buckets = hist_field->buckets;
 295
 296         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 297
 298         if (WARN_ON_ONCE(!buckets))
 299                 return val;
 300
 301         if (val >= LONG_MAX)
 302                 val = div64_ul(val, buckets);
 303         else
 304                 val = (u64)((unsigned long)val / buckets);
 305         return val * buckets;
 306 }
 307
 308 static u64 hist_field_plus(struct hist_field *hist_field,
 309                            struct tracing_map_elt *elt,
 310                            struct trace_buffer *buffer,
 311                            struct ring_buffer_event *rbe,
 312                            void *event)
 313 {
 314         struct hist_field *operand1 = hist_field->operands[0];
 315         struct hist_field *operand2 = hist_field->operands[1];
 316
 317         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 318         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 319
 320         return val1 + val2;
 321 }
 322
 323 static u64 hist_field_minus(struct hist_field *hist_field,
 324                             struct tracing_map_elt *elt,
 325                             struct trace_buffer *buffer,
 326                             struct ring_buffer_event *rbe,
 327                             void *event)
 328 {
 329         struct hist_field *operand1 = hist_field->operands[0];
 330         struct hist_field *operand2 = hist_field->operands[1];
 331
 332         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 333         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 334
 335         return val1 - val2;
 336 }
 337
 338 static u64 hist_field_div(struct hist_field *hist_field,
 339                            struct tracing_map_elt *elt,
 340                            struct trace_buffer *buffer,
 341                            struct ring_buffer_event *rbe,
 342                            void *event)
 343 {
 344         struct hist_field *operand1 = hist_field->operands[0];
 345         struct hist_field *operand2 = hist_field->operands[1];
 346
 347         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 348         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 349
 350         /* Return -1 for the undefined case */
 351         if (!val2)
 352                 return -1;
 353
 354         /* Use shift if the divisor is a power of 2 */
 355         if (!(val2 & (val2 - 1)))
 356                 return val1 >> __ffs64(val2);
 357
 358         return div64_u64(val1, val2);
 359 }
 360
 361 static u64 div_by_power_of_two(struct hist_field *hist_field,
 362                                 struct tracing_map_elt *elt,
 363                                 struct trace_buffer *buffer,
 364                                 struct ring_buffer_event *rbe,
 365                                 void *event)
 366 {
 367         struct hist_field *operand1 = hist_field->operands[0];
 368         struct hist_field *operand2 = hist_field->operands[1];
 369
 370         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 371
 372         return val1 >> __ffs64(operand2->constant);
 373 }
 374
 375 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 376                                 struct tracing_map_elt *elt,
 377                                 struct trace_buffer *buffer,
 378                                 struct ring_buffer_event *rbe,
 379                                 void *event)
 380 {
 381         struct hist_field *operand1 = hist_field->operands[0];
 382         struct hist_field *operand2 = hist_field->operands[1];
 383
 384         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 385
 386         return div64_u64(val1, operand2->constant);
 387 }
 388
 389 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 390                                 struct tracing_map_elt *elt,
 391                                 struct trace_buffer *buffer,
 392                                 struct ring_buffer_event *rbe,
 393                                 void *event)
 394 {
 395         struct hist_field *operand1 = hist_field->operands[0];
 396         struct hist_field *operand2 = hist_field->operands[1];
 397
 398         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 399
 400         /*
 401          * If the divisor is a constant, do a multiplication and shift instead.
 402          *
 403          * Choose Z = some power of 2. If Y <= Z, then:
 404          *     X / Y = (X * (Z / Y)) / Z
 405          *
 406          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 407          *     X / Y = (X * mult) / Z
 408          *
 409          * The division by Z can be replaced by a shift since Z is a power of 2:
 410          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 411          *
 412          * As long, as X < Z the results will not be off by more than 1.
 413          */
 414         if (val1 < (1 << HIST_DIV_SHIFT)) {
 415                 u64 mult = operand2->div_multiplier;
 416
 417                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 418         }
 419
 420         return div64_u64(val1, operand2->constant);
 421 }
 422
 423 static u64 hist_field_mult(struct hist_field *hist_field,
 424                            struct tracing_map_elt *elt,
 425                            struct trace_buffer *buffer,
 426                            struct ring_buffer_event *rbe,
 427                            void *event)
 428 {
 429         struct hist_field *operand1 = hist_field->operands[0];
 430         struct hist_field *operand2 = hist_field->operands[1];
 431
 432         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 433         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 434
 435         return val1 * val2;
 436 }
 437
 438 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 439                                   struct tracing_map_elt *elt,
 440                                   struct trace_buffer *buffer,
 441                                   struct ring_buffer_event *rbe,
 442                                   void *event)
 443 {
 444         struct hist_field *operand = hist_field->operands[0];
 445
 446         s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
 447         u64 val = (u64)-sval;
 448
 449         return val;
 450 }
 451
 452 #define DEFINE_HIST_FIELD_FN(type)                                      \
 453         static u64 hist_field_##type(struct hist_field *hist_field,     \
 454                                      struct tracing_map_elt *elt,       \
 455                                      struct trace_buffer *buffer,       \
 456                                      struct ring_buffer_event *rbe,     \
 457                                      void *event)                       \
 458 {                                                                       \
 459         type *addr = (type *)(event + hist_field->field->offset);       \
 460                                                                         \
 461         return (u64)(unsigned long)*addr;                               \
 462 }
 463
 464 DEFINE_HIST_FIELD_FN(s64);
 465 DEFINE_HIST_FIELD_FN(u64);
 466 DEFINE_HIST_FIELD_FN(s32);
 467 DEFINE_HIST_FIELD_FN(u32);
 468 DEFINE_HIST_FIELD_FN(s16);
 469 DEFINE_HIST_FIELD_FN(u16);
 470 DEFINE_HIST_FIELD_FN(s8);
 471 DEFINE_HIST_FIELD_FN(u8);
 472
 473 #define for_each_hist_field(i, hist_data)       \
 474         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 475
 476 #define for_each_hist_val_field(i, hist_data)   \
 477         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 478
 479 #define for_each_hist_key_field(i, hist_data)   \
 480         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 481
 482 #define HIST_STACKTRACE_DEPTH   16
 483 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 484 #define HIST_STACKTRACE_SKIP    5
 485
 486 #define HITCOUNT_IDX            0
 487 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 488
 489 enum hist_field_flags {
 490         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 491         HIST_FIELD_FL_KEY               = 1 << 1,
 492         HIST_FIELD_FL_STRING            = 1 << 2,
 493         HIST_FIELD_FL_HEX               = 1 << 3,
 494         HIST_FIELD_FL_SYM               = 1 << 4,
 495         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 496         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 497         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 498         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 499         HIST_FIELD_FL_LOG2              = 1 << 9,
 500         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 501         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 502         HIST_FIELD_FL_VAR               = 1 << 12,
 503         HIST_FIELD_FL_EXPR              = 1 << 13,
 504         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 505         HIST_FIELD_FL_CPU               = 1 << 15,
 506         HIST_FIELD_FL_ALIAS             = 1 << 16,
 507         HIST_FIELD_FL_BUCKET            = 1 << 17,
 508         HIST_FIELD_FL_CONST             = 1 << 18,
 509         HIST_FIELD_FL_PERCENT           = 1 << 19,
 510         HIST_FIELD_FL_GRAPH             = 1 << 20,
 511 };
 512
 513 struct var_defs {
 514         unsigned int    n_vars;
 515         char            *name[TRACING_MAP_VARS_MAX];
 516         char            *expr[TRACING_MAP_VARS_MAX];
 517 };
 518
 519 struct hist_trigger_attrs {
 520         char            *keys_str;
 521         char            *vals_str;
 522         char            *sort_key_str;
 523         char            *name;
 524         char            *clock;
 525         bool            pause;
 526         bool            cont;
 527         bool            clear;
 528         bool            ts_in_usecs;
 529         unsigned int    map_bits;
 530
 531         char            *assignment_str[TRACING_MAP_VARS_MAX];
 532         unsigned int    n_assignments;
 533
 534         char            *action_str[HIST_ACTIONS_MAX];
 535         unsigned int    n_actions;
 536
 537         struct var_defs var_defs;
 538 };
 539
 540 struct field_var {
 541         struct hist_field       *var;
 542         struct hist_field       *val;
 543 };
 544
 545 struct field_var_hist {
 546         struct hist_trigger_data        *hist_data;
 547         char                            *cmd;
 548 };
 549
 550 struct hist_trigger_data {
 551         struct hist_field               *fields[HIST_FIELDS_MAX];
 552         unsigned int                    n_vals;
 553         unsigned int                    n_keys;
 554         unsigned int                    n_fields;
 555         unsigned int                    n_vars;
 556         unsigned int                    n_var_str;
 557         unsigned int                    key_size;
 558         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 559         unsigned int                    n_sort_keys;
 560         struct trace_event_file         *event_file;
 561         struct hist_trigger_attrs       *attrs;
 562         struct tracing_map              *map;
 563         bool                            enable_timestamps;
 564         bool                            remove;
 565         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 566         unsigned int                    n_var_refs;
 567
 568         struct action_data              *actions[HIST_ACTIONS_MAX];
 569         unsigned int                    n_actions;
 570
 571         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 572         unsigned int                    n_field_vars;
 573         unsigned int                    n_field_var_str;
 574         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 575         unsigned int                    n_field_var_hists;
 576
 577         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 578         unsigned int                    n_save_vars;
 579         unsigned int                    n_save_var_str;
 580 };
 581
 582 struct action_data;
 583
 584 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 585                              struct tracing_map_elt *elt,
 586                              struct trace_buffer *buffer, void *rec,
 587                              struct ring_buffer_event *rbe, void *key,
 588                              struct action_data *data, u64 *var_ref_vals);
 589
 590 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 591
 592 enum handler_id {
 593         HANDLER_ONMATCH = 1,
 594         HANDLER_ONMAX,
 595         HANDLER_ONCHANGE,
 596 };
 597
 598 enum action_id {
 599         ACTION_SAVE = 1,
 600         ACTION_TRACE,
 601         ACTION_SNAPSHOT,
 602 };
 603
 604 struct action_data {
 605         enum handler_id         handler;
 606         enum action_id          action;
 607         char                    *action_name;
 608         action_fn_t             fn;
 609
 610         unsigned int            n_params;
 611         char                    *params[SYNTH_FIELDS_MAX];
 612
 613         /*
 614          * When a histogram trigger is hit, the values of any
 615          * references to variables, including variables being passed
 616          * as parameters to synthetic events, are collected into a
 617          * var_ref_vals array.  This var_ref_idx array is an array of
 618          * indices into the var_ref_vals array, one for each synthetic
 619          * event param, and is passed to the synthetic event
 620          * invocation.
 621          */
 622         unsigned int            var_ref_idx[SYNTH_FIELDS_MAX];
 623         struct synth_event      *synth_event;
 624         bool                    use_trace_keyword;
 625         char                    *synth_event_name;
 626
 627         union {
 628                 struct {
 629                         char                    *event;
 630                         char                    *event_system;
 631                 } match_data;
 632
 633                 struct {
 634                         /*
 635                          * var_str contains the $-unstripped variable
 636                          * name referenced by var_ref, and used when
 637                          * printing the action.  Because var_ref
 638                          * creation is deferred to create_actions(),
 639                          * we need a per-action way to save it until
 640                          * then, thus var_str.
 641                          */
 642                         char                    *var_str;
 643
 644                         /*
 645                          * var_ref refers to the variable being
 646                          * tracked e.g onmax($var).
 647                          */
 648                         struct hist_field       *var_ref;
 649
 650                         /*
 651                          * track_var contains the 'invisible' tracking
 652                          * variable created to keep the current
 653                          * e.g. max value.
 654                          */
 655                         struct hist_field       *track_var;
 656
 657                         check_track_val_fn_t    check_val;
 658                         action_fn_t             save_data;
 659                 } track_data;
 660         };
 661 };
 662
 663 struct track_data {
 664         u64                             track_val;
 665         bool                            updated;
 666
 667         unsigned int                    key_len;
 668         void                            *key;
 669         struct tracing_map_elt          elt;
 670
 671         struct action_data              *action_data;
 672         struct hist_trigger_data        *hist_data;
 673 };
 674
 675 struct hist_elt_data {
 676         char *comm;
 677         u64 *var_ref_vals;
 678         char **field_var_str;
 679         int n_field_var_str;
 680 };
 681
 682 struct snapshot_context {
 683         struct tracing_map_elt  *elt;
 684         void                    *key;
 685 };
 686
 687 /*
 688  * Returns the specific division function to use if the divisor
 689  * is constant. This avoids extra branches when the trigger is hit.
 690  */
 691 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
 692 {
 693         u64 div = divisor->constant;
 694
 695         if (!(div & (div - 1)))
 696                 return HIST_FIELD_FN_DIV_POWER2;
 697
 698         /* If the divisor is too large, do a regular division */
 699         if (div > (1 << HIST_DIV_SHIFT))
 700                 return HIST_FIELD_FN_DIV_NOT_POWER2;
 701
 702         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 703         return HIST_FIELD_FN_DIV_MULT_SHIFT;
 704 }
 705
 706 static void track_data_free(struct track_data *track_data)
 707 {
 708         struct hist_elt_data *elt_data;
 709
 710         if (!track_data)
 711                 return;
 712
 713         kfree(track_data->key);
 714
 715         elt_data = track_data->elt.private_data;
 716         if (elt_data) {
 717                 kfree(elt_data->comm);
 718                 kfree(elt_data);
 719         }
 720
 721         kfree(track_data);
 722 }
 723
 724 static struct track_data *track_data_alloc(unsigned int key_len,
 725                                            struct action_data *action_data,
 726                                            struct hist_trigger_data *hist_data)
 727 {
 728         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 729         struct hist_elt_data *elt_data;
 730
 731         if (!data)
 732                 return ERR_PTR(-ENOMEM);
 733
 734         data->key = kzalloc(key_len, GFP_KERNEL);
 735         if (!data->key) {
 736                 track_data_free(data);
 737                 return ERR_PTR(-ENOMEM);
 738         }
 739
 740         data->key_len = key_len;
 741         data->action_data = action_data;
 742         data->hist_data = hist_data;
 743
 744         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 745         if (!elt_data) {
 746                 track_data_free(data);
 747                 return ERR_PTR(-ENOMEM);
 748         }
 749
 750         data->elt.private_data = elt_data;
 751
 752         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 753         if (!elt_data->comm) {
 754                 track_data_free(data);
 755                 return ERR_PTR(-ENOMEM);
 756         }
 757
 758         return data;
 759 }
 760
 761 #define HIST_PREFIX "hist:"
 762
 763 static char *last_cmd;
 764 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 765
 766 static int errpos(char *str)
 767 {
 768         if (!str || !last_cmd)
 769                 return 0;
 770
 771         return err_pos(last_cmd, str);
 772 }
 773
 774 static void last_cmd_set(struct trace_event_file *file, char *str)
 775 {
 776         const char *system = NULL, *name = NULL;
 777         struct trace_event_call *call;
 778         int len;
 779
 780         if (!str)
 781                 return;
 782
 783         /* sizeof() contains the nul byte */
 784         len = sizeof(HIST_PREFIX) + strlen(str);
 785         kfree(last_cmd);
 786         last_cmd = kzalloc(len, GFP_KERNEL);
 787         if (!last_cmd)
 788                 return;
 789
 790         strcpy(last_cmd, HIST_PREFIX);
 791         /* Again, sizeof() contains the nul byte */
 792         len -= sizeof(HIST_PREFIX);
 793         strncat(last_cmd, str, len);
 794
 795         if (file) {
 796                 call = file->event_call;
 797                 system = call->class->system;
 798                 if (system) {
 799                         name = trace_event_name(call);
 800                         if (!name)
 801                                 system = NULL;
 802                 }
 803         }
 804
 805         if (system)
 806                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
 807 }
 808
 809 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
 810 {
 811         if (!last_cmd)
 812                 return;
 813
 814         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 815                         err_type, err_pos);
 816 }
 817
 818 static void hist_err_clear(void)
 819 {
 820         if (last_cmd)
 821                 last_cmd[0] = '\0';
 822         last_cmd_loc[0] = '\0';
 823 }
 824
 825 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 826                                     unsigned int *var_ref_idx);
 827
 828 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 829                                unsigned int *var_ref_idx)
 830 {
 831         struct tracepoint *tp = event->tp;
 832
 833         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 834                 struct tracepoint_func *probe_func_ptr;
 835                 synth_probe_func_t probe_func;
 836                 void *__data;
 837
 838                 if (!(cpu_online(raw_smp_processor_id())))
 839                         return;
 840
 841                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 842                 if (probe_func_ptr) {
 843                         do {
 844                                 probe_func = probe_func_ptr->func;
 845                                 __data = probe_func_ptr->data;
 846                                 probe_func(__data, var_ref_vals, var_ref_idx);
 847                         } while ((++probe_func_ptr)->func);
 848                 }
 849         }
 850 }
 851
 852 static void action_trace(struct hist_trigger_data *hist_data,
 853                          struct tracing_map_elt *elt,
 854                          struct trace_buffer *buffer, void *rec,
 855                          struct ring_buffer_event *rbe, void *key,
 856                          struct action_data *data, u64 *var_ref_vals)
 857 {
 858         struct synth_event *event = data->synth_event;
 859
 860         trace_synth(event, var_ref_vals, data->var_ref_idx);
 861 }
 862
 863 struct hist_var_data {
 864         struct list_head list;
 865         struct hist_trigger_data *hist_data;
 866 };
 867
 868 static u64 hist_field_timestamp(struct hist_field *hist_field,
 869                                 struct tracing_map_elt *elt,
 870                                 struct trace_buffer *buffer,
 871                                 struct ring_buffer_event *rbe,
 872                                 void *event)
 873 {
 874         struct hist_trigger_data *hist_data = hist_field->hist_data;
 875         struct trace_array *tr = hist_data->event_file->tr;
 876
 877         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 878
 879         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 880                 ts = ns2usecs(ts);
 881
 882         return ts;
 883 }
 884
 885 static u64 hist_field_cpu(struct hist_field *hist_field,
 886                           struct tracing_map_elt *elt,
 887                           struct trace_buffer *buffer,
 888                           struct ring_buffer_event *rbe,
 889                           void *event)
 890 {
 891         int cpu = smp_processor_id();
 892
 893         return cpu;
 894 }
 895
 896 /**
 897  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 898  * @hist_field: The VAR_REF field to check
 899  * @var_data: The hist trigger that owns the variable
 900  * @var_idx: The trigger variable identifier
 901  *
 902  * Check the given VAR_REF field to see whether or not it references
 903  * the given variable associated with the given trigger.
 904  *
 905  * Return: The VAR_REF field if it does reference the variable, NULL if not
 906  */
 907 static struct hist_field *
 908 check_field_for_var_ref(struct hist_field *hist_field,
 909                         struct hist_trigger_data *var_data,
 910                         unsigned int var_idx)
 911 {
 912         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 913
 914         if (hist_field && hist_field->var.idx == var_idx &&
 915             hist_field->var.hist_data == var_data)
 916                 return hist_field;
 917
 918         return NULL;
 919 }
 920
 921 /**
 922  * find_var_ref - Check if a trigger has a reference to a trigger variable
 923  * @hist_data: The hist trigger that might have a reference to the variable
 924  * @var_data: The hist trigger that owns the variable
 925  * @var_idx: The trigger variable identifier
 926  *
 927  * Check the list of var_refs[] on the first hist trigger to see
 928  * whether any of them are references to the variable on the second
 929  * trigger.
 930  *
 931  * Return: The VAR_REF field referencing the variable if so, NULL if not
 932  */
 933 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 934                                        struct hist_trigger_data *var_data,
 935                                        unsigned int var_idx)
 936 {
 937         struct hist_field *hist_field;
 938         unsigned int i;
 939
 940         for (i = 0; i < hist_data->n_var_refs; i++) {
 941                 hist_field = hist_data->var_refs[i];
 942                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 943                         return hist_field;
 944         }
 945
 946         return NULL;
 947 }
 948
 949 /**
 950  * find_any_var_ref - Check if there is a reference to a given trigger variable
 951  * @hist_data: The hist trigger
 952  * @var_idx: The trigger variable identifier
 953  *
 954  * Check to see whether the given variable is currently referenced by
 955  * any other trigger.
 956  *
 957  * The trigger the variable is defined on is explicitly excluded - the
 958  * assumption being that a self-reference doesn't prevent a trigger
 959  * from being removed.
 960  *
 961  * Return: The VAR_REF field referencing the variable if so, NULL if not
 962  */
 963 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 964                                            unsigned int var_idx)
 965 {
 966         struct trace_array *tr = hist_data->event_file->tr;
 967         struct hist_field *found = NULL;
 968         struct hist_var_data *var_data;
 969
 970         list_for_each_entry(var_data, &tr->hist_vars, list) {
 971                 if (var_data->hist_data == hist_data)
 972                         continue;
 973                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 974                 if (found)
 975                         break;
 976         }
 977
 978         return found;
 979 }
 980
 981 /**
 982  * check_var_refs - Check if there is a reference to any of trigger's variables
 983  * @hist_data: The hist trigger
 984  *
 985  * A trigger can define one or more variables.  If any one of them is
 986  * currently referenced by any other trigger, this function will
 987  * determine that.
 988  *
 989  * Typically used to determine whether or not a trigger can be removed
 990  * - if there are any references to a trigger's variables, it cannot.
 991  *
 992  * Return: True if there is a reference to any of trigger's variables
 993  */
 994 static bool check_var_refs(struct hist_trigger_data *hist_data)
 995 {
 996         struct hist_field *field;
 997         bool found = false;
 998         int i;
 999
1000         for_each_hist_field(i, hist_data) {
1001                 field = hist_data->fields[i];
1002                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1003                         if (find_any_var_ref(hist_data, field->var.idx)) {
1004                                 found = true;
1005                                 break;
1006                         }
1007                 }
1008         }
1009
1010         return found;
1011 }
1012
1013 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1014 {
1015         struct trace_array *tr = hist_data->event_file->tr;
1016         struct hist_var_data *var_data, *found = NULL;
1017
1018         list_for_each_entry(var_data, &tr->hist_vars, list) {
1019                 if (var_data->hist_data == hist_data) {
1020                         found = var_data;
1021                         break;
1022                 }
1023         }
1024
1025         return found;
1026 }
1027
1028 static bool field_has_hist_vars(struct hist_field *hist_field,
1029                                 unsigned int level)
1030 {
1031         int i;
1032
1033         if (level > 3)
1034                 return false;
1035
1036         if (!hist_field)
1037                 return false;
1038
1039         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1040             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1041                 return true;
1042
1043         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1044                 struct hist_field *operand;
1045
1046                 operand = hist_field->operands[i];
1047                 if (field_has_hist_vars(operand, level + 1))
1048                         return true;
1049         }
1050
1051         return false;
1052 }
1053
1054 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1055 {
1056         struct hist_field *hist_field;
1057         int i;
1058
1059         for_each_hist_field(i, hist_data) {
1060                 hist_field = hist_data->fields[i];
1061                 if (field_has_hist_vars(hist_field, 0))
1062                         return true;
1063         }
1064
1065         return false;
1066 }
1067
1068 static int save_hist_vars(struct hist_trigger_data *hist_data)
1069 {
1070         struct trace_array *tr = hist_data->event_file->tr;
1071         struct hist_var_data *var_data;
1072
1073         var_data = find_hist_vars(hist_data);
1074         if (var_data)
1075                 return 0;
1076
1077         if (tracing_check_open_get_tr(tr))
1078                 return -ENODEV;
1079
1080         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1081         if (!var_data) {
1082                 trace_array_put(tr);
1083                 return -ENOMEM;
1084         }
1085
1086         var_data->hist_data = hist_data;
1087         list_add(&var_data->list, &tr->hist_vars);
1088
1089         return 0;
1090 }
1091
1092 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1093 {
1094         struct trace_array *tr = hist_data->event_file->tr;
1095         struct hist_var_data *var_data;
1096
1097         var_data = find_hist_vars(hist_data);
1098         if (!var_data)
1099                 return;
1100
1101         if (WARN_ON(check_var_refs(hist_data)))
1102                 return;
1103
1104         list_del(&var_data->list);
1105
1106         kfree(var_data);
1107
1108         trace_array_put(tr);
1109 }
1110
1111 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1112                                          const char *var_name)
1113 {
1114         struct hist_field *hist_field, *found = NULL;
1115         int i;
1116
1117         for_each_hist_field(i, hist_data) {
1118                 hist_field = hist_data->fields[i];
1119                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1120                     strcmp(hist_field->var.name, var_name) == 0) {
1121                         found = hist_field;
1122                         break;
1123                 }
1124         }
1125
1126         return found;
1127 }
1128
1129 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1130                                    struct trace_event_file *file,
1131                                    const char *var_name)
1132 {
1133         struct hist_trigger_data *test_data;
1134         struct event_trigger_data *test;
1135         struct hist_field *hist_field;
1136
1137         lockdep_assert_held(&event_mutex);
1138
1139         hist_field = find_var_field(hist_data, var_name);
1140         if (hist_field)
1141                 return hist_field;
1142
1143         list_for_each_entry(test, &file->triggers, list) {
1144                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1145                         test_data = test->private_data;
1146                         hist_field = find_var_field(test_data, var_name);
1147                         if (hist_field)
1148                                 return hist_field;
1149                 }
1150         }
1151
1152         return NULL;
1153 }
1154
1155 static struct trace_event_file *find_var_file(struct trace_array *tr,
1156                                               char *system,
1157                                               char *event_name,
1158                                               char *var_name)
1159 {
1160         struct hist_trigger_data *var_hist_data;
1161         struct hist_var_data *var_data;
1162         struct trace_event_file *file, *found = NULL;
1163
1164         if (system)
1165                 return find_event_file(tr, system, event_name);
1166
1167         list_for_each_entry(var_data, &tr->hist_vars, list) {
1168                 var_hist_data = var_data->hist_data;
1169                 file = var_hist_data->event_file;
1170                 if (file == found)
1171                         continue;
1172
1173                 if (find_var_field(var_hist_data, var_name)) {
1174                         if (found) {
1175                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1176                                 return NULL;
1177                         }
1178
1179                         found = file;
1180                 }
1181         }
1182
1183         return found;
1184 }
1185
1186 static struct hist_field *find_file_var(struct trace_event_file *file,
1187                                         const char *var_name)
1188 {
1189         struct hist_trigger_data *test_data;
1190         struct event_trigger_data *test;
1191         struct hist_field *hist_field;
1192
1193         lockdep_assert_held(&event_mutex);
1194
1195         list_for_each_entry(test, &file->triggers, list) {
1196                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1197                         test_data = test->private_data;
1198                         hist_field = find_var_field(test_data, var_name);
1199                         if (hist_field)
1200                                 return hist_field;
1201                 }
1202         }
1203
1204         return NULL;
1205 }
1206
1207 static struct hist_field *
1208 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1209 {
1210         struct trace_array *tr = hist_data->event_file->tr;
1211         struct hist_field *hist_field, *found = NULL;
1212         struct trace_event_file *file;
1213         unsigned int i;
1214
1215         for (i = 0; i < hist_data->n_actions; i++) {
1216                 struct action_data *data = hist_data->actions[i];
1217
1218                 if (data->handler == HANDLER_ONMATCH) {
1219                         char *system = data->match_data.event_system;
1220                         char *event_name = data->match_data.event;
1221
1222                         file = find_var_file(tr, system, event_name, var_name);
1223                         if (!file)
1224                                 continue;
1225                         hist_field = find_file_var(file, var_name);
1226                         if (hist_field) {
1227                                 if (found) {
1228                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1229                                                  errpos(var_name));
1230                                         return ERR_PTR(-EINVAL);
1231                                 }
1232
1233                                 found = hist_field;
1234                         }
1235                 }
1236         }
1237         return found;
1238 }
1239
1240 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1241                                          char *system,
1242                                          char *event_name,
1243                                          char *var_name)
1244 {
1245         struct trace_array *tr = hist_data->event_file->tr;
1246         struct hist_field *hist_field = NULL;
1247         struct trace_event_file *file;
1248
1249         if (!system || !event_name) {
1250                 hist_field = find_match_var(hist_data, var_name);
1251                 if (IS_ERR(hist_field))
1252                         return NULL;
1253                 if (hist_field)
1254                         return hist_field;
1255         }
1256
1257         file = find_var_file(tr, system, event_name, var_name);
1258         if (!file)
1259                 return NULL;
1260
1261         hist_field = find_file_var(file, var_name);
1262
1263         return hist_field;
1264 }
1265
1266 static u64 hist_field_var_ref(struct hist_field *hist_field,
1267                               struct tracing_map_elt *elt,
1268                               struct trace_buffer *buffer,
1269                               struct ring_buffer_event *rbe,
1270                               void *event)
1271 {
1272         struct hist_elt_data *elt_data;
1273         u64 var_val = 0;
1274
1275         if (WARN_ON_ONCE(!elt))
1276                 return var_val;
1277
1278         elt_data = elt->private_data;
1279         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1280
1281         return var_val;
1282 }
1283
1284 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1285                              u64 *var_ref_vals, bool self)
1286 {
1287         struct hist_trigger_data *var_data;
1288         struct tracing_map_elt *var_elt;
1289         struct hist_field *hist_field;
1290         unsigned int i, var_idx;
1291         bool resolved = true;
1292         u64 var_val = 0;
1293
1294         for (i = 0; i < hist_data->n_var_refs; i++) {
1295                 hist_field = hist_data->var_refs[i];
1296                 var_idx = hist_field->var.idx;
1297                 var_data = hist_field->var.hist_data;
1298
1299                 if (var_data == NULL) {
1300                         resolved = false;
1301                         break;
1302                 }
1303
1304                 if ((self && var_data != hist_data) ||
1305                     (!self && var_data == hist_data))
1306                         continue;
1307
1308                 var_elt = tracing_map_lookup(var_data->map, key);
1309                 if (!var_elt) {
1310                         resolved = false;
1311                         break;
1312                 }
1313
1314                 if (!tracing_map_var_set(var_elt, var_idx)) {
1315                         resolved = false;
1316                         break;
1317                 }
1318
1319                 if (self || !hist_field->read_once)
1320                         var_val = tracing_map_read_var(var_elt, var_idx);
1321                 else
1322                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1323
1324                 var_ref_vals[i] = var_val;
1325         }
1326
1327         return resolved;
1328 }
1329
1330 static const char *hist_field_name(struct hist_field *field,
1331                                    unsigned int level)
1332 {
1333         const char *field_name = "";
1334
1335         if (WARN_ON_ONCE(!field))
1336                 return field_name;
1337
1338         if (level > 1)
1339                 return field_name;
1340
1341         if (field->field)
1342                 field_name = field->field->name;
1343         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1344                  field->flags & HIST_FIELD_FL_ALIAS ||
1345                  field->flags & HIST_FIELD_FL_BUCKET)
1346                 field_name = hist_field_name(field->operands[0], ++level);
1347         else if (field->flags & HIST_FIELD_FL_CPU)
1348                 field_name = "common_cpu";
1349         else if (field->flags & HIST_FIELD_FL_EXPR ||
1350                  field->flags & HIST_FIELD_FL_VAR_REF) {
1351                 if (field->system) {
1352                         static char full_name[MAX_FILTER_STR_VAL];
1353
1354                         strcat(full_name, field->system);
1355                         strcat(full_name, ".");
1356                         strcat(full_name, field->event_name);
1357                         strcat(full_name, ".");
1358                         strcat(full_name, field->name);
1359                         field_name = full_name;
1360                 } else
1361                         field_name = field->name;
1362         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1363                 field_name = "common_timestamp";
1364
1365         if (field_name == NULL)
1366                 field_name = "";
1367
1368         return field_name;
1369 }
1370
1371 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1372 {
1373         switch (field_size) {
1374         case 8:
1375                 if (field_is_signed)
1376                         return HIST_FIELD_FN_S64;
1377                 else
1378                         return HIST_FIELD_FN_U64;
1379         case 4:
1380                 if (field_is_signed)
1381                         return HIST_FIELD_FN_S32;
1382                 else
1383                         return HIST_FIELD_FN_U32;
1384         case 2:
1385                 if (field_is_signed)
1386                         return HIST_FIELD_FN_S16;
1387                 else
1388                         return HIST_FIELD_FN_U16;
1389         case 1:
1390                 if (field_is_signed)
1391                         return HIST_FIELD_FN_S8;
1392                 else
1393                         return HIST_FIELD_FN_U8;
1394         }
1395
1396         return HIST_FIELD_FN_NOP;
1397 }
1398
1399 static int parse_map_size(char *str)
1400 {
1401         unsigned long size, map_bits;
1402         int ret;
1403
1404         ret = kstrtoul(str, 0, &size);
1405         if (ret)
1406                 goto out;
1407
1408         map_bits = ilog2(roundup_pow_of_two(size));
1409         if (map_bits < TRACING_MAP_BITS_MIN ||
1410             map_bits > TRACING_MAP_BITS_MAX)
1411                 ret = -EINVAL;
1412         else
1413                 ret = map_bits;
1414  out:
1415         return ret;
1416 }
1417
1418 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1419 {
1420         unsigned int i;
1421
1422         if (!attrs)
1423                 return;
1424
1425         for (i = 0; i < attrs->n_assignments; i++)
1426                 kfree(attrs->assignment_str[i]);
1427
1428         for (i = 0; i < attrs->n_actions; i++)
1429                 kfree(attrs->action_str[i]);
1430
1431         kfree(attrs->name);
1432         kfree(attrs->sort_key_str);
1433         kfree(attrs->keys_str);
1434         kfree(attrs->vals_str);
1435         kfree(attrs->clock);
1436         kfree(attrs);
1437 }
1438
1439 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1440 {
1441         int ret = -EINVAL;
1442
1443         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1444                 return ret;
1445
1446         if ((str_has_prefix(str, "onmatch(")) ||
1447             (str_has_prefix(str, "onmax(")) ||
1448             (str_has_prefix(str, "onchange("))) {
1449                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1450                 if (!attrs->action_str[attrs->n_actions]) {
1451                         ret = -ENOMEM;
1452                         return ret;
1453                 }
1454                 attrs->n_actions++;
1455                 ret = 0;
1456         }
1457         return ret;
1458 }
1459
1460 static int parse_assignment(struct trace_array *tr,
1461                             char *str, struct hist_trigger_attrs *attrs)
1462 {
1463         int len, ret = 0;
1464
1465         if ((len = str_has_prefix(str, "key=")) ||
1466             (len = str_has_prefix(str, "keys="))) {
1467                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1468                 if (!attrs->keys_str) {
1469                         ret = -ENOMEM;
1470                         goto out;
1471                 }
1472         } else if ((len = str_has_prefix(str, "val=")) ||
1473                    (len = str_has_prefix(str, "vals=")) ||
1474                    (len = str_has_prefix(str, "values="))) {
1475                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1476                 if (!attrs->vals_str) {
1477                         ret = -ENOMEM;
1478                         goto out;
1479                 }
1480         } else if ((len = str_has_prefix(str, "sort="))) {
1481                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1482                 if (!attrs->sort_key_str) {
1483                         ret = -ENOMEM;
1484                         goto out;
1485                 }
1486         } else if (str_has_prefix(str, "name=")) {
1487                 attrs->name = kstrdup(str, GFP_KERNEL);
1488                 if (!attrs->name) {
1489                         ret = -ENOMEM;
1490                         goto out;
1491                 }
1492         } else if ((len = str_has_prefix(str, "clock="))) {
1493                 str += len;
1494
1495                 str = strstrip(str);
1496                 attrs->clock = kstrdup(str, GFP_KERNEL);
1497                 if (!attrs->clock) {
1498                         ret = -ENOMEM;
1499                         goto out;
1500                 }
1501         } else if ((len = str_has_prefix(str, "size="))) {
1502                 int map_bits = parse_map_size(str + len);
1503
1504                 if (map_bits < 0) {
1505                         ret = map_bits;
1506                         goto out;
1507                 }
1508                 attrs->map_bits = map_bits;
1509         } else {
1510                 char *assignment;
1511
1512                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1513                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1514                         ret = -EINVAL;
1515                         goto out;
1516                 }
1517
1518                 assignment = kstrdup(str, GFP_KERNEL);
1519                 if (!assignment) {
1520                         ret = -ENOMEM;
1521                         goto out;
1522                 }
1523
1524                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1525         }
1526  out:
1527         return ret;
1528 }
1529
1530 static struct hist_trigger_attrs *
1531 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1532 {
1533         struct hist_trigger_attrs *attrs;
1534         int ret = 0;
1535
1536         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1537         if (!attrs)
1538                 return ERR_PTR(-ENOMEM);
1539
1540         while (trigger_str) {
1541                 char *str = strsep(&trigger_str, ":");
1542                 char *rhs;
1543
1544                 rhs = strchr(str, '=');
1545                 if (rhs) {
1546                         if (!strlen(++rhs)) {
1547                                 ret = -EINVAL;
1548                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1549                                 goto free;
1550                         }
1551                         ret = parse_assignment(tr, str, attrs);
1552                         if (ret)
1553                                 goto free;
1554                 } else if (strcmp(str, "pause") == 0)
1555                         attrs->pause = true;
1556                 else if ((strcmp(str, "cont") == 0) ||
1557                          (strcmp(str, "continue") == 0))
1558                         attrs->cont = true;
1559                 else if (strcmp(str, "clear") == 0)
1560                         attrs->clear = true;
1561                 else {
1562                         ret = parse_action(str, attrs);
1563                         if (ret)
1564                                 goto free;
1565                 }
1566         }
1567
1568         if (!attrs->keys_str) {
1569                 ret = -EINVAL;
1570                 goto free;
1571         }
1572
1573         if (!attrs->clock) {
1574                 attrs->clock = kstrdup("global", GFP_KERNEL);
1575                 if (!attrs->clock) {
1576                         ret = -ENOMEM;
1577                         goto free;
1578                 }
1579         }
1580
1581         return attrs;
1582  free:
1583         destroy_hist_trigger_attrs(attrs);
1584
1585         return ERR_PTR(ret);
1586 }
1587
1588 static inline void save_comm(char *comm, struct task_struct *task)
1589 {
1590         if (!task->pid) {
1591                 strcpy(comm, "<idle>");
1592                 return;
1593         }
1594
1595         if (WARN_ON_ONCE(task->pid < 0)) {
1596                 strcpy(comm, "<XXX>");
1597                 return;
1598         }
1599
1600         strncpy(comm, task->comm, TASK_COMM_LEN);
1601 }
1602
1603 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1604 {
1605         unsigned int i;
1606
1607         for (i = 0; i < elt_data->n_field_var_str; i++)
1608                 kfree(elt_data->field_var_str[i]);
1609
1610         kfree(elt_data->field_var_str);
1611
1612         kfree(elt_data->comm);
1613         kfree(elt_data);
1614 }
1615
1616 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1617 {
1618         struct hist_elt_data *elt_data = elt->private_data;
1619
1620         hist_elt_data_free(elt_data);
1621 }
1622
1623 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1624 {
1625         struct hist_trigger_data *hist_data = elt->map->private_data;
1626         unsigned int size = TASK_COMM_LEN;
1627         struct hist_elt_data *elt_data;
1628         struct hist_field *hist_field;
1629         unsigned int i, n_str;
1630
1631         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1632         if (!elt_data)
1633                 return -ENOMEM;
1634
1635         for_each_hist_field(i, hist_data) {
1636                 hist_field = hist_data->fields[i];
1637
1638                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1639                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1640                         if (!elt_data->comm) {
1641                                 kfree(elt_data);
1642                                 return -ENOMEM;
1643                         }
1644                         break;
1645                 }
1646         }
1647
1648         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1649                 hist_data->n_var_str;
1650         if (n_str > SYNTH_FIELDS_MAX) {
1651                 hist_elt_data_free(elt_data);
1652                 return -EINVAL;
1653         }
1654
1655         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1656
1657         size = STR_VAR_LEN_MAX;
1658
1659         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1660         if (!elt_data->field_var_str) {
1661                 hist_elt_data_free(elt_data);
1662                 return -EINVAL;
1663         }
1664         elt_data->n_field_var_str = n_str;
1665
1666         for (i = 0; i < n_str; i++) {
1667                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1668                 if (!elt_data->field_var_str[i]) {
1669                         hist_elt_data_free(elt_data);
1670                         return -ENOMEM;
1671                 }
1672         }
1673
1674         elt->private_data = elt_data;
1675
1676         return 0;
1677 }
1678
1679 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1680 {
1681         struct hist_elt_data *elt_data = elt->private_data;
1682
1683         if (elt_data->comm)
1684                 save_comm(elt_data->comm, current);
1685 }
1686
1687 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1688         .elt_alloc      = hist_trigger_elt_data_alloc,
1689         .elt_free       = hist_trigger_elt_data_free,
1690         .elt_init       = hist_trigger_elt_data_init,
1691 };
1692
1693 static const char *get_hist_field_flags(struct hist_field *hist_field)
1694 {
1695         const char *flags_str = NULL;
1696
1697         if (hist_field->flags & HIST_FIELD_FL_HEX)
1698                 flags_str = "hex";
1699         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1700                 flags_str = "sym";
1701         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1702                 flags_str = "sym-offset";
1703         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1704                 flags_str = "execname";
1705         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1706                 flags_str = "syscall";
1707         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1708                 flags_str = "log2";
1709         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1710                 flags_str = "buckets";
1711         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1712                 flags_str = "usecs";
1713         else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1714                 flags_str = "percent";
1715         else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1716                 flags_str = "graph";
1717
1718         return flags_str;
1719 }
1720
1721 static void expr_field_str(struct hist_field *field, char *expr)
1722 {
1723         if (field->flags & HIST_FIELD_FL_VAR_REF)
1724                 strcat(expr, "$");
1725         else if (field->flags & HIST_FIELD_FL_CONST) {
1726                 char str[HIST_CONST_DIGITS_MAX];
1727
1728                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1729                 strcat(expr, str);
1730         }
1731
1732         strcat(expr, hist_field_name(field, 0));
1733
1734         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1735                 const char *flags_str = get_hist_field_flags(field);
1736
1737                 if (flags_str) {
1738                         strcat(expr, ".");
1739                         strcat(expr, flags_str);
1740                 }
1741         }
1742 }
1743
1744 static char *expr_str(struct hist_field *field, unsigned int level)
1745 {
1746         char *expr;
1747
1748         if (level > 1)
1749                 return NULL;
1750
1751         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1752         if (!expr)
1753                 return NULL;
1754
1755         if (!field->operands[0]) {
1756                 expr_field_str(field, expr);
1757                 return expr;
1758         }
1759
1760         if (field->operator == FIELD_OP_UNARY_MINUS) {
1761                 char *subexpr;
1762
1763                 strcat(expr, "-(");
1764                 subexpr = expr_str(field->operands[0], ++level);
1765                 if (!subexpr) {
1766                         kfree(expr);
1767                         return NULL;
1768                 }
1769                 strcat(expr, subexpr);
1770                 strcat(expr, ")");
1771
1772                 kfree(subexpr);
1773
1774                 return expr;
1775         }
1776
1777         expr_field_str(field->operands[0], expr);
1778
1779         switch (field->operator) {
1780         case FIELD_OP_MINUS:
1781                 strcat(expr, "-");
1782                 break;
1783         case FIELD_OP_PLUS:
1784                 strcat(expr, "+");
1785                 break;
1786         case FIELD_OP_DIV:
1787                 strcat(expr, "/");
1788                 break;
1789         case FIELD_OP_MULT:
1790                 strcat(expr, "*");
1791                 break;
1792         default:
1793                 kfree(expr);
1794                 return NULL;
1795         }
1796
1797         expr_field_str(field->operands[1], expr);
1798
1799         return expr;
1800 }
1801
1802 /*
1803  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1804  * of the expression tree to be evaluated.
1805  */
1806 static int contains_operator(char *str, char **sep)
1807 {
1808         enum field_op_id field_op = FIELD_OP_NONE;
1809         char *minus_op, *plus_op, *div_op, *mult_op;
1810
1811
1812         /*
1813          * Report the last occurrence of the operators first, so that the
1814          * expression is evaluated left to right. This is important since
1815          * subtraction and division are not associative.
1816          *
1817          *      e.g
1818          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1819          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1820          */
1821
1822         /*
1823          * First, find lower precedence addition and subtraction
1824          * since the expression will be evaluated recursively.
1825          */
1826         minus_op = strrchr(str, '-');
1827         if (minus_op) {
1828                 /*
1829                  * Unary minus is not supported in sub-expressions. If
1830                  * present, it is always the next root operator.
1831                  */
1832                 if (minus_op == str) {
1833                         field_op = FIELD_OP_UNARY_MINUS;
1834                         goto out;
1835                 }
1836
1837                 field_op = FIELD_OP_MINUS;
1838         }
1839
1840         plus_op = strrchr(str, '+');
1841         if (plus_op || minus_op) {
1842                 /*
1843                  * For operators of the same precedence use to rightmost as the
1844                  * root, so that the expression is evaluated left to right.
1845                  */
1846                 if (plus_op > minus_op)
1847                         field_op = FIELD_OP_PLUS;
1848                 goto out;
1849         }
1850
1851         /*
1852          * Multiplication and division have higher precedence than addition and
1853          * subtraction.
1854          */
1855         div_op = strrchr(str, '/');
1856         if (div_op)
1857                 field_op = FIELD_OP_DIV;
1858
1859         mult_op = strrchr(str, '*');
1860         /*
1861          * For operators of the same precedence use to rightmost as the
1862          * root, so that the expression is evaluated left to right.
1863          */
1864         if (mult_op > div_op)
1865                 field_op = FIELD_OP_MULT;
1866
1867 out:
1868         if (sep) {
1869                 switch (field_op) {
1870                 case FIELD_OP_UNARY_MINUS:
1871                 case FIELD_OP_MINUS:
1872                         *sep = minus_op;
1873                         break;
1874                 case FIELD_OP_PLUS:
1875                         *sep = plus_op;
1876                         break;
1877                 case FIELD_OP_DIV:
1878                         *sep = div_op;
1879                         break;
1880                 case FIELD_OP_MULT:
1881                         *sep = mult_op;
1882                         break;
1883                 case FIELD_OP_NONE:
1884                 default:
1885                         *sep = NULL;
1886                         break;
1887                 }
1888         }
1889
1890         return field_op;
1891 }
1892
1893 static void get_hist_field(struct hist_field *hist_field)
1894 {
1895         hist_field->ref++;
1896 }
1897
1898 static void __destroy_hist_field(struct hist_field *hist_field)
1899 {
1900         if (--hist_field->ref > 1)
1901                 return;
1902
1903         kfree(hist_field->var.name);
1904         kfree(hist_field->name);
1905
1906         /* Can likely be a const */
1907         kfree_const(hist_field->type);
1908
1909         kfree(hist_field->system);
1910         kfree(hist_field->event_name);
1911
1912         kfree(hist_field);
1913 }
1914
1915 static void destroy_hist_field(struct hist_field *hist_field,
1916                                unsigned int level)
1917 {
1918         unsigned int i;
1919
1920         if (level > 3)
1921                 return;
1922
1923         if (!hist_field)
1924                 return;
1925
1926         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1927                 return; /* var refs will be destroyed separately */
1928
1929         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1930                 destroy_hist_field(hist_field->operands[i], level + 1);
1931
1932         __destroy_hist_field(hist_field);
1933 }
1934
1935 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1936                                             struct ftrace_event_field *field,
1937                                             unsigned long flags,
1938                                             char *var_name)
1939 {
1940         struct hist_field *hist_field;
1941
1942         if (field && is_function_field(field))
1943                 return NULL;
1944
1945         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1946         if (!hist_field)
1947                 return NULL;
1948
1949         hist_field->ref = 1;
1950
1951         hist_field->hist_data = hist_data;
1952
1953         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1954                 goto out; /* caller will populate */
1955
1956         if (flags & HIST_FIELD_FL_VAR_REF) {
1957                 hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1958                 goto out;
1959         }
1960
1961         if (flags & HIST_FIELD_FL_HITCOUNT) {
1962                 hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1963                 hist_field->size = sizeof(u64);
1964                 hist_field->type = "u64";
1965                 goto out;
1966         }
1967
1968         if (flags & HIST_FIELD_FL_CONST) {
1969                 hist_field->fn_num = HIST_FIELD_FN_CONST;
1970                 hist_field->size = sizeof(u64);
1971                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1972                 if (!hist_field->type)
1973                         goto free;
1974                 goto out;
1975         }
1976
1977         if (flags & HIST_FIELD_FL_STACKTRACE) {
1978                 hist_field->fn_num = HIST_FIELD_FN_NOP;
1979                 goto out;
1980         }
1981
1982         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1983                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1984                 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
1985                         HIST_FIELD_FN_BUCKET;
1986                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1987                 if (!hist_field->operands[0])
1988                         goto free;
1989                 hist_field->size = hist_field->operands[0]->size;
1990                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1991                 if (!hist_field->type)
1992                         goto free;
1993                 goto out;
1994         }
1995
1996         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1997                 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
1998                 hist_field->size = sizeof(u64);
1999                 hist_field->type = "u64";
2000                 goto out;
2001         }
2002
2003         if (flags & HIST_FIELD_FL_CPU) {
2004                 hist_field->fn_num = HIST_FIELD_FN_CPU;
2005                 hist_field->size = sizeof(int);
2006                 hist_field->type = "unsigned int";
2007                 goto out;
2008         }
2009
2010         if (WARN_ON_ONCE(!field))
2011                 goto out;
2012
2013         /* Pointers to strings are just pointers and dangerous to dereference */
2014         if (is_string_field(field) &&
2015             (field->filter_type != FILTER_PTR_STRING)) {
2016                 flags |= HIST_FIELD_FL_STRING;
2017
2018                 hist_field->size = MAX_FILTER_STR_VAL;
2019                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2020                 if (!hist_field->type)
2021                         goto free;
2022
2023                 if (field->filter_type == FILTER_STATIC_STRING) {
2024                         hist_field->fn_num = HIST_FIELD_FN_STRING;
2025                         hist_field->size = field->size;
2026                 } else if (field->filter_type == FILTER_DYN_STRING) {
2027                         hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2028                 } else if (field->filter_type == FILTER_RDYN_STRING)
2029                         hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2030                 else
2031                         hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2032         } else {
2033                 hist_field->size = field->size;
2034                 hist_field->is_signed = field->is_signed;
2035                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2036                 if (!hist_field->type)
2037                         goto free;
2038
2039                 hist_field->fn_num = select_value_fn(field->size,
2040                                                      field->is_signed);
2041                 if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2042                         destroy_hist_field(hist_field, 0);
2043                         return NULL;
2044                 }
2045         }
2046  out:
2047         hist_field->field = field;
2048         hist_field->flags = flags;
2049
2050         if (var_name) {
2051                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2052                 if (!hist_field->var.name)
2053                         goto free;
2054         }
2055
2056         return hist_field;
2057  free:
2058         destroy_hist_field(hist_field, 0);
2059         return NULL;
2060 }
2061
2062 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2063 {
2064         unsigned int i;
2065
2066         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2067                 if (hist_data->fields[i]) {
2068                         destroy_hist_field(hist_data->fields[i], 0);
2069                         hist_data->fields[i] = NULL;
2070                 }
2071         }
2072
2073         for (i = 0; i < hist_data->n_var_refs; i++) {
2074                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2075                 __destroy_hist_field(hist_data->var_refs[i]);
2076                 hist_data->var_refs[i] = NULL;
2077         }
2078 }
2079
2080 static int init_var_ref(struct hist_field *ref_field,
2081                         struct hist_field *var_field,
2082                         char *system, char *event_name)
2083 {
2084         int err = 0;
2085
2086         ref_field->var.idx = var_field->var.idx;
2087         ref_field->var.hist_data = var_field->hist_data;
2088         ref_field->size = var_field->size;
2089         ref_field->is_signed = var_field->is_signed;
2090         ref_field->flags |= var_field->flags &
2091                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2092
2093         if (system) {
2094                 ref_field->system = kstrdup(system, GFP_KERNEL);
2095                 if (!ref_field->system)
2096                         return -ENOMEM;
2097         }
2098
2099         if (event_name) {
2100                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2101                 if (!ref_field->event_name) {
2102                         err = -ENOMEM;
2103                         goto free;
2104                 }
2105         }
2106
2107         if (var_field->var.name) {
2108                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2109                 if (!ref_field->name) {
2110                         err = -ENOMEM;
2111                         goto free;
2112                 }
2113         } else if (var_field->name) {
2114                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2115                 if (!ref_field->name) {
2116                         err = -ENOMEM;
2117                         goto free;
2118                 }
2119         }
2120
2121         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2122         if (!ref_field->type) {
2123                 err = -ENOMEM;
2124                 goto free;
2125         }
2126  out:
2127         return err;
2128  free:
2129         kfree(ref_field->system);
2130         ref_field->system = NULL;
2131         kfree(ref_field->event_name);
2132         ref_field->event_name = NULL;
2133         kfree(ref_field->name);
2134         ref_field->name = NULL;
2135
2136         goto out;
2137 }
2138
2139 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2140                             struct hist_field *var_field)
2141 {
2142         struct hist_field *ref_field;
2143         int i;
2144
2145         for (i = 0; i < hist_data->n_var_refs; i++) {
2146                 ref_field = hist_data->var_refs[i];
2147                 if (ref_field->var.idx == var_field->var.idx &&
2148                     ref_field->var.hist_data == var_field->hist_data)
2149                         return i;
2150         }
2151
2152         return -ENOENT;
2153 }
2154
2155 /**
2156  * create_var_ref - Create a variable reference and attach it to trigger
2157  * @hist_data: The trigger that will be referencing the variable
2158  * @var_field: The VAR field to create a reference to
2159  * @system: The optional system string
2160  * @event_name: The optional event_name string
2161  *
2162  * Given a variable hist_field, create a VAR_REF hist_field that
2163  * represents a reference to it.
2164  *
2165  * This function also adds the reference to the trigger that
2166  * now references the variable.
2167  *
2168  * Return: The VAR_REF field if successful, NULL if not
2169  */
2170 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2171                                          struct hist_field *var_field,
2172                                          char *system, char *event_name)
2173 {
2174         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2175         struct hist_field *ref_field;
2176         int i;
2177
2178         /* Check if the variable already exists */
2179         for (i = 0; i < hist_data->n_var_refs; i++) {
2180                 ref_field = hist_data->var_refs[i];
2181                 if (ref_field->var.idx == var_field->var.idx &&
2182                     ref_field->var.hist_data == var_field->hist_data) {
2183                         get_hist_field(ref_field);
2184                         return ref_field;
2185                 }
2186         }
2187         /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2188         if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2189                 return NULL;
2190         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2191         if (ref_field) {
2192                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2193                         destroy_hist_field(ref_field, 0);
2194                         return NULL;
2195                 }
2196
2197                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2198                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2199         }
2200
2201         return ref_field;
2202 }
2203
2204 static bool is_var_ref(char *var_name)
2205 {
2206         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2207                 return false;
2208
2209         return true;
2210 }
2211
2212 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2213                                  char *var_name)
2214 {
2215         char *name, *field;
2216         unsigned int i;
2217
2218         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2219                 name = hist_data->attrs->var_defs.name[i];
2220
2221                 if (strcmp(var_name, name) == 0) {
2222                         field = hist_data->attrs->var_defs.expr[i];
2223                         if (contains_operator(field, NULL) || is_var_ref(field))
2224                                 continue;
2225                         return field;
2226                 }
2227         }
2228
2229         return NULL;
2230 }
2231
2232 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2233                                  char *system, char *event_name,
2234                                  char *var_name)
2235 {
2236         struct trace_event_call *call;
2237
2238         if (system && event_name) {
2239                 call = hist_data->event_file->event_call;
2240
2241                 if (strcmp(system, call->class->system) != 0)
2242                         return NULL;
2243
2244                 if (strcmp(event_name, trace_event_name(call)) != 0)
2245                         return NULL;
2246         }
2247
2248         if (!!system != !!event_name)
2249                 return NULL;
2250
2251         if (!is_var_ref(var_name))
2252                 return NULL;
2253
2254         var_name++;
2255
2256         return field_name_from_var(hist_data, var_name);
2257 }
2258
2259 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2260                                         char *system, char *event_name,
2261                                         char *var_name)
2262 {
2263         struct hist_field *var_field = NULL, *ref_field = NULL;
2264         struct trace_array *tr = hist_data->event_file->tr;
2265
2266         if (!is_var_ref(var_name))
2267                 return NULL;
2268
2269         var_name++;
2270
2271         var_field = find_event_var(hist_data, system, event_name, var_name);
2272         if (var_field)
2273                 ref_field = create_var_ref(hist_data, var_field,
2274                                            system, event_name);
2275
2276         if (!ref_field)
2277                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2278
2279         return ref_field;
2280 }
2281
2282 static struct ftrace_event_field *
2283 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2284             char *field_str, unsigned long *flags, unsigned long *buckets)
2285 {
2286         struct ftrace_event_field *field = NULL;
2287         char *field_name, *modifier, *str;
2288         struct trace_array *tr = file->tr;
2289
2290         modifier = str = kstrdup(field_str, GFP_KERNEL);
2291         if (!modifier)
2292                 return ERR_PTR(-ENOMEM);
2293
2294         field_name = strsep(&modifier, ".");
2295         if (modifier) {
2296                 if (strcmp(modifier, "hex") == 0)
2297                         *flags |= HIST_FIELD_FL_HEX;
2298                 else if (strcmp(modifier, "sym") == 0)
2299                         *flags |= HIST_FIELD_FL_SYM;
2300                 /*
2301                  * 'sym-offset' occurrences in the trigger string are modified
2302                  * to 'symXoffset' to simplify arithmetic expression parsing.
2303                  */
2304                 else if (strcmp(modifier, "symXoffset") == 0)
2305                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2306                 else if ((strcmp(modifier, "execname") == 0) &&
2307                          (strcmp(field_name, "common_pid") == 0))
2308                         *flags |= HIST_FIELD_FL_EXECNAME;
2309                 else if (strcmp(modifier, "syscall") == 0)
2310                         *flags |= HIST_FIELD_FL_SYSCALL;
2311                 else if (strcmp(modifier, "log2") == 0)
2312                         *flags |= HIST_FIELD_FL_LOG2;
2313                 else if (strcmp(modifier, "usecs") == 0)
2314                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2315                 else if (strncmp(modifier, "bucket", 6) == 0) {
2316                         int ret;
2317
2318                         modifier += 6;
2319
2320                         if (*modifier == 's')
2321                                 modifier++;
2322                         if (*modifier != '=')
2323                                 goto error;
2324                         modifier++;
2325                         ret = kstrtoul(modifier, 0, buckets);
2326                         if (ret || !(*buckets))
2327                                 goto error;
2328                         *flags |= HIST_FIELD_FL_BUCKET;
2329                 } else if (strncmp(modifier, "percent", 7) == 0) {
2330                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2331                                 goto error;
2332                         *flags |= HIST_FIELD_FL_PERCENT;
2333                 } else if (strncmp(modifier, "graph", 5) == 0) {
2334                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2335                                 goto error;
2336                         *flags |= HIST_FIELD_FL_GRAPH;
2337                 } else {
2338  error:
2339                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2340                         field = ERR_PTR(-EINVAL);
2341                         goto out;
2342                 }
2343         }
2344
2345         if (strcmp(field_name, "common_timestamp") == 0) {
2346                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2347                 hist_data->enable_timestamps = true;
2348                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2349                         hist_data->attrs->ts_in_usecs = true;
2350         } else if (strcmp(field_name, "common_cpu") == 0)
2351                 *flags |= HIST_FIELD_FL_CPU;
2352         else {
2353                 field = trace_find_event_field(file->event_call, field_name);
2354                 if (!field || !field->size) {
2355                         /*
2356                          * For backward compatibility, if field_name
2357                          * was "cpu", then we treat this the same as
2358                          * common_cpu. This also works for "CPU".
2359                          */
2360                         if (field && field->filter_type == FILTER_CPU) {
2361                                 *flags |= HIST_FIELD_FL_CPU;
2362                         } else {
2363                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2364                                          errpos(field_name));
2365                                 field = ERR_PTR(-EINVAL);
2366                                 goto out;
2367                         }
2368                 }
2369         }
2370  out:
2371         kfree(str);
2372
2373         return field;
2374 }
2375
2376 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2377                                        struct hist_field *var_ref,
2378                                        char *var_name)
2379 {
2380         struct hist_field *alias = NULL;
2381         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2382
2383         alias = create_hist_field(hist_data, NULL, flags, var_name);
2384         if (!alias)
2385                 return NULL;
2386
2387         alias->fn_num = var_ref->fn_num;
2388         alias->operands[0] = var_ref;
2389
2390         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2391                 destroy_hist_field(alias, 0);
2392                 return NULL;
2393         }
2394
2395         alias->var_ref_idx = var_ref->var_ref_idx;
2396
2397         return alias;
2398 }
2399
2400 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2401                                       char *str, char *var_name,
2402                                       unsigned long *flags)
2403 {
2404         struct trace_array *tr = hist_data->event_file->tr;
2405         struct hist_field *field = NULL;
2406         u64 constant;
2407
2408         if (kstrtoull(str, 0, &constant)) {
2409                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2410                 return NULL;
2411         }
2412
2413         *flags |= HIST_FIELD_FL_CONST;
2414         field = create_hist_field(hist_data, NULL, *flags, var_name);
2415         if (!field)
2416                 return NULL;
2417
2418         field->constant = constant;
2419
2420         return field;
2421 }
2422
2423 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2424                                      struct trace_event_file *file, char *str,
2425                                      unsigned long *flags, char *var_name)
2426 {
2427         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2428         struct ftrace_event_field *field = NULL;
2429         struct hist_field *hist_field = NULL;
2430         unsigned long buckets = 0;
2431         int ret = 0;
2432
2433         if (isdigit(str[0])) {
2434                 hist_field = parse_const(hist_data, str, var_name, flags);
2435                 if (!hist_field) {
2436                         ret = -EINVAL;
2437                         goto out;
2438                 }
2439                 return hist_field;
2440         }
2441
2442         s = strchr(str, '.');
2443         if (s) {
2444                 s = strchr(++s, '.');
2445                 if (s) {
2446                         ref_system = strsep(&str, ".");
2447                         if (!str) {
2448                                 ret = -EINVAL;
2449                                 goto out;
2450                         }
2451                         ref_event = strsep(&str, ".");
2452                         if (!str) {
2453                                 ret = -EINVAL;
2454                                 goto out;
2455                         }
2456                         ref_var = str;
2457                 }
2458         }
2459
2460         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2461         if (!s) {
2462                 hist_field = parse_var_ref(hist_data, ref_system,
2463                                            ref_event, ref_var);
2464                 if (hist_field) {
2465                         if (var_name) {
2466                                 hist_field = create_alias(hist_data, hist_field, var_name);
2467                                 if (!hist_field) {
2468                                         ret = -ENOMEM;
2469                                         goto out;
2470                                 }
2471                         }
2472                         return hist_field;
2473                 }
2474         } else
2475                 str = s;
2476
2477         field = parse_field(hist_data, file, str, flags, &buckets);
2478         if (IS_ERR(field)) {
2479                 ret = PTR_ERR(field);
2480                 goto out;
2481         }
2482
2483         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2484         if (!hist_field) {
2485                 ret = -ENOMEM;
2486                 goto out;
2487         }
2488         hist_field->buckets = buckets;
2489
2490         return hist_field;
2491  out:
2492         return ERR_PTR(ret);
2493 }
2494
2495 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2496                                      struct trace_event_file *file,
2497                                      char *str, unsigned long flags,
2498                                      char *var_name, unsigned int *n_subexprs);
2499
2500 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2501                                       struct trace_event_file *file,
2502                                       char *str, unsigned long flags,
2503                                       char *var_name, unsigned int *n_subexprs)
2504 {
2505         struct hist_field *operand1, *expr = NULL;
2506         unsigned long operand_flags;
2507         int ret = 0;
2508         char *s;
2509
2510         /* Unary minus operator, increment n_subexprs */
2511         ++*n_subexprs;
2512
2513         /* we support only -(xxx) i.e. explicit parens required */
2514
2515         if (*n_subexprs > 3) {
2516                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2517                 ret = -EINVAL;
2518                 goto free;
2519         }
2520
2521         str++; /* skip leading '-' */
2522
2523         s = strchr(str, '(');
2524         if (s)
2525                 str++;
2526         else {
2527                 ret = -EINVAL;
2528                 goto free;
2529         }
2530
2531         s = strrchr(str, ')');
2532         if (s) {
2533                  /* unary minus not supported in sub-expressions */
2534                 if (*(s+1) != '\0') {
2535                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2536                                  errpos(str));
2537                         ret = -EINVAL;
2538                         goto free;
2539                 }
2540                 *s = '\0';
2541         }
2542         else {
2543                 ret = -EINVAL; /* no closing ')' */
2544                 goto free;
2545         }
2546
2547         flags |= HIST_FIELD_FL_EXPR;
2548         expr = create_hist_field(hist_data, NULL, flags, var_name);
2549         if (!expr) {
2550                 ret = -ENOMEM;
2551                 goto free;
2552         }
2553
2554         operand_flags = 0;
2555         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2556         if (IS_ERR(operand1)) {
2557                 ret = PTR_ERR(operand1);
2558                 goto free;
2559         }
2560         if (operand1->flags & HIST_FIELD_FL_STRING) {
2561                 /* String type can not be the operand of unary operator. */
2562                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2563                 destroy_hist_field(operand1, 0);
2564                 ret = -EINVAL;
2565                 goto free;
2566         }
2567
2568         expr->flags |= operand1->flags &
2569                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2570         expr->fn_num = HIST_FIELD_FN_UMINUS;
2571         expr->operands[0] = operand1;
2572         expr->size = operand1->size;
2573         expr->is_signed = operand1->is_signed;
2574         expr->operator = FIELD_OP_UNARY_MINUS;
2575         expr->name = expr_str(expr, 0);
2576         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2577         if (!expr->type) {
2578                 ret = -ENOMEM;
2579                 goto free;
2580         }
2581
2582         return expr;
2583  free:
2584         destroy_hist_field(expr, 0);
2585         return ERR_PTR(ret);
2586 }
2587
2588 /*
2589  * If the operands are var refs, return pointers the
2590  * variable(s) referenced in var1 and var2, else NULL.
2591  */
2592 static int check_expr_operands(struct trace_array *tr,
2593                                struct hist_field *operand1,
2594                                struct hist_field *operand2,
2595                                struct hist_field **var1,
2596                                struct hist_field **var2)
2597 {
2598         unsigned long operand1_flags = operand1->flags;
2599         unsigned long operand2_flags = operand2->flags;
2600
2601         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2602             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2603                 struct hist_field *var;
2604
2605                 var = find_var_field(operand1->var.hist_data, operand1->name);
2606                 if (!var)
2607                         return -EINVAL;
2608                 operand1_flags = var->flags;
2609                 *var1 = var;
2610         }
2611
2612         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2613             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2614                 struct hist_field *var;
2615
2616                 var = find_var_field(operand2->var.hist_data, operand2->name);
2617                 if (!var)
2618                         return -EINVAL;
2619                 operand2_flags = var->flags;
2620                 *var2 = var;
2621         }
2622
2623         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2624             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2625                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2626                 return -EINVAL;
2627         }
2628
2629         return 0;
2630 }
2631
2632 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2633                                      struct trace_event_file *file,
2634                                      char *str, unsigned long flags,
2635                                      char *var_name, unsigned int *n_subexprs)
2636 {
2637         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2638         struct hist_field *var1 = NULL, *var2 = NULL;
2639         unsigned long operand_flags, operand2_flags;
2640         int field_op, ret = -EINVAL;
2641         char *sep, *operand1_str;
2642         enum hist_field_fn op_fn;
2643         bool combine_consts;
2644
2645         if (*n_subexprs > 3) {
2646                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2647                 return ERR_PTR(-EINVAL);
2648         }
2649
2650         field_op = contains_operator(str, &sep);
2651
2652         if (field_op == FIELD_OP_NONE)
2653                 return parse_atom(hist_data, file, str, &flags, var_name);
2654
2655         if (field_op == FIELD_OP_UNARY_MINUS)
2656                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2657
2658         /* Binary operator found, increment n_subexprs */
2659         ++*n_subexprs;
2660
2661         /* Split the expression string at the root operator */
2662         if (!sep)
2663                 return ERR_PTR(-EINVAL);
2664
2665         *sep = '\0';
2666         operand1_str = str;
2667         str = sep+1;
2668
2669         /* Binary operator requires both operands */
2670         if (*operand1_str == '\0' || *str == '\0')
2671                 return ERR_PTR(-EINVAL);
2672
2673         operand_flags = 0;
2674
2675         /* LHS of string is an expression e.g. a+b in a+b+c */
2676         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2677         if (IS_ERR(operand1))
2678                 return ERR_CAST(operand1);
2679
2680         if (operand1->flags & HIST_FIELD_FL_STRING) {
2681                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2682                 ret = -EINVAL;
2683                 goto free_op1;
2684         }
2685
2686         /* RHS of string is another expression e.g. c in a+b+c */
2687         operand_flags = 0;
2688         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2689         if (IS_ERR(operand2)) {
2690                 ret = PTR_ERR(operand2);
2691                 goto free_op1;
2692         }
2693         if (operand2->flags & HIST_FIELD_FL_STRING) {
2694                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2695                 ret = -EINVAL;
2696                 goto free_operands;
2697         }
2698
2699         switch (field_op) {
2700         case FIELD_OP_MINUS:
2701                 op_fn = HIST_FIELD_FN_MINUS;
2702                 break;
2703         case FIELD_OP_PLUS:
2704                 op_fn = HIST_FIELD_FN_PLUS;
2705                 break;
2706         case FIELD_OP_DIV:
2707                 op_fn = HIST_FIELD_FN_DIV;
2708                 break;
2709         case FIELD_OP_MULT:
2710                 op_fn = HIST_FIELD_FN_MULT;
2711                 break;
2712         default:
2713                 ret = -EINVAL;
2714                 goto free_operands;
2715         }
2716
2717         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2718         if (ret)
2719                 goto free_operands;
2720
2721         operand_flags = var1 ? var1->flags : operand1->flags;
2722         operand2_flags = var2 ? var2->flags : operand2->flags;
2723
2724         /*
2725          * If both operands are constant, the expression can be
2726          * collapsed to a single constant.
2727          */
2728         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2729
2730         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2731
2732         flags |= operand1->flags &
2733                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2734
2735         expr = create_hist_field(hist_data, NULL, flags, var_name);
2736         if (!expr) {
2737                 ret = -ENOMEM;
2738                 goto free_operands;
2739         }
2740
2741         operand1->read_once = true;
2742         operand2->read_once = true;
2743
2744         /* The operands are now owned and free'd by 'expr' */
2745         expr->operands[0] = operand1;
2746         expr->operands[1] = operand2;
2747
2748         if (field_op == FIELD_OP_DIV &&
2749                         operand2_flags & HIST_FIELD_FL_CONST) {
2750                 u64 divisor = var2 ? var2->constant : operand2->constant;
2751
2752                 if (!divisor) {
2753                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2754                         ret = -EDOM;
2755                         goto free_expr;
2756                 }
2757
2758                 /*
2759                  * Copy the divisor here so we don't have to look it up
2760                  * later if this is a var ref
2761                  */
2762                 operand2->constant = divisor;
2763                 op_fn = hist_field_get_div_fn(operand2);
2764         }
2765
2766         expr->fn_num = op_fn;
2767
2768         if (combine_consts) {
2769                 if (var1)
2770                         expr->operands[0] = var1;
2771                 if (var2)
2772                         expr->operands[1] = var2;
2773
2774                 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2775                 expr->fn_num = HIST_FIELD_FN_CONST;
2776
2777                 expr->operands[0] = NULL;
2778                 expr->operands[1] = NULL;
2779
2780                 /*
2781                  * var refs won't be destroyed immediately
2782                  * See: destroy_hist_field()
2783                  */
2784                 destroy_hist_field(operand2, 0);
2785                 destroy_hist_field(operand1, 0);
2786
2787                 expr->name = expr_str(expr, 0);
2788         } else {
2789                 /* The operand sizes should be the same, so just pick one */
2790                 expr->size = operand1->size;
2791                 expr->is_signed = operand1->is_signed;
2792
2793                 expr->operator = field_op;
2794                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2795                 if (!expr->type) {
2796                         ret = -ENOMEM;
2797                         goto free_expr;
2798                 }
2799
2800                 expr->name = expr_str(expr, 0);
2801         }
2802
2803         return expr;
2804
2805 free_operands:
2806         destroy_hist_field(operand2, 0);
2807 free_op1:
2808         destroy_hist_field(operand1, 0);
2809         return ERR_PTR(ret);
2810
2811 free_expr:
2812         destroy_hist_field(expr, 0);
2813         return ERR_PTR(ret);
2814 }
2815
2816 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2817                                  struct trace_event_file *file)
2818 {
2819         struct event_trigger_data *test;
2820
2821         lockdep_assert_held(&event_mutex);
2822
2823         list_for_each_entry(test, &file->triggers, list) {
2824                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2825                         if (test->private_data == hist_data)
2826                                 return test->filter_str;
2827                 }
2828         }
2829
2830         return NULL;
2831 }
2832
2833 static struct event_command trigger_hist_cmd;
2834 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2835                                     struct trace_event_file *file,
2836                                     char *glob, char *cmd,
2837                                     char *param_and_filter);
2838
2839 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2840                             struct hist_trigger_data *hist_data,
2841                             unsigned int n_keys)
2842 {
2843         struct hist_field *target_hist_field, *hist_field;
2844         unsigned int n, i, j;
2845
2846         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2847                 return false;
2848
2849         i = hist_data->n_vals;
2850         j = target_hist_data->n_vals;
2851
2852         for (n = 0; n < n_keys; n++) {
2853                 hist_field = hist_data->fields[i + n];
2854                 target_hist_field = target_hist_data->fields[j + n];
2855
2856                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2857                         return false;
2858                 if (hist_field->size != target_hist_field->size)
2859                         return false;
2860                 if (hist_field->is_signed != target_hist_field->is_signed)
2861                         return false;
2862         }
2863
2864         return true;
2865 }
2866
2867 static struct hist_trigger_data *
2868 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2869                      struct trace_event_file *file)
2870 {
2871         struct hist_trigger_data *hist_data;
2872         struct event_trigger_data *test;
2873         unsigned int n_keys;
2874
2875         lockdep_assert_held(&event_mutex);
2876
2877         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2878
2879         list_for_each_entry(test, &file->triggers, list) {
2880                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2881                         hist_data = test->private_data;
2882
2883                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2884                                 return hist_data;
2885                 }
2886         }
2887
2888         return NULL;
2889 }
2890
2891 static struct trace_event_file *event_file(struct trace_array *tr,
2892                                            char *system, char *event_name)
2893 {
2894         struct trace_event_file *file;
2895
2896         file = __find_event_file(tr, system, event_name);
2897         if (!file)
2898                 return ERR_PTR(-EINVAL);
2899
2900         return file;
2901 }
2902
2903 static struct hist_field *
2904 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2905                          char *system, char *event_name, char *field_name)
2906 {
2907         struct hist_field *event_var;
2908         char *synthetic_name;
2909
2910         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2911         if (!synthetic_name)
2912                 return ERR_PTR(-ENOMEM);
2913
2914         strcpy(synthetic_name, "synthetic_");
2915         strcat(synthetic_name, field_name);
2916
2917         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2918
2919         kfree(synthetic_name);
2920
2921         return event_var;
2922 }
2923
2924 /**
2925  * create_field_var_hist - Automatically create a histogram and var for a field
2926  * @target_hist_data: The target hist trigger
2927  * @subsys_name: Optional subsystem name
2928  * @event_name: Optional event name
2929  * @field_name: The name of the field (and the resulting variable)
2930  *
2931  * Hist trigger actions fetch data from variables, not directly from
2932  * events.  However, for convenience, users are allowed to directly
2933  * specify an event field in an action, which will be automatically
2934  * converted into a variable on their behalf.
2935  *
2936  * If a user specifies a field on an event that isn't the event the
2937  * histogram currently being defined (the target event histogram), the
2938  * only way that can be accomplished is if a new hist trigger is
2939  * created and the field variable defined on that.
2940  *
2941  * This function creates a new histogram compatible with the target
2942  * event (meaning a histogram with the same key as the target
2943  * histogram), and creates a variable for the specified field, but
2944  * with 'synthetic_' prepended to the variable name in order to avoid
2945  * collision with normal field variables.
2946  *
2947  * Return: The variable created for the field.
2948  */
2949 static struct hist_field *
2950 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2951                       char *subsys_name, char *event_name, char *field_name)
2952 {
2953         struct trace_array *tr = target_hist_data->event_file->tr;
2954         struct hist_trigger_data *hist_data;
2955         unsigned int i, n, first = true;
2956         struct field_var_hist *var_hist;
2957         struct trace_event_file *file;
2958         struct hist_field *key_field;
2959         struct hist_field *event_var;
2960         char *saved_filter;
2961         char *cmd;
2962         int ret;
2963
2964         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2965                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2966                 return ERR_PTR(-EINVAL);
2967         }
2968
2969         file = event_file(tr, subsys_name, event_name);
2970
2971         if (IS_ERR(file)) {
2972                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2973                 ret = PTR_ERR(file);
2974                 return ERR_PTR(ret);
2975         }
2976
2977         /*
2978          * Look for a histogram compatible with target.  We'll use the
2979          * found histogram specification to create a new matching
2980          * histogram with our variable on it.  target_hist_data is not
2981          * yet a registered histogram so we can't use that.
2982          */
2983         hist_data = find_compatible_hist(target_hist_data, file);
2984         if (!hist_data) {
2985                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2986                 return ERR_PTR(-EINVAL);
2987         }
2988
2989         /* See if a synthetic field variable has already been created */
2990         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2991                                              event_name, field_name);
2992         if (!IS_ERR_OR_NULL(event_var))
2993                 return event_var;
2994
2995         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2996         if (!var_hist)
2997                 return ERR_PTR(-ENOMEM);
2998
2999         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3000         if (!cmd) {
3001                 kfree(var_hist);
3002                 return ERR_PTR(-ENOMEM);
3003         }
3004
3005         /* Use the same keys as the compatible histogram */
3006         strcat(cmd, "keys=");
3007
3008         for_each_hist_key_field(i, hist_data) {
3009                 key_field = hist_data->fields[i];
3010                 if (!first)
3011                         strcat(cmd, ",");
3012                 strcat(cmd, key_field->field->name);
3013                 first = false;
3014         }
3015
3016         /* Create the synthetic field variable specification */
3017         strcat(cmd, ":synthetic_");
3018         strcat(cmd, field_name);
3019         strcat(cmd, "=");
3020         strcat(cmd, field_name);
3021
3022         /* Use the same filter as the compatible histogram */
3023         saved_filter = find_trigger_filter(hist_data, file);
3024         if (saved_filter) {
3025                 strcat(cmd, " if ");
3026                 strcat(cmd, saved_filter);
3027         }
3028
3029         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3030         if (!var_hist->cmd) {
3031                 kfree(cmd);
3032                 kfree(var_hist);
3033                 return ERR_PTR(-ENOMEM);
3034         }
3035
3036         /* Save the compatible histogram information */
3037         var_hist->hist_data = hist_data;
3038
3039         /* Create the new histogram with our variable */
3040         ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3041                                        "", "hist", cmd);
3042         if (ret) {
3043                 kfree(cmd);
3044                 kfree(var_hist->cmd);
3045                 kfree(var_hist);
3046                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3047                 return ERR_PTR(ret);
3048         }
3049
3050         kfree(cmd);
3051
3052         /* If we can't find the variable, something went wrong */
3053         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3054                                              event_name, field_name);
3055         if (IS_ERR_OR_NULL(event_var)) {
3056                 kfree(var_hist->cmd);
3057                 kfree(var_hist);
3058                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3059                 return ERR_PTR(-EINVAL);
3060         }
3061
3062         n = target_hist_data->n_field_var_hists;
3063         target_hist_data->field_var_hists[n] = var_hist;
3064         target_hist_data->n_field_var_hists++;
3065
3066         return event_var;
3067 }
3068
3069 static struct hist_field *
3070 find_target_event_var(struct hist_trigger_data *hist_data,
3071                       char *subsys_name, char *event_name, char *var_name)
3072 {
3073         struct trace_event_file *file = hist_data->event_file;
3074         struct hist_field *hist_field = NULL;
3075
3076         if (subsys_name) {
3077                 struct trace_event_call *call;
3078
3079                 if (!event_name)
3080                         return NULL;
3081
3082                 call = file->event_call;
3083
3084                 if (strcmp(subsys_name, call->class->system) != 0)
3085                         return NULL;
3086
3087                 if (strcmp(event_name, trace_event_name(call)) != 0)
3088                         return NULL;
3089         }
3090
3091         hist_field = find_var_field(hist_data, var_name);
3092
3093         return hist_field;
3094 }
3095
3096 static inline void __update_field_vars(struct tracing_map_elt *elt,
3097                                        struct trace_buffer *buffer,
3098                                        struct ring_buffer_event *rbe,
3099                                        void *rec,
3100                                        struct field_var **field_vars,
3101                                        unsigned int n_field_vars,
3102                                        unsigned int field_var_str_start)
3103 {
3104         struct hist_elt_data *elt_data = elt->private_data;
3105         unsigned int i, j, var_idx;
3106         u64 var_val;
3107
3108         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3109                 struct field_var *field_var = field_vars[i];
3110                 struct hist_field *var = field_var->var;
3111                 struct hist_field *val = field_var->val;
3112
3113                 var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3114                 var_idx = var->var.idx;
3115
3116                 if (val->flags & HIST_FIELD_FL_STRING) {
3117                         char *str = elt_data->field_var_str[j++];
3118                         char *val_str = (char *)(uintptr_t)var_val;
3119                         unsigned int size;
3120
3121                         size = min(val->size, STR_VAR_LEN_MAX);
3122                         strscpy(str, val_str, size);
3123                         var_val = (u64)(uintptr_t)str;
3124                 }
3125                 tracing_map_set_var(elt, var_idx, var_val);
3126         }
3127 }
3128
3129 static void update_field_vars(struct hist_trigger_data *hist_data,
3130                               struct tracing_map_elt *elt,
3131                               struct trace_buffer *buffer,
3132                               struct ring_buffer_event *rbe,
3133                               void *rec)
3134 {
3135         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3136                             hist_data->n_field_vars, 0);
3137 }
3138
3139 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3140                                  struct tracing_map_elt *elt,
3141                                  struct trace_buffer *buffer,  void *rec,
3142                                  struct ring_buffer_event *rbe, void *key,
3143                                  struct action_data *data, u64 *var_ref_vals)
3144 {
3145         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3146                             hist_data->n_save_vars, hist_data->n_field_var_str);
3147 }
3148
3149 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3150                                      struct trace_event_file *file,
3151                                      char *name, int size, const char *type)
3152 {
3153         struct hist_field *var;
3154         int idx;
3155
3156         if (find_var(hist_data, file, name) && !hist_data->remove) {
3157                 var = ERR_PTR(-EINVAL);
3158                 goto out;
3159         }
3160
3161         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3162         if (!var) {
3163                 var = ERR_PTR(-ENOMEM);
3164                 goto out;
3165         }
3166
3167         idx = tracing_map_add_var(hist_data->map);
3168         if (idx < 0) {
3169                 kfree(var);
3170                 var = ERR_PTR(-EINVAL);
3171                 goto out;
3172         }
3173
3174         var->ref = 1;
3175         var->flags = HIST_FIELD_FL_VAR;
3176         var->var.idx = idx;
3177         var->var.hist_data = var->hist_data = hist_data;
3178         var->size = size;
3179         var->var.name = kstrdup(name, GFP_KERNEL);
3180         var->type = kstrdup_const(type, GFP_KERNEL);
3181         if (!var->var.name || !var->type) {
3182                 kfree_const(var->type);
3183                 kfree(var->var.name);
3184                 kfree(var);
3185                 var = ERR_PTR(-ENOMEM);
3186         }
3187  out:
3188         return var;
3189 }
3190
3191 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3192                                           struct trace_event_file *file,
3193                                           char *field_name)
3194 {
3195         struct hist_field *val = NULL, *var = NULL;
3196         unsigned long flags = HIST_FIELD_FL_VAR;
3197         struct trace_array *tr = file->tr;
3198         struct field_var *field_var;
3199         int ret = 0;
3200
3201         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3202                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3203                 ret = -EINVAL;
3204                 goto err;
3205         }
3206
3207         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3208         if (IS_ERR(val)) {
3209                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3210                 ret = PTR_ERR(val);
3211                 goto err;
3212         }
3213
3214         var = create_var(hist_data, file, field_name, val->size, val->type);
3215         if (IS_ERR(var)) {
3216                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3217                 kfree(val);
3218                 ret = PTR_ERR(var);
3219                 goto err;
3220         }
3221
3222         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3223         if (!field_var) {
3224                 kfree(val);
3225                 kfree(var);
3226                 ret =  -ENOMEM;
3227                 goto err;
3228         }
3229
3230         field_var->var = var;
3231         field_var->val = val;
3232  out:
3233         return field_var;
3234  err:
3235         field_var = ERR_PTR(ret);
3236         goto out;
3237 }
3238
3239 /**
3240  * create_target_field_var - Automatically create a variable for a field
3241  * @target_hist_data: The target hist trigger
3242  * @subsys_name: Optional subsystem name
3243  * @event_name: Optional event name
3244  * @var_name: The name of the field (and the resulting variable)
3245  *
3246  * Hist trigger actions fetch data from variables, not directly from
3247  * events.  However, for convenience, users are allowed to directly
3248  * specify an event field in an action, which will be automatically
3249  * converted into a variable on their behalf.
3250  *
3251  * This function creates a field variable with the name var_name on
3252  * the hist trigger currently being defined on the target event.  If
3253  * subsys_name and event_name are specified, this function simply
3254  * verifies that they do in fact match the target event subsystem and
3255  * event name.
3256  *
3257  * Return: The variable created for the field.
3258  */
3259 static struct field_var *
3260 create_target_field_var(struct hist_trigger_data *target_hist_data,
3261                         char *subsys_name, char *event_name, char *var_name)
3262 {
3263         struct trace_event_file *file = target_hist_data->event_file;
3264
3265         if (subsys_name) {
3266                 struct trace_event_call *call;
3267
3268                 if (!event_name)
3269                         return NULL;
3270
3271                 call = file->event_call;
3272
3273                 if (strcmp(subsys_name, call->class->system) != 0)
3274                         return NULL;
3275
3276                 if (strcmp(event_name, trace_event_name(call)) != 0)
3277                         return NULL;
3278         }
3279
3280         return create_field_var(target_hist_data, file, var_name);
3281 }
3282
3283 static bool check_track_val_max(u64 track_val, u64 var_val)
3284 {
3285         if (var_val <= track_val)
3286                 return false;
3287
3288         return true;
3289 }
3290
3291 static bool check_track_val_changed(u64 track_val, u64 var_val)
3292 {
3293         if (var_val == track_val)
3294                 return false;
3295
3296         return true;
3297 }
3298
3299 static u64 get_track_val(struct hist_trigger_data *hist_data,
3300                          struct tracing_map_elt *elt,
3301                          struct action_data *data)
3302 {
3303         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3304         u64 track_val;
3305
3306         track_val = tracing_map_read_var(elt, track_var_idx);
3307
3308         return track_val;
3309 }
3310
3311 static void save_track_val(struct hist_trigger_data *hist_data,
3312                            struct tracing_map_elt *elt,
3313                            struct action_data *data, u64 var_val)
3314 {
3315         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3316
3317         tracing_map_set_var(elt, track_var_idx, var_val);
3318 }
3319
3320 static void save_track_data(struct hist_trigger_data *hist_data,
3321                             struct tracing_map_elt *elt,
3322                             struct trace_buffer *buffer, void *rec,
3323                             struct ring_buffer_event *rbe, void *key,
3324                             struct action_data *data, u64 *var_ref_vals)
3325 {
3326         if (data->track_data.save_data)
3327                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3328                                            key, data, var_ref_vals);
3329 }
3330
3331 static bool check_track_val(struct tracing_map_elt *elt,
3332                             struct action_data *data,
3333                             u64 var_val)
3334 {
3335         struct hist_trigger_data *hist_data;
3336         u64 track_val;
3337
3338         hist_data = data->track_data.track_var->hist_data;
3339         track_val = get_track_val(hist_data, elt, data);
3340
3341         return data->track_data.check_val(track_val, var_val);
3342 }
3343
3344 #ifdef CONFIG_TRACER_SNAPSHOT
3345 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3346 {
3347         /* called with tr->max_lock held */
3348         struct track_data *track_data = tr->cond_snapshot->cond_data;
3349         struct hist_elt_data *elt_data, *track_elt_data;
3350         struct snapshot_context *context = cond_data;
3351         struct action_data *action;
3352         u64 track_val;
3353
3354         if (!track_data)
3355                 return false;
3356
3357         action = track_data->action_data;
3358
3359         track_val = get_track_val(track_data->hist_data, context->elt,
3360                                   track_data->action_data);
3361
3362         if (!action->track_data.check_val(track_data->track_val, track_val))
3363                 return false;
3364
3365         track_data->track_val = track_val;
3366         memcpy(track_data->key, context->key, track_data->key_len);
3367
3368         elt_data = context->elt->private_data;
3369         track_elt_data = track_data->elt.private_data;
3370         if (elt_data->comm)
3371                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3372
3373         track_data->updated = true;
3374
3375         return true;
3376 }
3377
3378 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3379                                      struct tracing_map_elt *elt,
3380                                      struct trace_buffer *buffer, void *rec,
3381                                      struct ring_buffer_event *rbe, void *key,
3382                                      struct action_data *data,
3383                                      u64 *var_ref_vals)
3384 {
3385         struct trace_event_file *file = hist_data->event_file;
3386         struct snapshot_context context;
3387
3388         context.elt = elt;
3389         context.key = key;
3390
3391         tracing_snapshot_cond(file->tr, &context);
3392 }
3393
3394 static void hist_trigger_print_key(struct seq_file *m,
3395                                    struct hist_trigger_data *hist_data,
3396                                    void *key,
3397                                    struct tracing_map_elt *elt);
3398
3399 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3400 {
3401         unsigned int i;
3402
3403         if (!hist_data->n_actions)
3404                 return NULL;
3405
3406         for (i = 0; i < hist_data->n_actions; i++) {
3407                 struct action_data *data = hist_data->actions[i];
3408
3409                 if (data->action == ACTION_SNAPSHOT)
3410                         return data;
3411         }
3412
3413         return NULL;
3414 }
3415
3416 static void track_data_snapshot_print(struct seq_file *m,
3417                                       struct hist_trigger_data *hist_data)
3418 {
3419         struct trace_event_file *file = hist_data->event_file;
3420         struct track_data *track_data;
3421         struct action_data *action;
3422
3423         track_data = tracing_cond_snapshot_data(file->tr);
3424         if (!track_data)
3425                 return;
3426
3427         if (!track_data->updated)
3428                 return;
3429
3430         action = snapshot_action(hist_data);
3431         if (!action)
3432                 return;
3433
3434         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3435         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3436                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3437                    action->track_data.var_str, track_data->track_val);
3438
3439         seq_puts(m, "\ttriggered by event with key: ");
3440         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3441         seq_putc(m, '\n');
3442 }
3443 #else
3444 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3445 {
3446         return false;
3447 }
3448 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3449                                      struct tracing_map_elt *elt,
3450                                      struct trace_buffer *buffer, void *rec,
3451                                      struct ring_buffer_event *rbe, void *key,
3452                                      struct action_data *data,
3453                                      u64 *var_ref_vals) {}
3454 static void track_data_snapshot_print(struct seq_file *m,
3455                                       struct hist_trigger_data *hist_data) {}
3456 #endif /* CONFIG_TRACER_SNAPSHOT */
3457
3458 static void track_data_print(struct seq_file *m,
3459                              struct hist_trigger_data *hist_data,
3460                              struct tracing_map_elt *elt,
3461                              struct action_data *data)
3462 {
3463         u64 track_val = get_track_val(hist_data, elt, data);
3464         unsigned int i, save_var_idx;
3465
3466         if (data->handler == HANDLER_ONMAX)
3467                 seq_printf(m, "\n\tmax: %10llu", track_val);
3468         else if (data->handler == HANDLER_ONCHANGE)
3469                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3470
3471         if (data->action == ACTION_SNAPSHOT)
3472                 return;
3473
3474         for (i = 0; i < hist_data->n_save_vars; i++) {
3475                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3476                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3477                 u64 val;
3478
3479                 save_var_idx = save_var->var.idx;
3480
3481                 val = tracing_map_read_var(elt, save_var_idx);
3482
3483                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3484                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3485                                    (char *)(uintptr_t)(val));
3486                 } else
3487                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3488         }
3489 }
3490
3491 static void ontrack_action(struct hist_trigger_data *hist_data,
3492                            struct tracing_map_elt *elt,
3493                            struct trace_buffer *buffer, void *rec,
3494                            struct ring_buffer_event *rbe, void *key,
3495                            struct action_data *data, u64 *var_ref_vals)
3496 {
3497         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3498
3499         if (check_track_val(elt, data, var_val)) {
3500                 save_track_val(hist_data, elt, data, var_val);
3501                 save_track_data(hist_data, elt, buffer, rec, rbe,
3502                                 key, data, var_ref_vals);
3503         }
3504 }
3505
3506 static void action_data_destroy(struct action_data *data)
3507 {
3508         unsigned int i;
3509
3510         lockdep_assert_held(&event_mutex);
3511
3512         kfree(data->action_name);
3513
3514         for (i = 0; i < data->n_params; i++)
3515                 kfree(data->params[i]);
3516
3517         if (data->synth_event)
3518                 data->synth_event->ref--;
3519
3520         kfree(data->synth_event_name);
3521
3522         kfree(data);
3523 }
3524
3525 static void track_data_destroy(struct hist_trigger_data *hist_data,
3526                                struct action_data *data)
3527 {
3528         struct trace_event_file *file = hist_data->event_file;
3529
3530         destroy_hist_field(data->track_data.track_var, 0);
3531
3532         if (data->action == ACTION_SNAPSHOT) {
3533                 struct track_data *track_data;
3534
3535                 track_data = tracing_cond_snapshot_data(file->tr);
3536                 if (track_data && track_data->hist_data == hist_data) {
3537                         tracing_snapshot_cond_disable(file->tr);
3538                         track_data_free(track_data);
3539                 }
3540         }
3541
3542         kfree(data->track_data.var_str);
3543
3544         action_data_destroy(data);
3545 }
3546
3547 static int action_create(struct hist_trigger_data *hist_data,
3548                          struct action_data *data);
3549
3550 static int track_data_create(struct hist_trigger_data *hist_data,
3551                              struct action_data *data)
3552 {
3553         struct hist_field *var_field, *ref_field, *track_var = NULL;
3554         struct trace_event_file *file = hist_data->event_file;
3555         struct trace_array *tr = file->tr;
3556         char *track_data_var_str;
3557         int ret = 0;
3558
3559         track_data_var_str = data->track_data.var_str;
3560         if (track_data_var_str[0] != '$') {
3561                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3562                 return -EINVAL;
3563         }
3564         track_data_var_str++;
3565
3566         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3567         if (!var_field) {
3568                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3569                 return -EINVAL;
3570         }
3571
3572         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3573         if (!ref_field)
3574                 return -ENOMEM;
3575
3576         data->track_data.var_ref = ref_field;
3577
3578         if (data->handler == HANDLER_ONMAX)
3579                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3580         if (IS_ERR(track_var)) {
3581                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3582                 ret = PTR_ERR(track_var);
3583                 goto out;
3584         }
3585
3586         if (data->handler == HANDLER_ONCHANGE)
3587                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3588         if (IS_ERR(track_var)) {
3589                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3590                 ret = PTR_ERR(track_var);
3591                 goto out;
3592         }
3593         data->track_data.track_var = track_var;
3594
3595         ret = action_create(hist_data, data);
3596  out:
3597         return ret;
3598 }
3599
3600 static int parse_action_params(struct trace_array *tr, char *params,
3601                                struct action_data *data)
3602 {
3603         char *param, *saved_param;
3604         bool first_param = true;
3605         int ret = 0;
3606
3607         while (params) {
3608                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3609                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3610                         ret = -EINVAL;
3611                         goto out;
3612                 }
3613
3614                 param = strsep(&params, ",");
3615                 if (!param) {
3616                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3617                         ret = -EINVAL;
3618                         goto out;
3619                 }
3620
3621                 param = strstrip(param);
3622                 if (strlen(param) < 2) {
3623                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3624                         ret = -EINVAL;
3625                         goto out;
3626                 }
3627
3628                 saved_param = kstrdup(param, GFP_KERNEL);
3629                 if (!saved_param) {
3630                         ret = -ENOMEM;
3631                         goto out;
3632                 }
3633
3634                 if (first_param && data->use_trace_keyword) {
3635                         data->synth_event_name = saved_param;
3636                         first_param = false;
3637                         continue;
3638                 }
3639                 first_param = false;
3640
3641                 data->params[data->n_params++] = saved_param;
3642         }
3643  out:
3644         return ret;
3645 }
3646
3647 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3648                         enum handler_id handler)
3649 {
3650         char *action_name;
3651         int ret = 0;
3652
3653         strsep(&str, ".");
3654         if (!str) {
3655                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3656                 ret = -EINVAL;
3657                 goto out;
3658         }
3659
3660         action_name = strsep(&str, "(");
3661         if (!action_name || !str) {
3662                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3663                 ret = -EINVAL;
3664                 goto out;
3665         }
3666
3667         if (str_has_prefix(action_name, "save")) {
3668                 char *params = strsep(&str, ")");
3669
3670                 if (!params) {
3671                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3672                         ret = -EINVAL;
3673                         goto out;
3674                 }
3675
3676                 ret = parse_action_params(tr, params, data);
3677                 if (ret)
3678                         goto out;
3679
3680                 if (handler == HANDLER_ONMAX)
3681                         data->track_data.check_val = check_track_val_max;
3682                 else if (handler == HANDLER_ONCHANGE)
3683                         data->track_data.check_val = check_track_val_changed;
3684                 else {
3685                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3686                         ret = -EINVAL;
3687                         goto out;
3688                 }
3689
3690                 data->track_data.save_data = save_track_data_vars;
3691                 data->fn = ontrack_action;
3692                 data->action = ACTION_SAVE;
3693         } else if (str_has_prefix(action_name, "snapshot")) {
3694                 char *params = strsep(&str, ")");
3695
3696                 if (!str) {
3697                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3698                         ret = -EINVAL;
3699                         goto out;
3700                 }
3701
3702                 if (handler == HANDLER_ONMAX)
3703                         data->track_data.check_val = check_track_val_max;
3704                 else if (handler == HANDLER_ONCHANGE)
3705                         data->track_data.check_val = check_track_val_changed;
3706                 else {
3707                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3708                         ret = -EINVAL;
3709                         goto out;
3710                 }
3711
3712                 data->track_data.save_data = save_track_data_snapshot;
3713                 data->fn = ontrack_action;
3714                 data->action = ACTION_SNAPSHOT;
3715         } else {
3716                 char *params = strsep(&str, ")");
3717
3718                 if (str_has_prefix(action_name, "trace"))
3719                         data->use_trace_keyword = true;
3720
3721                 if (params) {
3722                         ret = parse_action_params(tr, params, data);
3723                         if (ret)
3724                                 goto out;
3725                 }
3726
3727                 if (handler == HANDLER_ONMAX)
3728                         data->track_data.check_val = check_track_val_max;
3729                 else if (handler == HANDLER_ONCHANGE)
3730                         data->track_data.check_val = check_track_val_changed;
3731
3732                 if (handler != HANDLER_ONMATCH) {
3733                         data->track_data.save_data = action_trace;
3734                         data->fn = ontrack_action;
3735                 } else
3736                         data->fn = action_trace;
3737
3738                 data->action = ACTION_TRACE;
3739         }
3740
3741         data->action_name = kstrdup(action_name, GFP_KERNEL);
3742         if (!data->action_name) {
3743                 ret = -ENOMEM;
3744                 goto out;
3745         }
3746
3747         data->handler = handler;
3748  out:
3749         return ret;
3750 }
3751
3752 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3753                                             char *str, enum handler_id handler)
3754 {
3755         struct action_data *data;
3756         int ret = -EINVAL;
3757         char *var_str;
3758
3759         data = kzalloc(sizeof(*data), GFP_KERNEL);
3760         if (!data)
3761                 return ERR_PTR(-ENOMEM);
3762
3763         var_str = strsep(&str, ")");
3764         if (!var_str || !str) {
3765                 ret = -EINVAL;
3766                 goto free;
3767         }
3768
3769         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3770         if (!data->track_data.var_str) {
3771                 ret = -ENOMEM;
3772                 goto free;
3773         }
3774
3775         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3776         if (ret)
3777                 goto free;
3778  out:
3779         return data;
3780  free:
3781         track_data_destroy(hist_data, data);
3782         data = ERR_PTR(ret);
3783         goto out;
3784 }
3785
3786 static void onmatch_destroy(struct action_data *data)
3787 {
3788         kfree(data->match_data.event);
3789         kfree(data->match_data.event_system);
3790
3791         action_data_destroy(data);
3792 }
3793
3794 static void destroy_field_var(struct field_var *field_var)
3795 {
3796         if (!field_var)
3797                 return;
3798
3799         destroy_hist_field(field_var->var, 0);
3800         destroy_hist_field(field_var->val, 0);
3801
3802         kfree(field_var);
3803 }
3804
3805 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3806 {
3807         unsigned int i;
3808
3809         for (i = 0; i < hist_data->n_field_vars; i++)
3810                 destroy_field_var(hist_data->field_vars[i]);
3811
3812         for (i = 0; i < hist_data->n_save_vars; i++)
3813                 destroy_field_var(hist_data->save_vars[i]);
3814 }
3815
3816 static void save_field_var(struct hist_trigger_data *hist_data,
3817                            struct field_var *field_var)
3818 {
3819         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3820
3821         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3822                 hist_data->n_field_var_str++;
3823 }
3824
3825
3826 static int check_synth_field(struct synth_event *event,
3827                              struct hist_field *hist_field,
3828                              unsigned int field_pos)
3829 {
3830         struct synth_field *field;
3831
3832         if (field_pos >= event->n_fields)
3833                 return -EINVAL;
3834
3835         field = event->fields[field_pos];
3836
3837         /*
3838          * A dynamic string synth field can accept static or
3839          * dynamic. A static string synth field can only accept a
3840          * same-sized static string, which is checked for later.
3841          */
3842         if (strstr(hist_field->type, "char[") && field->is_string
3843             && field->is_dynamic)
3844                 return 0;
3845
3846         if (strcmp(field->type, hist_field->type) != 0) {
3847                 if (field->size != hist_field->size ||
3848                     (!field->is_string && field->is_signed != hist_field->is_signed))
3849                         return -EINVAL;
3850         }
3851
3852         return 0;
3853 }
3854
3855 static struct hist_field *
3856 trace_action_find_var(struct hist_trigger_data *hist_data,
3857                       struct action_data *data,
3858                       char *system, char *event, char *var)
3859 {
3860         struct trace_array *tr = hist_data->event_file->tr;
3861         struct hist_field *hist_field;
3862
3863         var++; /* skip '$' */
3864
3865         hist_field = find_target_event_var(hist_data, system, event, var);
3866         if (!hist_field) {
3867                 if (!system && data->handler == HANDLER_ONMATCH) {
3868                         system = data->match_data.event_system;
3869                         event = data->match_data.event;
3870                 }
3871
3872                 hist_field = find_event_var(hist_data, system, event, var);
3873         }
3874
3875         if (!hist_field)
3876                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3877
3878         return hist_field;
3879 }
3880
3881 static struct hist_field *
3882 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3883                               struct action_data *data, char *system,
3884                               char *event, char *var)
3885 {
3886         struct hist_field *hist_field = NULL;
3887         struct field_var *field_var;
3888
3889         /*
3890          * First try to create a field var on the target event (the
3891          * currently being defined).  This will create a variable for
3892          * unqualified fields on the target event, or if qualified,
3893          * target fields that have qualified names matching the target.
3894          */
3895         field_var = create_target_field_var(hist_data, system, event, var);
3896
3897         if (field_var && !IS_ERR(field_var)) {
3898                 save_field_var(hist_data, field_var);
3899                 hist_field = field_var->var;
3900         } else {
3901                 field_var = NULL;
3902                 /*
3903                  * If no explicit system.event is specified, default to
3904                  * looking for fields on the onmatch(system.event.xxx)
3905                  * event.
3906                  */
3907                 if (!system && data->handler == HANDLER_ONMATCH) {
3908                         system = data->match_data.event_system;
3909                         event = data->match_data.event;
3910                 }
3911
3912                 if (!event)
3913                         goto free;
3914                 /*
3915                  * At this point, we're looking at a field on another
3916                  * event.  Because we can't modify a hist trigger on
3917                  * another event to add a variable for a field, we need
3918                  * to create a new trigger on that event and create the
3919                  * variable at the same time.
3920                  */
3921                 hist_field = create_field_var_hist(hist_data, system, event, var);
3922                 if (IS_ERR(hist_field))
3923                         goto free;
3924         }
3925  out:
3926         return hist_field;
3927  free:
3928         destroy_field_var(field_var);
3929         hist_field = NULL;
3930         goto out;
3931 }
3932
3933 static int trace_action_create(struct hist_trigger_data *hist_data,
3934                                struct action_data *data)
3935 {
3936         struct trace_array *tr = hist_data->event_file->tr;
3937         char *event_name, *param, *system = NULL;
3938         struct hist_field *hist_field, *var_ref;
3939         unsigned int i;
3940         unsigned int field_pos = 0;
3941         struct synth_event *event;
3942         char *synth_event_name;
3943         int var_ref_idx, ret = 0;
3944
3945         lockdep_assert_held(&event_mutex);
3946
3947         /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
3948         if (data->n_params > SYNTH_FIELDS_MAX)
3949                 return -EINVAL;
3950
3951         if (data->use_trace_keyword)
3952                 synth_event_name = data->synth_event_name;
3953         else
3954                 synth_event_name = data->action_name;
3955
3956         event = find_synth_event(synth_event_name);
3957         if (!event) {
3958                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3959                 return -EINVAL;
3960         }
3961
3962         event->ref++;
3963
3964         for (i = 0; i < data->n_params; i++) {
3965                 char *p;
3966
3967                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3968                 if (!param) {
3969                         ret = -ENOMEM;
3970                         goto err;
3971                 }
3972
3973                 system = strsep(&param, ".");
3974                 if (!param) {
3975                         param = (char *)system;
3976                         system = event_name = NULL;
3977                 } else {
3978                         event_name = strsep(&param, ".");
3979                         if (!param) {
3980                                 kfree(p);
3981                                 ret = -EINVAL;
3982                                 goto err;
3983                         }
3984                 }
3985
3986                 if (param[0] == '$')
3987                         hist_field = trace_action_find_var(hist_data, data,
3988                                                            system, event_name,
3989                                                            param);
3990                 else
3991                         hist_field = trace_action_create_field_var(hist_data,
3992                                                                    data,
3993                                                                    system,
3994                                                                    event_name,
3995                                                                    param);
3996
3997                 if (!hist_field) {
3998                         kfree(p);
3999                         ret = -EINVAL;
4000                         goto err;
4001                 }
4002
4003                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4004                         var_ref = create_var_ref(hist_data, hist_field,
4005                                                  system, event_name);
4006                         if (!var_ref) {
4007                                 kfree(p);
4008                                 ret = -ENOMEM;
4009                                 goto err;
4010                         }
4011
4012                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4013                         if (WARN_ON(var_ref_idx < 0)) {
4014                                 kfree(p);
4015                                 ret = var_ref_idx;
4016                                 goto err;
4017                         }
4018
4019                         data->var_ref_idx[i] = var_ref_idx;
4020
4021                         field_pos++;
4022                         kfree(p);
4023                         continue;
4024                 }
4025
4026                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4027                 kfree(p);
4028                 ret = -EINVAL;
4029                 goto err;
4030         }
4031
4032         if (field_pos != event->n_fields) {
4033                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4034                 ret = -EINVAL;
4035                 goto err;
4036         }
4037
4038         data->synth_event = event;
4039  out:
4040         return ret;
4041  err:
4042         event->ref--;
4043
4044         goto out;
4045 }
4046
4047 static int action_create(struct hist_trigger_data *hist_data,
4048                          struct action_data *data)
4049 {
4050         struct trace_event_file *file = hist_data->event_file;
4051         struct trace_array *tr = file->tr;
4052         struct track_data *track_data;
4053         struct field_var *field_var;
4054         unsigned int i;
4055         char *param;
4056         int ret = 0;
4057
4058         if (data->action == ACTION_TRACE)
4059                 return trace_action_create(hist_data, data);
4060
4061         if (data->action == ACTION_SNAPSHOT) {
4062                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4063                 if (IS_ERR(track_data)) {
4064                         ret = PTR_ERR(track_data);
4065                         goto out;
4066                 }
4067
4068                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4069                                                    cond_snapshot_update);
4070                 if (ret)
4071                         track_data_free(track_data);
4072
4073                 goto out;
4074         }
4075
4076         if (data->action == ACTION_SAVE) {
4077                 if (hist_data->n_save_vars) {
4078                         ret = -EEXIST;
4079                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4080                         goto out;
4081                 }
4082
4083                 for (i = 0; i < data->n_params; i++) {
4084                         param = kstrdup(data->params[i], GFP_KERNEL);
4085                         if (!param) {
4086                                 ret = -ENOMEM;
4087                                 goto out;
4088                         }
4089
4090                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4091                         if (IS_ERR(field_var)) {
4092                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4093                                          errpos(param));
4094                                 ret = PTR_ERR(field_var);
4095                                 kfree(param);
4096                                 goto out;
4097                         }
4098
4099                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4100                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4101                                 hist_data->n_save_var_str++;
4102                         kfree(param);
4103                 }
4104         }
4105  out:
4106         return ret;
4107 }
4108
4109 static int onmatch_create(struct hist_trigger_data *hist_data,
4110                           struct action_data *data)
4111 {
4112         return action_create(hist_data, data);
4113 }
4114
4115 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4116 {
4117         char *match_event, *match_event_system;
4118         struct action_data *data;
4119         int ret = -EINVAL;
4120
4121         data = kzalloc(sizeof(*data), GFP_KERNEL);
4122         if (!data)
4123                 return ERR_PTR(-ENOMEM);
4124
4125         match_event = strsep(&str, ")");
4126         if (!match_event || !str) {
4127                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4128                 goto free;
4129         }
4130
4131         match_event_system = strsep(&match_event, ".");
4132         if (!match_event) {
4133                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4134                 goto free;
4135         }
4136
4137         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4138                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4139                 goto free;
4140         }
4141
4142         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4143         if (!data->match_data.event) {
4144                 ret = -ENOMEM;
4145                 goto free;
4146         }
4147
4148         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4149         if (!data->match_data.event_system) {
4150                 ret = -ENOMEM;
4151                 goto free;
4152         }
4153
4154         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4155         if (ret)
4156                 goto free;
4157  out:
4158         return data;
4159  free:
4160         onmatch_destroy(data);
4161         data = ERR_PTR(ret);
4162         goto out;
4163 }
4164
4165 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4166 {
4167         hist_data->fields[HITCOUNT_IDX] =
4168                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4169         if (!hist_data->fields[HITCOUNT_IDX])
4170                 return -ENOMEM;
4171
4172         hist_data->n_vals++;
4173         hist_data->n_fields++;
4174
4175         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4176                 return -EINVAL;
4177
4178         return 0;
4179 }
4180
4181 static int __create_val_field(struct hist_trigger_data *hist_data,
4182                               unsigned int val_idx,
4183                               struct trace_event_file *file,
4184                               char *var_name, char *field_str,
4185                               unsigned long flags)
4186 {
4187         struct hist_field *hist_field;
4188         int ret = 0, n_subexprs = 0;
4189
4190         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4191         if (IS_ERR(hist_field)) {
4192                 ret = PTR_ERR(hist_field);
4193                 goto out;
4194         }
4195
4196         /* Some types cannot be a value */
4197         if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4198                                  HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 |
4199                                  HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET |
4200                                  HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE)) {
4201                 hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str));
4202                 ret = -EINVAL;
4203         }
4204
4205         hist_data->fields[val_idx] = hist_field;
4206
4207         ++hist_data->n_vals;
4208         ++hist_data->n_fields;
4209
4210         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4211                 ret = -EINVAL;
4212  out:
4213         return ret;
4214 }
4215
4216 static int create_val_field(struct hist_trigger_data *hist_data,
4217                             unsigned int val_idx,
4218                             struct trace_event_file *file,
4219                             char *field_str)
4220 {
4221         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4222                 return -EINVAL;
4223
4224         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4225 }
4226
4227 static const char no_comm[] = "(no comm)";
4228
4229 static u64 hist_field_execname(struct hist_field *hist_field,
4230                                struct tracing_map_elt *elt,
4231                                struct trace_buffer *buffer,
4232                                struct ring_buffer_event *rbe,
4233                                void *event)
4234 {
4235         struct hist_elt_data *elt_data;
4236
4237         if (WARN_ON_ONCE(!elt))
4238                 return (u64)(unsigned long)no_comm;
4239
4240         elt_data = elt->private_data;
4241
4242         if (WARN_ON_ONCE(!elt_data->comm))
4243                 return (u64)(unsigned long)no_comm;
4244
4245         return (u64)(unsigned long)(elt_data->comm);
4246 }
4247
4248 static u64 hist_fn_call(struct hist_field *hist_field,
4249                         struct tracing_map_elt *elt,
4250                         struct trace_buffer *buffer,
4251                         struct ring_buffer_event *rbe,
4252                         void *event)
4253 {
4254         switch (hist_field->fn_num) {
4255         case HIST_FIELD_FN_VAR_REF:
4256                 return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4257         case HIST_FIELD_FN_COUNTER:
4258                 return hist_field_counter(hist_field, elt, buffer, rbe, event);
4259         case HIST_FIELD_FN_CONST:
4260                 return hist_field_const(hist_field, elt, buffer, rbe, event);
4261         case HIST_FIELD_FN_LOG2:
4262                 return hist_field_log2(hist_field, elt, buffer, rbe, event);
4263         case HIST_FIELD_FN_BUCKET:
4264                 return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4265         case HIST_FIELD_FN_TIMESTAMP:
4266                 return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4267         case HIST_FIELD_FN_CPU:
4268                 return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4269         case HIST_FIELD_FN_STRING:
4270                 return hist_field_string(hist_field, elt, buffer, rbe, event);
4271         case HIST_FIELD_FN_DYNSTRING:
4272                 return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4273         case HIST_FIELD_FN_RELDYNSTRING:
4274                 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4275         case HIST_FIELD_FN_PSTRING:
4276                 return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4277         case HIST_FIELD_FN_S64:
4278                 return hist_field_s64(hist_field, elt, buffer, rbe, event);
4279         case HIST_FIELD_FN_U64:
4280                 return hist_field_u64(hist_field, elt, buffer, rbe, event);
4281         case HIST_FIELD_FN_S32:
4282                 return hist_field_s32(hist_field, elt, buffer, rbe, event);
4283         case HIST_FIELD_FN_U32:
4284                 return hist_field_u32(hist_field, elt, buffer, rbe, event);
4285         case HIST_FIELD_FN_S16:
4286                 return hist_field_s16(hist_field, elt, buffer, rbe, event);
4287         case HIST_FIELD_FN_U16:
4288                 return hist_field_u16(hist_field, elt, buffer, rbe, event);
4289         case HIST_FIELD_FN_S8:
4290                 return hist_field_s8(hist_field, elt, buffer, rbe, event);
4291         case HIST_FIELD_FN_U8:
4292                 return hist_field_u8(hist_field, elt, buffer, rbe, event);
4293         case HIST_FIELD_FN_UMINUS:
4294                 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4295         case HIST_FIELD_FN_MINUS:
4296                 return hist_field_minus(hist_field, elt, buffer, rbe, event);
4297         case HIST_FIELD_FN_PLUS:
4298                 return hist_field_plus(hist_field, elt, buffer, rbe, event);
4299         case HIST_FIELD_FN_DIV:
4300                 return hist_field_div(hist_field, elt, buffer, rbe, event);
4301         case HIST_FIELD_FN_MULT:
4302                 return hist_field_mult(hist_field, elt, buffer, rbe, event);
4303         case HIST_FIELD_FN_DIV_POWER2:
4304                 return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4305         case HIST_FIELD_FN_DIV_NOT_POWER2:
4306                 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4307         case HIST_FIELD_FN_DIV_MULT_SHIFT:
4308                 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4309         case HIST_FIELD_FN_EXECNAME:
4310                 return hist_field_execname(hist_field, elt, buffer, rbe, event);
4311         default:
4312                 return 0;
4313         }
4314 }
4315
4316 /* Convert a var that points to common_pid.execname to a string */
4317 static void update_var_execname(struct hist_field *hist_field)
4318 {
4319         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4320                 HIST_FIELD_FL_EXECNAME;
4321         hist_field->size = MAX_FILTER_STR_VAL;
4322         hist_field->is_signed = 0;
4323
4324         kfree_const(hist_field->type);
4325         hist_field->type = "char[]";
4326
4327         hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4328 }
4329
4330 static int create_var_field(struct hist_trigger_data *hist_data,
4331                             unsigned int val_idx,
4332                             struct trace_event_file *file,
4333                             char *var_name, char *expr_str)
4334 {
4335         struct trace_array *tr = hist_data->event_file->tr;
4336         unsigned long flags = 0;
4337         int ret;
4338
4339         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4340                 return -EINVAL;
4341
4342         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4343                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4344                 return -EINVAL;
4345         }
4346
4347         flags |= HIST_FIELD_FL_VAR;
4348         hist_data->n_vars++;
4349         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4350                 return -EINVAL;
4351
4352         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4353
4354         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4355                 update_var_execname(hist_data->fields[val_idx]);
4356
4357         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
4358                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4359
4360         return ret;
4361 }
4362
4363 static int create_val_fields(struct hist_trigger_data *hist_data,
4364                              struct trace_event_file *file)
4365 {
4366         char *fields_str, *field_str;
4367         unsigned int i, j = 1;
4368         int ret;
4369
4370         ret = create_hitcount_val(hist_data);
4371         if (ret)
4372                 goto out;
4373
4374         fields_str = hist_data->attrs->vals_str;
4375         if (!fields_str)
4376                 goto out;
4377
4378         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4379                      j < TRACING_MAP_VALS_MAX; i++) {
4380                 field_str = strsep(&fields_str, ",");
4381                 if (!field_str)
4382                         break;
4383
4384                 if (strcmp(field_str, "hitcount") == 0)
4385                         continue;
4386
4387                 ret = create_val_field(hist_data, j++, file, field_str);
4388                 if (ret)
4389                         goto out;
4390         }
4391
4392         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4393                 ret = -EINVAL;
4394  out:
4395         return ret;
4396 }
4397
4398 static int create_key_field(struct hist_trigger_data *hist_data,
4399                             unsigned int key_idx,
4400                             unsigned int key_offset,
4401                             struct trace_event_file *file,
4402                             char *field_str)
4403 {
4404         struct trace_array *tr = hist_data->event_file->tr;
4405         struct hist_field *hist_field = NULL;
4406         unsigned long flags = 0;
4407         unsigned int key_size;
4408         int ret = 0, n_subexprs = 0;
4409
4410         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4411                 return -EINVAL;
4412
4413         flags |= HIST_FIELD_FL_KEY;
4414
4415         if (strcmp(field_str, "stacktrace") == 0) {
4416                 flags |= HIST_FIELD_FL_STACKTRACE;
4417                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4418                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4419         } else {
4420                 hist_field = parse_expr(hist_data, file, field_str, flags,
4421                                         NULL, &n_subexprs);
4422                 if (IS_ERR(hist_field)) {
4423                         ret = PTR_ERR(hist_field);
4424                         goto out;
4425                 }
4426
4427                 if (field_has_hist_vars(hist_field, 0)) {
4428                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4429                         destroy_hist_field(hist_field, 0);
4430                         ret = -EINVAL;
4431                         goto out;
4432                 }
4433
4434                 key_size = hist_field->size;
4435         }
4436
4437         hist_data->fields[key_idx] = hist_field;
4438
4439         key_size = ALIGN(key_size, sizeof(u64));
4440         hist_data->fields[key_idx]->size = key_size;
4441         hist_data->fields[key_idx]->offset = key_offset;
4442
4443         hist_data->key_size += key_size;
4444
4445         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4446                 ret = -EINVAL;
4447                 goto out;
4448         }
4449
4450         hist_data->n_keys++;
4451         hist_data->n_fields++;
4452
4453         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4454                 return -EINVAL;
4455
4456         ret = key_size;
4457  out:
4458         return ret;
4459 }
4460
4461 static int create_key_fields(struct hist_trigger_data *hist_data,
4462                              struct trace_event_file *file)
4463 {
4464         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4465         char *fields_str, *field_str;
4466         int ret = -EINVAL;
4467
4468         fields_str = hist_data->attrs->keys_str;
4469         if (!fields_str)
4470                 goto out;
4471
4472         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4473                 field_str = strsep(&fields_str, ",");
4474                 if (!field_str)
4475                         break;
4476                 ret = create_key_field(hist_data, i, key_offset,
4477                                        file, field_str);
4478                 if (ret < 0)
4479                         goto out;
4480                 key_offset += ret;
4481         }
4482         if (fields_str) {
4483                 ret = -EINVAL;
4484                 goto out;
4485         }
4486         ret = 0;
4487  out:
4488         return ret;
4489 }
4490
4491 static int create_var_fields(struct hist_trigger_data *hist_data,
4492                              struct trace_event_file *file)
4493 {
4494         unsigned int i, j = hist_data->n_vals;
4495         int ret = 0;
4496
4497         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4498
4499         for (i = 0; i < n_vars; i++) {
4500                 char *var_name = hist_data->attrs->var_defs.name[i];
4501                 char *expr = hist_data->attrs->var_defs.expr[i];
4502
4503                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4504                 if (ret)
4505                         goto out;
4506         }
4507  out:
4508         return ret;
4509 }
4510
4511 static void free_var_defs(struct hist_trigger_data *hist_data)
4512 {
4513         unsigned int i;
4514
4515         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4516                 kfree(hist_data->attrs->var_defs.name[i]);
4517                 kfree(hist_data->attrs->var_defs.expr[i]);
4518         }
4519
4520         hist_data->attrs->var_defs.n_vars = 0;
4521 }
4522
4523 static int parse_var_defs(struct hist_trigger_data *hist_data)
4524 {
4525         struct trace_array *tr = hist_data->event_file->tr;
4526         char *s, *str, *var_name, *field_str;
4527         unsigned int i, j, n_vars = 0;
4528         int ret = 0;
4529
4530         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4531                 str = hist_data->attrs->assignment_str[i];
4532                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4533                         field_str = strsep(&str, ",");
4534                         if (!field_str)
4535                                 break;
4536
4537                         var_name = strsep(&field_str, "=");
4538                         if (!var_name || !field_str) {
4539                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4540                                          errpos(var_name));
4541                                 ret = -EINVAL;
4542                                 goto free;
4543                         }
4544
4545                         if (n_vars == TRACING_MAP_VARS_MAX) {
4546                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4547                                 ret = -EINVAL;
4548                                 goto free;
4549                         }
4550
4551                         s = kstrdup(var_name, GFP_KERNEL);
4552                         if (!s) {
4553                                 ret = -ENOMEM;
4554                                 goto free;
4555                         }
4556                         hist_data->attrs->var_defs.name[n_vars] = s;
4557
4558                         s = kstrdup(field_str, GFP_KERNEL);
4559                         if (!s) {
4560                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4561                                 hist_data->attrs->var_defs.name[n_vars] = NULL;
4562                                 ret = -ENOMEM;
4563                                 goto free;
4564                         }
4565                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4566
4567                         hist_data->attrs->var_defs.n_vars = n_vars;
4568                 }
4569         }
4570
4571         return ret;
4572  free:
4573         free_var_defs(hist_data);
4574
4575         return ret;
4576 }
4577
4578 static int create_hist_fields(struct hist_trigger_data *hist_data,
4579                               struct trace_event_file *file)
4580 {
4581         int ret;
4582
4583         ret = parse_var_defs(hist_data);
4584         if (ret)
4585                 return ret;
4586
4587         ret = create_val_fields(hist_data, file);
4588         if (ret)
4589                 goto out;
4590
4591         ret = create_var_fields(hist_data, file);
4592         if (ret)
4593                 goto out;
4594
4595         ret = create_key_fields(hist_data, file);
4596
4597  out:
4598         free_var_defs(hist_data);
4599
4600         return ret;
4601 }
4602
4603 static int is_descending(struct trace_array *tr, const char *str)
4604 {
4605         if (!str)
4606                 return 0;
4607
4608         if (strcmp(str, "descending") == 0)
4609                 return 1;
4610
4611         if (strcmp(str, "ascending") == 0)
4612                 return 0;
4613
4614         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4615
4616         return -EINVAL;
4617 }
4618
4619 static int create_sort_keys(struct hist_trigger_data *hist_data)
4620 {
4621         struct trace_array *tr = hist_data->event_file->tr;
4622         char *fields_str = hist_data->attrs->sort_key_str;
4623         struct tracing_map_sort_key *sort_key;
4624         int descending, ret = 0;
4625         unsigned int i, j, k;
4626
4627         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4628
4629         if (!fields_str)
4630                 goto out;
4631
4632         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4633                 struct hist_field *hist_field;
4634                 char *field_str, *field_name;
4635                 const char *test_name;
4636
4637                 sort_key = &hist_data->sort_keys[i];
4638
4639                 field_str = strsep(&fields_str, ",");
4640                 if (!field_str)
4641                         break;
4642
4643                 if (!*field_str) {
4644                         ret = -EINVAL;
4645                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4646                         break;
4647                 }
4648
4649                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4650                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4651                         ret = -EINVAL;
4652                         break;
4653                 }
4654
4655                 field_name = strsep(&field_str, ".");
4656                 if (!field_name || !*field_name) {
4657                         ret = -EINVAL;
4658                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4659                         break;
4660                 }
4661
4662                 if (strcmp(field_name, "hitcount") == 0) {
4663                         descending = is_descending(tr, field_str);
4664                         if (descending < 0) {
4665                                 ret = descending;
4666                                 break;
4667                         }
4668                         sort_key->descending = descending;
4669                         continue;
4670                 }
4671
4672                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4673                         unsigned int idx;
4674
4675                         hist_field = hist_data->fields[j];
4676                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4677                                 continue;
4678
4679                         idx = k++;
4680
4681                         test_name = hist_field_name(hist_field, 0);
4682
4683                         if (strcmp(field_name, test_name) == 0) {
4684                                 sort_key->field_idx = idx;
4685                                 descending = is_descending(tr, field_str);
4686                                 if (descending < 0) {
4687                                         ret = descending;
4688                                         goto out;
4689                                 }
4690                                 sort_key->descending = descending;
4691                                 break;
4692                         }
4693                 }
4694                 if (j == hist_data->n_fields) {
4695                         ret = -EINVAL;
4696                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4697                         break;
4698                 }
4699         }
4700
4701         hist_data->n_sort_keys = i;
4702  out:
4703         return ret;
4704 }
4705
4706 static void destroy_actions(struct hist_trigger_data *hist_data)
4707 {
4708         unsigned int i;
4709
4710         for (i = 0; i < hist_data->n_actions; i++) {
4711                 struct action_data *data = hist_data->actions[i];
4712
4713                 if (data->handler == HANDLER_ONMATCH)
4714                         onmatch_destroy(data);
4715                 else if (data->handler == HANDLER_ONMAX ||
4716                          data->handler == HANDLER_ONCHANGE)
4717                         track_data_destroy(hist_data, data);
4718                 else
4719                         kfree(data);
4720         }
4721 }
4722
4723 static int parse_actions(struct hist_trigger_data *hist_data)
4724 {
4725         struct trace_array *tr = hist_data->event_file->tr;
4726         struct action_data *data;
4727         unsigned int i;
4728         int ret = 0;
4729         char *str;
4730         int len;
4731
4732         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4733                 str = hist_data->attrs->action_str[i];
4734
4735                 if ((len = str_has_prefix(str, "onmatch("))) {
4736                         char *action_str = str + len;
4737
4738                         data = onmatch_parse(tr, action_str);
4739                         if (IS_ERR(data)) {
4740                                 ret = PTR_ERR(data);
4741                                 break;
4742                         }
4743                 } else if ((len = str_has_prefix(str, "onmax("))) {
4744                         char *action_str = str + len;
4745
4746                         data = track_data_parse(hist_data, action_str,
4747                                                 HANDLER_ONMAX);
4748                         if (IS_ERR(data)) {
4749                                 ret = PTR_ERR(data);
4750                                 break;
4751                         }
4752                 } else if ((len = str_has_prefix(str, "onchange("))) {
4753                         char *action_str = str + len;
4754
4755                         data = track_data_parse(hist_data, action_str,
4756                                                 HANDLER_ONCHANGE);
4757                         if (IS_ERR(data)) {
4758                                 ret = PTR_ERR(data);
4759                                 break;
4760                         }
4761                 } else {
4762                         ret = -EINVAL;
4763                         break;
4764                 }
4765
4766                 hist_data->actions[hist_data->n_actions++] = data;
4767         }
4768
4769         return ret;
4770 }
4771
4772 static int create_actions(struct hist_trigger_data *hist_data)
4773 {
4774         struct action_data *data;
4775         unsigned int i;
4776         int ret = 0;
4777
4778         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4779                 data = hist_data->actions[i];
4780
4781                 if (data->handler == HANDLER_ONMATCH) {
4782                         ret = onmatch_create(hist_data, data);
4783                         if (ret)
4784                                 break;
4785                 } else if (data->handler == HANDLER_ONMAX ||
4786                            data->handler == HANDLER_ONCHANGE) {
4787                         ret = track_data_create(hist_data, data);
4788                         if (ret)
4789                                 break;
4790                 } else {
4791                         ret = -EINVAL;
4792                         break;
4793                 }
4794         }
4795
4796         return ret;
4797 }
4798
4799 static void print_actions(struct seq_file *m,
4800                           struct hist_trigger_data *hist_data,
4801                           struct tracing_map_elt *elt)
4802 {
4803         unsigned int i;
4804
4805         for (i = 0; i < hist_data->n_actions; i++) {
4806                 struct action_data *data = hist_data->actions[i];
4807
4808                 if (data->action == ACTION_SNAPSHOT)
4809                         continue;
4810
4811                 if (data->handler == HANDLER_ONMAX ||
4812                     data->handler == HANDLER_ONCHANGE)
4813                         track_data_print(m, hist_data, elt, data);
4814         }
4815 }
4816
4817 static void print_action_spec(struct seq_file *m,
4818                               struct hist_trigger_data *hist_data,
4819                               struct action_data *data)
4820 {
4821         unsigned int i;
4822
4823         if (data->action == ACTION_SAVE) {
4824                 for (i = 0; i < hist_data->n_save_vars; i++) {
4825                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4826                         if (i < hist_data->n_save_vars - 1)
4827                                 seq_puts(m, ",");
4828                 }
4829         } else if (data->action == ACTION_TRACE) {
4830                 if (data->use_trace_keyword)
4831                         seq_printf(m, "%s", data->synth_event_name);
4832                 for (i = 0; i < data->n_params; i++) {
4833                         if (i || data->use_trace_keyword)
4834                                 seq_puts(m, ",");
4835                         seq_printf(m, "%s", data->params[i]);
4836                 }
4837         }
4838 }
4839
4840 static void print_track_data_spec(struct seq_file *m,
4841                                   struct hist_trigger_data *hist_data,
4842                                   struct action_data *data)
4843 {
4844         if (data->handler == HANDLER_ONMAX)
4845                 seq_puts(m, ":onmax(");
4846         else if (data->handler == HANDLER_ONCHANGE)
4847                 seq_puts(m, ":onchange(");
4848         seq_printf(m, "%s", data->track_data.var_str);
4849         seq_printf(m, ").%s(", data->action_name);
4850
4851         print_action_spec(m, hist_data, data);
4852
4853         seq_puts(m, ")");
4854 }
4855
4856 static void print_onmatch_spec(struct seq_file *m,
4857                                struct hist_trigger_data *hist_data,
4858                                struct action_data *data)
4859 {
4860         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4861                    data->match_data.event);
4862
4863         seq_printf(m, "%s(", data->action_name);
4864
4865         print_action_spec(m, hist_data, data);
4866
4867         seq_puts(m, ")");
4868 }
4869
4870 static bool actions_match(struct hist_trigger_data *hist_data,
4871                           struct hist_trigger_data *hist_data_test)
4872 {
4873         unsigned int i, j;
4874
4875         if (hist_data->n_actions != hist_data_test->n_actions)
4876                 return false;
4877
4878         for (i = 0; i < hist_data->n_actions; i++) {
4879                 struct action_data *data = hist_data->actions[i];
4880                 struct action_data *data_test = hist_data_test->actions[i];
4881                 char *action_name, *action_name_test;
4882
4883                 if (data->handler != data_test->handler)
4884                         return false;
4885                 if (data->action != data_test->action)
4886                         return false;
4887
4888                 if (data->n_params != data_test->n_params)
4889                         return false;
4890
4891                 for (j = 0; j < data->n_params; j++) {
4892                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4893                                 return false;
4894                 }
4895
4896                 if (data->use_trace_keyword)
4897                         action_name = data->synth_event_name;
4898                 else
4899                         action_name = data->action_name;
4900
4901                 if (data_test->use_trace_keyword)
4902                         action_name_test = data_test->synth_event_name;
4903                 else
4904                         action_name_test = data_test->action_name;
4905
4906                 if (strcmp(action_name, action_name_test) != 0)
4907                         return false;
4908
4909                 if (data->handler == HANDLER_ONMATCH) {
4910                         if (strcmp(data->match_data.event_system,
4911                                    data_test->match_data.event_system) != 0)
4912                                 return false;
4913                         if (strcmp(data->match_data.event,
4914                                    data_test->match_data.event) != 0)
4915                                 return false;
4916                 } else if (data->handler == HANDLER_ONMAX ||
4917                            data->handler == HANDLER_ONCHANGE) {
4918                         if (strcmp(data->track_data.var_str,
4919                                    data_test->track_data.var_str) != 0)
4920                                 return false;
4921                 }
4922         }
4923
4924         return true;
4925 }
4926
4927
4928 static void print_actions_spec(struct seq_file *m,
4929                                struct hist_trigger_data *hist_data)
4930 {
4931         unsigned int i;
4932
4933         for (i = 0; i < hist_data->n_actions; i++) {
4934                 struct action_data *data = hist_data->actions[i];
4935
4936                 if (data->handler == HANDLER_ONMATCH)
4937                         print_onmatch_spec(m, hist_data, data);
4938                 else if (data->handler == HANDLER_ONMAX ||
4939                          data->handler == HANDLER_ONCHANGE)
4940                         print_track_data_spec(m, hist_data, data);
4941         }
4942 }
4943
4944 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4945 {
4946         unsigned int i;
4947
4948         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4949                 kfree(hist_data->field_var_hists[i]->cmd);
4950                 kfree(hist_data->field_var_hists[i]);
4951         }
4952 }
4953
4954 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4955 {
4956         if (!hist_data)
4957                 return;
4958
4959         destroy_hist_trigger_attrs(hist_data->attrs);
4960         destroy_hist_fields(hist_data);
4961         tracing_map_destroy(hist_data->map);
4962
4963         destroy_actions(hist_data);
4964         destroy_field_vars(hist_data);
4965         destroy_field_var_hists(hist_data);
4966
4967         kfree(hist_data);
4968 }
4969
4970 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4971 {
4972         struct tracing_map *map = hist_data->map;
4973         struct ftrace_event_field *field;
4974         struct hist_field *hist_field;
4975         int i, idx = 0;
4976
4977         for_each_hist_field(i, hist_data) {
4978                 hist_field = hist_data->fields[i];
4979                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4980                         tracing_map_cmp_fn_t cmp_fn;
4981
4982                         field = hist_field->field;
4983
4984                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4985                                 cmp_fn = tracing_map_cmp_none;
4986                         else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
4987                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4988                                                              hist_field->is_signed);
4989                         else if (is_string_field(field))
4990                                 cmp_fn = tracing_map_cmp_string;
4991                         else
4992                                 cmp_fn = tracing_map_cmp_num(field->size,
4993                                                              field->is_signed);
4994                         idx = tracing_map_add_key_field(map,
4995                                                         hist_field->offset,
4996                                                         cmp_fn);
4997                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4998                         idx = tracing_map_add_sum_field(map);
4999
5000                 if (idx < 0)
5001                         return idx;
5002
5003                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5004                         idx = tracing_map_add_var(map);
5005                         if (idx < 0)
5006                                 return idx;
5007                         hist_field->var.idx = idx;
5008                         hist_field->var.hist_data = hist_data;
5009                 }
5010         }
5011
5012         return 0;
5013 }
5014
5015 static struct hist_trigger_data *
5016 create_hist_data(unsigned int map_bits,
5017                  struct hist_trigger_attrs *attrs,
5018                  struct trace_event_file *file,
5019                  bool remove)
5020 {
5021         const struct tracing_map_ops *map_ops = NULL;
5022         struct hist_trigger_data *hist_data;
5023         int ret = 0;
5024
5025         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5026         if (!hist_data)
5027                 return ERR_PTR(-ENOMEM);
5028
5029         hist_data->attrs = attrs;
5030         hist_data->remove = remove;
5031         hist_data->event_file = file;
5032
5033         ret = parse_actions(hist_data);
5034         if (ret)
5035                 goto free;
5036
5037         ret = create_hist_fields(hist_data, file);
5038         if (ret)
5039                 goto free;
5040
5041         ret = create_sort_keys(hist_data);
5042         if (ret)
5043                 goto free;
5044
5045         map_ops = &hist_trigger_elt_data_ops;
5046
5047         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5048                                             map_ops, hist_data);
5049         if (IS_ERR(hist_data->map)) {
5050                 ret = PTR_ERR(hist_data->map);
5051                 hist_data->map = NULL;
5052                 goto free;
5053         }
5054
5055         ret = create_tracing_map_fields(hist_data);
5056         if (ret)
5057                 goto free;
5058  out:
5059         return hist_data;
5060  free:
5061         hist_data->attrs = NULL;
5062
5063         destroy_hist_data(hist_data);
5064
5065         hist_data = ERR_PTR(ret);
5066
5067         goto out;
5068 }
5069
5070 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5071                                     struct tracing_map_elt *elt,
5072                                     struct trace_buffer *buffer, void *rec,
5073                                     struct ring_buffer_event *rbe,
5074                                     u64 *var_ref_vals)
5075 {
5076         struct hist_elt_data *elt_data;
5077         struct hist_field *hist_field;
5078         unsigned int i, var_idx;
5079         u64 hist_val;
5080
5081         elt_data = elt->private_data;
5082         elt_data->var_ref_vals = var_ref_vals;
5083
5084         for_each_hist_val_field(i, hist_data) {
5085                 hist_field = hist_data->fields[i];
5086                 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5087                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5088                         var_idx = hist_field->var.idx;
5089
5090                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
5091                                 unsigned int str_start, var_str_idx, idx;
5092                                 char *str, *val_str;
5093                                 unsigned int size;
5094
5095                                 str_start = hist_data->n_field_var_str +
5096                                         hist_data->n_save_var_str;
5097                                 var_str_idx = hist_field->var_str_idx;
5098                                 idx = str_start + var_str_idx;
5099
5100                                 str = elt_data->field_var_str[idx];
5101                                 val_str = (char *)(uintptr_t)hist_val;
5102
5103                                 size = min(hist_field->size, STR_VAR_LEN_MAX);
5104                                 strscpy(str, val_str, size);
5105
5106                                 hist_val = (u64)(uintptr_t)str;
5107                         }
5108                         tracing_map_set_var(elt, var_idx, hist_val);
5109                         continue;
5110                 }
5111                 tracing_map_update_sum(elt, i, hist_val);
5112         }
5113
5114         for_each_hist_key_field(i, hist_data) {
5115                 hist_field = hist_data->fields[i];
5116                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5117                         hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5118                         var_idx = hist_field->var.idx;
5119                         tracing_map_set_var(elt, var_idx, hist_val);
5120                 }
5121         }
5122
5123         update_field_vars(hist_data, elt, buffer, rbe, rec);
5124 }
5125
5126 static inline void add_to_key(char *compound_key, void *key,
5127                               struct hist_field *key_field, void *rec)
5128 {
5129         size_t size = key_field->size;
5130
5131         if (key_field->flags & HIST_FIELD_FL_STRING) {
5132                 struct ftrace_event_field *field;
5133
5134                 field = key_field->field;
5135                 if (field->filter_type == FILTER_DYN_STRING ||
5136                     field->filter_type == FILTER_RDYN_STRING)
5137                         size = *(u32 *)(rec + field->offset) >> 16;
5138                 else if (field->filter_type == FILTER_STATIC_STRING)
5139                         size = field->size;
5140
5141                 /* ensure NULL-termination */
5142                 if (size > key_field->size - 1)
5143                         size = key_field->size - 1;
5144
5145                 strncpy(compound_key + key_field->offset, (char *)key, size);
5146         } else
5147                 memcpy(compound_key + key_field->offset, key, size);
5148 }
5149
5150 static void
5151 hist_trigger_actions(struct hist_trigger_data *hist_data,
5152                      struct tracing_map_elt *elt,
5153                      struct trace_buffer *buffer, void *rec,
5154                      struct ring_buffer_event *rbe, void *key,
5155                      u64 *var_ref_vals)
5156 {
5157         struct action_data *data;
5158         unsigned int i;
5159
5160         for (i = 0; i < hist_data->n_actions; i++) {
5161                 data = hist_data->actions[i];
5162                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5163         }
5164 }
5165
5166 static void event_hist_trigger(struct event_trigger_data *data,
5167                                struct trace_buffer *buffer, void *rec,
5168                                struct ring_buffer_event *rbe)
5169 {
5170         struct hist_trigger_data *hist_data = data->private_data;
5171         bool use_compound_key = (hist_data->n_keys > 1);
5172         unsigned long entries[HIST_STACKTRACE_DEPTH];
5173         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5174         char compound_key[HIST_KEY_SIZE_MAX];
5175         struct tracing_map_elt *elt = NULL;
5176         struct hist_field *key_field;
5177         u64 field_contents;
5178         void *key = NULL;
5179         unsigned int i;
5180
5181         if (unlikely(!rbe))
5182                 return;
5183
5184         memset(compound_key, 0, hist_data->key_size);
5185
5186         for_each_hist_key_field(i, hist_data) {
5187                 key_field = hist_data->fields[i];
5188
5189                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5190                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5191                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5192                                          HIST_STACKTRACE_SKIP);
5193                         key = entries;
5194                 } else {
5195                         field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5196                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5197                                 key = (void *)(unsigned long)field_contents;
5198                                 use_compound_key = true;
5199                         } else
5200                                 key = (void *)&field_contents;
5201                 }
5202
5203                 if (use_compound_key)
5204                         add_to_key(compound_key, key, key_field, rec);
5205         }
5206
5207         if (use_compound_key)
5208                 key = compound_key;
5209
5210         if (hist_data->n_var_refs &&
5211             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5212                 return;
5213
5214         elt = tracing_map_insert(hist_data->map, key);
5215         if (!elt)
5216                 return;
5217
5218         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5219
5220         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5221                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5222 }
5223
5224 static void hist_trigger_stacktrace_print(struct seq_file *m,
5225                                           unsigned long *stacktrace_entries,
5226                                           unsigned int max_entries)
5227 {
5228         unsigned int spaces = 8;
5229         unsigned int i;
5230
5231         for (i = 0; i < max_entries; i++) {
5232                 if (!stacktrace_entries[i])
5233                         return;
5234
5235                 seq_printf(m, "%*c", 1 + spaces, ' ');
5236                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5237         }
5238 }
5239
5240 static void hist_trigger_print_key(struct seq_file *m,
5241                                    struct hist_trigger_data *hist_data,
5242                                    void *key,
5243                                    struct tracing_map_elt *elt)
5244 {
5245         struct hist_field *key_field;
5246         bool multiline = false;
5247         const char *field_name;
5248         unsigned int i;
5249         u64 uval;
5250
5251         seq_puts(m, "{ ");
5252
5253         for_each_hist_key_field(i, hist_data) {
5254                 key_field = hist_data->fields[i];
5255
5256                 if (i > hist_data->n_vals)
5257                         seq_puts(m, ", ");
5258
5259                 field_name = hist_field_name(key_field, 0);
5260
5261                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5262                         uval = *(u64 *)(key + key_field->offset);
5263                         seq_printf(m, "%s: %llx", field_name, uval);
5264                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5265                         uval = *(u64 *)(key + key_field->offset);
5266                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5267                                    uval, (void *)(uintptr_t)uval);
5268                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5269                         uval = *(u64 *)(key + key_field->offset);
5270                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5271                                    uval, (void *)(uintptr_t)uval);
5272                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5273                         struct hist_elt_data *elt_data = elt->private_data;
5274                         char *comm;
5275
5276                         if (WARN_ON_ONCE(!elt_data))
5277                                 return;
5278
5279                         comm = elt_data->comm;
5280
5281                         uval = *(u64 *)(key + key_field->offset);
5282                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5283                                    comm, uval);
5284                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5285                         const char *syscall_name;
5286
5287                         uval = *(u64 *)(key + key_field->offset);
5288                         syscall_name = get_syscall_name(uval);
5289                         if (!syscall_name)
5290                                 syscall_name = "unknown_syscall";
5291
5292                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5293                                    syscall_name, uval);
5294                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5295                         seq_puts(m, "stacktrace:\n");
5296                         hist_trigger_stacktrace_print(m,
5297                                                       key + key_field->offset,
5298                                                       HIST_STACKTRACE_DEPTH);
5299                         multiline = true;
5300                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5301                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5302                                    *(u64 *)(key + key_field->offset));
5303                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5304                         unsigned long buckets = key_field->buckets;
5305                         uval = *(u64 *)(key + key_field->offset);
5306                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5307                                    uval, uval + buckets -1);
5308                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5309                         seq_printf(m, "%s: %-50s", field_name,
5310                                    (char *)(key + key_field->offset));
5311                 } else {
5312                         uval = *(u64 *)(key + key_field->offset);
5313                         seq_printf(m, "%s: %10llu", field_name, uval);
5314                 }
5315         }
5316
5317         if (!multiline)
5318                 seq_puts(m, " ");
5319
5320         seq_puts(m, "}");
5321 }
5322
5323 /* Get the 100 times of the percentage of @val in @total */
5324 static inline unsigned int __get_percentage(u64 val, u64 total)
5325 {
5326         if (!total)
5327                 goto div0;
5328
5329         if (val < (U64_MAX / 10000))
5330                 return (unsigned int)div64_ul(val * 10000, total);
5331
5332         total = div64_u64(total, 10000);
5333         if (!total)
5334                 goto div0;
5335
5336         return (unsigned int)div64_ul(val, total);
5337 div0:
5338         return val ? UINT_MAX : 0;
5339 }
5340
5341 #define BAR_CHAR '#'
5342
5343 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5344 {
5345         unsigned int len = __get_percentage(val, max);
5346         int i;
5347
5348         if (len == UINT_MAX) {
5349                 snprintf(buf, size, "[ERROR]");
5350                 return buf;
5351         }
5352
5353         len = len * size / 10000;
5354         for (i = 0; i < len && i < size; i++)
5355                 buf[i] = BAR_CHAR;
5356         while (i < size)
5357                 buf[i++] = ' ';
5358         buf[size] = '\0';
5359
5360         return buf;
5361 }
5362
5363 struct hist_val_stat {
5364         u64 max;
5365         u64 total;
5366 };
5367
5368 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5369                                    const char *field_name, unsigned long flags,
5370                                    struct hist_val_stat *stats,
5371                                    struct tracing_map_elt *elt)
5372 {
5373         u64 val = tracing_map_read_sum(elt, idx);
5374         unsigned int pc;
5375         char bar[21];
5376
5377         if (flags & HIST_FIELD_FL_PERCENT) {
5378                 pc = __get_percentage(val, stats[idx].total);
5379                 if (pc == UINT_MAX)
5380                         seq_printf(m, " %s (%%):[ERROR]", field_name);
5381                 else
5382                         seq_printf(m, " %s (%%): %3u.%02u", field_name,
5383                                         pc / 100, pc % 100);
5384         } else if (flags & HIST_FIELD_FL_GRAPH) {
5385                 seq_printf(m, " %s: %20s", field_name,
5386                            __fill_bar_str(bar, 20, val, stats[idx].max));
5387         } else if (flags & HIST_FIELD_FL_HEX) {
5388                 seq_printf(m, " %s: %10llx", field_name, val);
5389         } else {
5390                 seq_printf(m, " %s: %10llu", field_name, val);
5391         }
5392 }
5393
5394 static void hist_trigger_entry_print(struct seq_file *m,
5395                                      struct hist_trigger_data *hist_data,
5396                                      struct hist_val_stat *stats,
5397                                      void *key,
5398                                      struct tracing_map_elt *elt)
5399 {
5400         const char *field_name;
5401         unsigned int i = HITCOUNT_IDX;
5402         unsigned long flags;
5403
5404         hist_trigger_print_key(m, hist_data, key, elt);
5405
5406         /* At first, show the raw hitcount always */
5407         hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5408
5409         for (i = 1; i < hist_data->n_vals; i++) {
5410                 field_name = hist_field_name(hist_data->fields[i], 0);
5411                 flags = hist_data->fields[i]->flags;
5412
5413                 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
5414                         continue;
5415
5416                 seq_puts(m, " ");
5417                 hist_trigger_print_val(m, i, field_name, flags, stats, elt);
5418         }
5419
5420         print_actions(m, hist_data, elt);
5421
5422         seq_puts(m, "\n");
5423 }
5424
5425 static int print_entries(struct seq_file *m,
5426                          struct hist_trigger_data *hist_data)
5427 {
5428         struct tracing_map_sort_entry **sort_entries = NULL;
5429         struct tracing_map *map = hist_data->map;
5430         int i, j, n_entries;
5431         struct hist_val_stat *stats = NULL;
5432         u64 val;
5433
5434         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5435                                              hist_data->n_sort_keys,
5436                                              &sort_entries);
5437         if (n_entries < 0)
5438                 return n_entries;
5439
5440         /* Calculate the max and the total for each field if needed. */
5441         for (j = 0; j < hist_data->n_vals; j++) {
5442                 if (!(hist_data->fields[j]->flags &
5443                         (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5444                         continue;
5445                 if (!stats) {
5446                         stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5447                                        GFP_KERNEL);
5448                         if (!stats) {
5449                                 n_entries = -ENOMEM;
5450                                 goto out;
5451                         }
5452                 }
5453                 for (i = 0; i < n_entries; i++) {
5454                         val = tracing_map_read_sum(sort_entries[i]->elt, j);
5455                         stats[j].total += val;
5456                         if (stats[j].max < val)
5457                                 stats[j].max = val;
5458                 }
5459         }
5460
5461         for (i = 0; i < n_entries; i++)
5462                 hist_trigger_entry_print(m, hist_data, stats,
5463                                          sort_entries[i]->key,
5464                                          sort_entries[i]->elt);
5465
5466         kfree(stats);
5467 out:
5468         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5469
5470         return n_entries;
5471 }
5472
5473 static void hist_trigger_show(struct seq_file *m,
5474                               struct event_trigger_data *data, int n)
5475 {
5476         struct hist_trigger_data *hist_data;
5477         int n_entries;
5478
5479         if (n > 0)
5480                 seq_puts(m, "\n\n");
5481
5482         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5483         data->ops->print(m, data);
5484         seq_puts(m, "#\n\n");
5485
5486         hist_data = data->private_data;
5487         n_entries = print_entries(m, hist_data);
5488         if (n_entries < 0)
5489                 n_entries = 0;
5490
5491         track_data_snapshot_print(m, hist_data);
5492
5493         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5494                    (u64)atomic64_read(&hist_data->map->hits),
5495                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5496 }
5497
5498 static int hist_show(struct seq_file *m, void *v)
5499 {
5500         struct event_trigger_data *data;
5501         struct trace_event_file *event_file;
5502         int n = 0, ret = 0;
5503
5504         mutex_lock(&event_mutex);
5505
5506         event_file = event_file_data(m->private);
5507         if (unlikely(!event_file)) {
5508                 ret = -ENODEV;
5509                 goto out_unlock;
5510         }
5511
5512         list_for_each_entry(data, &event_file->triggers, list) {
5513                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5514                         hist_trigger_show(m, data, n++);
5515         }
5516
5517  out_unlock:
5518         mutex_unlock(&event_mutex);
5519
5520         return ret;
5521 }
5522
5523 static int event_hist_open(struct inode *inode, struct file *file)
5524 {
5525         int ret;
5526
5527         ret = security_locked_down(LOCKDOWN_TRACEFS);
5528         if (ret)
5529                 return ret;
5530
5531         return single_open(file, hist_show, file);
5532 }
5533
5534 const struct file_operations event_hist_fops = {
5535         .open = event_hist_open,
5536         .read = seq_read,
5537         .llseek = seq_lseek,
5538         .release = single_release,
5539 };
5540
5541 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5542 static void hist_field_debug_show_flags(struct seq_file *m,
5543                                         unsigned long flags)
5544 {
5545         seq_puts(m, "      flags:\n");
5546
5547         if (flags & HIST_FIELD_FL_KEY)
5548                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5549         else if (flags & HIST_FIELD_FL_HITCOUNT)
5550                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5551         else if (flags & HIST_FIELD_FL_VAR)
5552                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5553         else if (flags & HIST_FIELD_FL_VAR_REF)
5554                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5555         else
5556                 seq_puts(m, "        VAL: normal u64 value\n");
5557
5558         if (flags & HIST_FIELD_FL_ALIAS)
5559                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5560         else if (flags & HIST_FIELD_FL_CONST)
5561                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5562 }
5563
5564 static int hist_field_debug_show(struct seq_file *m,
5565                                  struct hist_field *field, unsigned long flags)
5566 {
5567         if ((field->flags & flags) != flags) {
5568                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5569                 return -EINVAL;
5570         }
5571
5572         hist_field_debug_show_flags(m, field->flags);
5573         if (field->field)
5574                 seq_printf(m, "      ftrace_event_field name: %s\n",
5575                            field->field->name);
5576
5577         if (field->flags & HIST_FIELD_FL_VAR) {
5578                 seq_printf(m, "      var.name: %s\n", field->var.name);
5579                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5580                            field->var.idx);
5581         }
5582
5583         if (field->flags & HIST_FIELD_FL_CONST)
5584                 seq_printf(m, "      constant: %llu\n", field->constant);
5585
5586         if (field->flags & HIST_FIELD_FL_ALIAS)
5587                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5588                            field->var_ref_idx);
5589
5590         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5591                 seq_printf(m, "      name: %s\n", field->name);
5592                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5593                            field->var.idx);
5594                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5595                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5596                            field->var_ref_idx);
5597                 if (field->system)
5598                         seq_printf(m, "      system: %s\n", field->system);
5599                 if (field->event_name)
5600                         seq_printf(m, "      event_name: %s\n", field->event_name);
5601         }
5602
5603         seq_printf(m, "      type: %s\n", field->type);
5604         seq_printf(m, "      size: %u\n", field->size);
5605         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5606
5607         return 0;
5608 }
5609
5610 static int field_var_debug_show(struct seq_file *m,
5611                                 struct field_var *field_var, unsigned int i,
5612                                 bool save_vars)
5613 {
5614         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5615         struct hist_field *field;
5616         int ret = 0;
5617
5618         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5619
5620         field = field_var->var;
5621
5622         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5623
5624         hist_field_debug_show_flags(m, field->flags);
5625         seq_printf(m, "      var.name: %s\n", field->var.name);
5626         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5627                    field->var.idx);
5628
5629         field = field_var->val;
5630
5631         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5632         if (field->field)
5633                 seq_printf(m, "      ftrace_event_field name: %s\n",
5634                            field->field->name);
5635         else {
5636                 ret = -EINVAL;
5637                 goto out;
5638         }
5639
5640         seq_printf(m, "      type: %s\n", field->type);
5641         seq_printf(m, "      size: %u\n", field->size);
5642         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5643 out:
5644         return ret;
5645 }
5646
5647 static int hist_action_debug_show(struct seq_file *m,
5648                                   struct action_data *data, int i)
5649 {
5650         int ret = 0;
5651
5652         if (data->handler == HANDLER_ONMAX ||
5653             data->handler == HANDLER_ONCHANGE) {
5654                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5655                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5656                                             HIST_FIELD_FL_VAR_REF);
5657                 if (ret)
5658                         goto out;
5659
5660                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5661                 ret = hist_field_debug_show(m, data->track_data.track_var,
5662                                             HIST_FIELD_FL_VAR);
5663                 if (ret)
5664                         goto out;
5665         }
5666
5667         if (data->handler == HANDLER_ONMATCH) {
5668                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5669                            i, data->match_data.event_system);
5670                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5671                            i, data->match_data.event);
5672         }
5673 out:
5674         return ret;
5675 }
5676
5677 static int hist_actions_debug_show(struct seq_file *m,
5678                                    struct hist_trigger_data *hist_data)
5679 {
5680         int i, ret = 0;
5681
5682         if (hist_data->n_actions)
5683                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5684
5685         for (i = 0; i < hist_data->n_actions; i++) {
5686                 struct action_data *action = hist_data->actions[i];
5687
5688                 ret = hist_action_debug_show(m, action, i);
5689                 if (ret)
5690                         goto out;
5691         }
5692
5693         if (hist_data->n_save_vars)
5694                 seq_puts(m, "\n  save action variables (save() params):\n");
5695
5696         for (i = 0; i < hist_data->n_save_vars; i++) {
5697                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5698                 if (ret)
5699                         goto out;
5700         }
5701 out:
5702         return ret;
5703 }
5704
5705 static void hist_trigger_debug_show(struct seq_file *m,
5706                                     struct event_trigger_data *data, int n)
5707 {
5708         struct hist_trigger_data *hist_data;
5709         int i, ret;
5710
5711         if (n > 0)
5712                 seq_puts(m, "\n\n");
5713
5714         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5715         data->ops->print(m, data);
5716         seq_puts(m, "#\n\n");
5717
5718         hist_data = data->private_data;
5719
5720         seq_printf(m, "hist_data: %p\n\n", hist_data);
5721         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5722         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5723         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5724
5725         seq_puts(m, "\n  val fields:\n\n");
5726
5727         seq_puts(m, "    hist_data->fields[0]:\n");
5728         ret = hist_field_debug_show(m, hist_data->fields[0],
5729                                     HIST_FIELD_FL_HITCOUNT);
5730         if (ret)
5731                 return;
5732
5733         for (i = 1; i < hist_data->n_vals; i++) {
5734                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5735                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5736                 if (ret)
5737                         return;
5738         }
5739
5740         seq_puts(m, "\n  key fields:\n");
5741
5742         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5743                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5744                 ret = hist_field_debug_show(m, hist_data->fields[i],
5745                                             HIST_FIELD_FL_KEY);
5746                 if (ret)
5747                         return;
5748         }
5749
5750         if (hist_data->n_var_refs)
5751                 seq_puts(m, "\n  variable reference fields:\n");
5752
5753         for (i = 0; i < hist_data->n_var_refs; i++) {
5754                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5755                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5756                                             HIST_FIELD_FL_VAR_REF);
5757                 if (ret)
5758                         return;
5759         }
5760
5761         if (hist_data->n_field_vars)
5762                 seq_puts(m, "\n  field variables:\n");
5763
5764         for (i = 0; i < hist_data->n_field_vars; i++) {
5765                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5766                 if (ret)
5767                         return;
5768         }
5769
5770         ret = hist_actions_debug_show(m, hist_data);
5771         if (ret)
5772                 return;
5773 }
5774
5775 static int hist_debug_show(struct seq_file *m, void *v)
5776 {
5777         struct event_trigger_data *data;
5778         struct trace_event_file *event_file;
5779         int n = 0, ret = 0;
5780
5781         mutex_lock(&event_mutex);
5782
5783         event_file = event_file_data(m->private);
5784         if (unlikely(!event_file)) {
5785                 ret = -ENODEV;
5786                 goto out_unlock;
5787         }
5788
5789         list_for_each_entry(data, &event_file->triggers, list) {
5790                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5791                         hist_trigger_debug_show(m, data, n++);
5792         }
5793
5794  out_unlock:
5795         mutex_unlock(&event_mutex);
5796
5797         return ret;
5798 }
5799
5800 static int event_hist_debug_open(struct inode *inode, struct file *file)
5801 {
5802         int ret;
5803
5804         ret = security_locked_down(LOCKDOWN_TRACEFS);
5805         if (ret)
5806                 return ret;
5807
5808         return single_open(file, hist_debug_show, file);
5809 }
5810
5811 const struct file_operations event_hist_debug_fops = {
5812         .open = event_hist_debug_open,
5813         .read = seq_read,
5814         .llseek = seq_lseek,
5815         .release = single_release,
5816 };
5817 #endif
5818
5819 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5820 {
5821         const char *field_name = hist_field_name(hist_field, 0);
5822
5823         if (hist_field->var.name)
5824                 seq_printf(m, "%s=", hist_field->var.name);
5825
5826         if (hist_field->flags & HIST_FIELD_FL_CPU)
5827                 seq_puts(m, "common_cpu");
5828         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5829                 seq_printf(m, "%llu", hist_field->constant);
5830         else if (field_name) {
5831                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5832                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5833                         seq_putc(m, '$');
5834                 seq_printf(m, "%s", field_name);
5835         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5836                 seq_puts(m, "common_timestamp");
5837
5838         if (hist_field->flags) {
5839                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5840                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5841                         const char *flags = get_hist_field_flags(hist_field);
5842
5843                         if (flags)
5844                                 seq_printf(m, ".%s", flags);
5845                 }
5846         }
5847         if (hist_field->buckets)
5848                 seq_printf(m, "=%ld", hist_field->buckets);
5849 }
5850
5851 static int event_hist_trigger_print(struct seq_file *m,
5852                                     struct event_trigger_data *data)
5853 {
5854         struct hist_trigger_data *hist_data = data->private_data;
5855         struct hist_field *field;
5856         bool have_var = false;
5857         unsigned int i;
5858
5859         seq_puts(m, HIST_PREFIX);
5860
5861         if (data->name)
5862                 seq_printf(m, "%s:", data->name);
5863
5864         seq_puts(m, "keys=");
5865
5866         for_each_hist_key_field(i, hist_data) {
5867                 field = hist_data->fields[i];
5868
5869                 if (i > hist_data->n_vals)
5870                         seq_puts(m, ",");
5871
5872                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5873                         seq_puts(m, "stacktrace");
5874                 else
5875                         hist_field_print(m, field);
5876         }
5877
5878         seq_puts(m, ":vals=");
5879
5880         for_each_hist_val_field(i, hist_data) {
5881                 field = hist_data->fields[i];
5882                 if (field->flags & HIST_FIELD_FL_VAR) {
5883                         have_var = true;
5884                         continue;
5885                 }
5886
5887                 if (i == HITCOUNT_IDX)
5888                         seq_puts(m, "hitcount");
5889                 else {
5890                         seq_puts(m, ",");
5891                         hist_field_print(m, field);
5892                 }
5893         }
5894
5895         if (have_var) {
5896                 unsigned int n = 0;
5897
5898                 seq_puts(m, ":");
5899
5900                 for_each_hist_val_field(i, hist_data) {
5901                         field = hist_data->fields[i];
5902
5903                         if (field->flags & HIST_FIELD_FL_VAR) {
5904                                 if (n++)
5905                                         seq_puts(m, ",");
5906                                 hist_field_print(m, field);
5907                         }
5908                 }
5909         }
5910
5911         seq_puts(m, ":sort=");
5912
5913         for (i = 0; i < hist_data->n_sort_keys; i++) {
5914                 struct tracing_map_sort_key *sort_key;
5915                 unsigned int idx, first_key_idx;
5916
5917                 /* skip VAR vals */
5918                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5919
5920                 sort_key = &hist_data->sort_keys[i];
5921                 idx = sort_key->field_idx;
5922
5923                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5924                         return -EINVAL;
5925
5926                 if (i > 0)
5927                         seq_puts(m, ",");
5928
5929                 if (idx == HITCOUNT_IDX)
5930                         seq_puts(m, "hitcount");
5931                 else {
5932                         if (idx >= first_key_idx)
5933                                 idx += hist_data->n_vars;
5934                         hist_field_print(m, hist_data->fields[idx]);
5935                 }
5936
5937                 if (sort_key->descending)
5938                         seq_puts(m, ".descending");
5939         }
5940         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5941         if (hist_data->enable_timestamps)
5942                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5943
5944         print_actions_spec(m, hist_data);
5945
5946         if (data->filter_str)
5947                 seq_printf(m, " if %s", data->filter_str);
5948
5949         if (data->paused)
5950                 seq_puts(m, " [paused]");
5951         else
5952                 seq_puts(m, " [active]");
5953
5954         seq_putc(m, '\n');
5955
5956         return 0;
5957 }
5958
5959 static int event_hist_trigger_init(struct event_trigger_data *data)
5960 {
5961         struct hist_trigger_data *hist_data = data->private_data;
5962
5963         if (!data->ref && hist_data->attrs->name)
5964                 save_named_trigger(hist_data->attrs->name, data);
5965
5966         data->ref++;
5967
5968         return 0;
5969 }
5970
5971 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5972 {
5973         struct trace_event_file *file;
5974         unsigned int i;
5975         char *cmd;
5976         int ret;
5977
5978         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5979                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5980                 cmd = hist_data->field_var_hists[i]->cmd;
5981                 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
5982                                                "!hist", "hist", cmd);
5983                 WARN_ON_ONCE(ret < 0);
5984         }
5985 }
5986
5987 static void event_hist_trigger_free(struct event_trigger_data *data)
5988 {
5989         struct hist_trigger_data *hist_data = data->private_data;
5990
5991         if (WARN_ON_ONCE(data->ref <= 0))
5992                 return;
5993
5994         data->ref--;
5995         if (!data->ref) {
5996                 if (data->name)
5997                         del_named_trigger(data);
5998
5999                 trigger_data_free(data);
6000
6001                 remove_hist_vars(hist_data);
6002
6003                 unregister_field_var_hists(hist_data);
6004
6005                 destroy_hist_data(hist_data);
6006         }
6007 }
6008
6009 static struct event_trigger_ops event_hist_trigger_ops = {
6010         .trigger                = event_hist_trigger,
6011         .print                  = event_hist_trigger_print,
6012         .init                   = event_hist_trigger_init,
6013         .free                   = event_hist_trigger_free,
6014 };
6015
6016 static int event_hist_trigger_named_init(struct event_trigger_data *data)
6017 {
6018         data->ref++;
6019
6020         save_named_trigger(data->named_data->name, data);
6021
6022         event_hist_trigger_init(data->named_data);
6023
6024         return 0;
6025 }
6026
6027 static void event_hist_trigger_named_free(struct event_trigger_data *data)
6028 {
6029         if (WARN_ON_ONCE(data->ref <= 0))
6030                 return;
6031
6032         event_hist_trigger_free(data->named_data);
6033
6034         data->ref--;
6035         if (!data->ref) {
6036                 del_named_trigger(data);
6037                 trigger_data_free(data);
6038         }
6039 }
6040
6041 static struct event_trigger_ops event_hist_trigger_named_ops = {
6042         .trigger                = event_hist_trigger,
6043         .print                  = event_hist_trigger_print,
6044         .init                   = event_hist_trigger_named_init,
6045         .free                   = event_hist_trigger_named_free,
6046 };
6047
6048 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6049                                                             char *param)
6050 {
6051         return &event_hist_trigger_ops;
6052 }
6053
6054 static void hist_clear(struct event_trigger_data *data)
6055 {
6056         struct hist_trigger_data *hist_data = data->private_data;
6057
6058         if (data->name)
6059                 pause_named_trigger(data);
6060
6061         tracepoint_synchronize_unregister();
6062
6063         tracing_map_clear(hist_data->map);
6064
6065         if (data->name)
6066                 unpause_named_trigger(data);
6067 }
6068
6069 static bool compatible_field(struct ftrace_event_field *field,
6070                              struct ftrace_event_field *test_field)
6071 {
6072         if (field == test_field)
6073                 return true;
6074         if (field == NULL || test_field == NULL)
6075                 return false;
6076         if (strcmp(field->name, test_field->name) != 0)
6077                 return false;
6078         if (strcmp(field->type, test_field->type) != 0)
6079                 return false;
6080         if (field->size != test_field->size)
6081                 return false;
6082         if (field->is_signed != test_field->is_signed)
6083                 return false;
6084
6085         return true;
6086 }
6087
6088 static bool hist_trigger_match(struct event_trigger_data *data,
6089                                struct event_trigger_data *data_test,
6090                                struct event_trigger_data *named_data,
6091                                bool ignore_filter)
6092 {
6093         struct tracing_map_sort_key *sort_key, *sort_key_test;
6094         struct hist_trigger_data *hist_data, *hist_data_test;
6095         struct hist_field *key_field, *key_field_test;
6096         unsigned int i;
6097
6098         if (named_data && (named_data != data_test) &&
6099             (named_data != data_test->named_data))
6100                 return false;
6101
6102         if (!named_data && is_named_trigger(data_test))
6103                 return false;
6104
6105         hist_data = data->private_data;
6106         hist_data_test = data_test->private_data;
6107
6108         if (hist_data->n_vals != hist_data_test->n_vals ||
6109             hist_data->n_fields != hist_data_test->n_fields ||
6110             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6111                 return false;
6112
6113         if (!ignore_filter) {
6114                 if ((data->filter_str && !data_test->filter_str) ||
6115                    (!data->filter_str && data_test->filter_str))
6116                         return false;
6117         }
6118
6119         for_each_hist_field(i, hist_data) {
6120                 key_field = hist_data->fields[i];
6121                 key_field_test = hist_data_test->fields[i];
6122
6123                 if (key_field->flags != key_field_test->flags)
6124                         return false;
6125                 if (!compatible_field(key_field->field, key_field_test->field))
6126                         return false;
6127                 if (key_field->offset != key_field_test->offset)
6128                         return false;
6129                 if (key_field->size != key_field_test->size)
6130                         return false;
6131                 if (key_field->is_signed != key_field_test->is_signed)
6132                         return false;
6133                 if (!!key_field->var.name != !!key_field_test->var.name)
6134                         return false;
6135                 if (key_field->var.name &&
6136                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
6137                         return false;
6138         }
6139
6140         for (i = 0; i < hist_data->n_sort_keys; i++) {
6141                 sort_key = &hist_data->sort_keys[i];
6142                 sort_key_test = &hist_data_test->sort_keys[i];
6143
6144                 if (sort_key->field_idx != sort_key_test->field_idx ||
6145                     sort_key->descending != sort_key_test->descending)
6146                         return false;
6147         }
6148
6149         if (!ignore_filter && data->filter_str &&
6150             (strcmp(data->filter_str, data_test->filter_str) != 0))
6151                 return false;
6152
6153         if (!actions_match(hist_data, hist_data_test))
6154                 return false;
6155
6156         return true;
6157 }
6158
6159 static bool existing_hist_update_only(char *glob,
6160                                       struct event_trigger_data *data,
6161                                       struct trace_event_file *file)
6162 {
6163         struct hist_trigger_data *hist_data = data->private_data;
6164         struct event_trigger_data *test, *named_data = NULL;
6165         bool updated = false;
6166
6167         if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6168             !hist_data->attrs->clear)
6169                 goto out;
6170
6171         if (hist_data->attrs->name) {
6172                 named_data = find_named_trigger(hist_data->attrs->name);
6173                 if (named_data) {
6174                         if (!hist_trigger_match(data, named_data, named_data,
6175                                                 true))
6176                                 goto out;
6177                 }
6178         }
6179
6180         if (hist_data->attrs->name && !named_data)
6181                 goto out;
6182
6183         list_for_each_entry(test, &file->triggers, list) {
6184                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6185                         if (!hist_trigger_match(data, test, named_data, false))
6186                                 continue;
6187                         if (hist_data->attrs->pause)
6188                                 test->paused = true;
6189                         else if (hist_data->attrs->cont)
6190                                 test->paused = false;
6191                         else if (hist_data->attrs->clear)
6192                                 hist_clear(test);
6193                         updated = true;
6194                         goto out;
6195                 }
6196         }
6197  out:
6198         return updated;
6199 }
6200
6201 static int hist_register_trigger(char *glob,
6202                                  struct event_trigger_data *data,
6203                                  struct trace_event_file *file)
6204 {
6205         struct hist_trigger_data *hist_data = data->private_data;
6206         struct event_trigger_data *test, *named_data = NULL;
6207         struct trace_array *tr = file->tr;
6208         int ret = 0;
6209
6210         if (hist_data->attrs->name) {
6211                 named_data = find_named_trigger(hist_data->attrs->name);
6212                 if (named_data) {
6213                         if (!hist_trigger_match(data, named_data, named_data,
6214                                                 true)) {
6215                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6216                                 ret = -EINVAL;
6217                                 goto out;
6218                         }
6219                 }
6220         }
6221
6222         if (hist_data->attrs->name && !named_data)
6223                 goto new;
6224
6225         lockdep_assert_held(&event_mutex);
6226
6227         list_for_each_entry(test, &file->triggers, list) {
6228                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6229                         if (hist_trigger_match(data, test, named_data, false)) {
6230                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6231                                 ret = -EEXIST;
6232                                 goto out;
6233                         }
6234                 }
6235         }
6236  new:
6237         if (hist_data->attrs->cont || hist_data->attrs->clear) {
6238                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6239                 ret = -ENOENT;
6240                 goto out;
6241         }
6242
6243         if (hist_data->attrs->pause)
6244                 data->paused = true;
6245
6246         if (named_data) {
6247                 data->private_data = named_data->private_data;
6248                 set_named_trigger_data(data, named_data);
6249                 data->ops = &event_hist_trigger_named_ops;
6250         }
6251
6252         if (data->ops->init) {
6253                 ret = data->ops->init(data);
6254                 if (ret < 0)
6255                         goto out;
6256         }
6257
6258         if (hist_data->enable_timestamps) {
6259                 char *clock = hist_data->attrs->clock;
6260
6261                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6262                 if (ret) {
6263                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6264                         goto out;
6265                 }
6266
6267                 tracing_set_filter_buffering(file->tr, true);
6268         }
6269
6270         if (named_data)
6271                 destroy_hist_data(hist_data);
6272  out:
6273         return ret;
6274 }
6275
6276 static int hist_trigger_enable(struct event_trigger_data *data,
6277                                struct trace_event_file *file)
6278 {
6279         int ret = 0;
6280
6281         list_add_tail_rcu(&data->list, &file->triggers);
6282
6283         update_cond_flag(file);
6284
6285         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6286                 list_del_rcu(&data->list);
6287                 update_cond_flag(file);
6288                 ret--;
6289         }
6290
6291         return ret;
6292 }
6293
6294 static bool have_hist_trigger_match(struct event_trigger_data *data,
6295                                     struct trace_event_file *file)
6296 {
6297         struct hist_trigger_data *hist_data = data->private_data;
6298         struct event_trigger_data *test, *named_data = NULL;
6299         bool match = false;
6300
6301         lockdep_assert_held(&event_mutex);
6302
6303         if (hist_data->attrs->name)
6304                 named_data = find_named_trigger(hist_data->attrs->name);
6305
6306         list_for_each_entry(test, &file->triggers, list) {
6307                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6308                         if (hist_trigger_match(data, test, named_data, false)) {
6309                                 match = true;
6310                                 break;
6311                         }
6312                 }
6313         }
6314
6315         return match;
6316 }
6317
6318 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6319                                     struct trace_event_file *file)
6320 {
6321         struct hist_trigger_data *hist_data = data->private_data;
6322         struct event_trigger_data *test, *named_data = NULL;
6323
6324         lockdep_assert_held(&event_mutex);
6325
6326         if (hist_data->attrs->name)
6327                 named_data = find_named_trigger(hist_data->attrs->name);
6328
6329         list_for_each_entry(test, &file->triggers, list) {
6330                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6331                         if (!hist_trigger_match(data, test, named_data, false))
6332                                 continue;
6333                         hist_data = test->private_data;
6334                         if (check_var_refs(hist_data))
6335                                 return true;
6336                         break;
6337                 }
6338         }
6339
6340         return false;
6341 }
6342
6343 static void hist_unregister_trigger(char *glob,
6344                                     struct event_trigger_data *data,
6345                                     struct trace_event_file *file)
6346 {
6347         struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6348         struct hist_trigger_data *hist_data = data->private_data;
6349
6350         lockdep_assert_held(&event_mutex);
6351
6352         if (hist_data->attrs->name)
6353                 named_data = find_named_trigger(hist_data->attrs->name);
6354
6355         list_for_each_entry(iter, &file->triggers, list) {
6356                 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6357                         if (!hist_trigger_match(data, iter, named_data, false))
6358                                 continue;
6359                         test = iter;
6360                         list_del_rcu(&test->list);
6361                         trace_event_trigger_enable_disable(file, 0);
6362                         update_cond_flag(file);
6363                         break;
6364                 }
6365         }
6366
6367         if (test && test->ops->free)
6368                 test->ops->free(test);
6369
6370         if (hist_data->enable_timestamps) {
6371                 if (!hist_data->remove || test)
6372                         tracing_set_filter_buffering(file->tr, false);
6373         }
6374 }
6375
6376 static bool hist_file_check_refs(struct trace_event_file *file)
6377 {
6378         struct hist_trigger_data *hist_data;
6379         struct event_trigger_data *test;
6380
6381         lockdep_assert_held(&event_mutex);
6382
6383         list_for_each_entry(test, &file->triggers, list) {
6384                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6385                         hist_data = test->private_data;
6386                         if (check_var_refs(hist_data))
6387                                 return true;
6388                 }
6389         }
6390
6391         return false;
6392 }
6393
6394 static void hist_unreg_all(struct trace_event_file *file)
6395 {
6396         struct event_trigger_data *test, *n;
6397         struct hist_trigger_data *hist_data;
6398         struct synth_event *se;
6399         const char *se_name;
6400
6401         lockdep_assert_held(&event_mutex);
6402
6403         if (hist_file_check_refs(file))
6404                 return;
6405
6406         list_for_each_entry_safe(test, n, &file->triggers, list) {
6407                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6408                         hist_data = test->private_data;
6409                         list_del_rcu(&test->list);
6410                         trace_event_trigger_enable_disable(file, 0);
6411
6412                         se_name = trace_event_name(file->event_call);
6413                         se = find_synth_event(se_name);
6414                         if (se)
6415                                 se->ref--;
6416
6417                         update_cond_flag(file);
6418                         if (hist_data->enable_timestamps)
6419                                 tracing_set_filter_buffering(file->tr, false);
6420                         if (test->ops->free)
6421                                 test->ops->free(test);
6422                 }
6423         }
6424 }
6425
6426 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6427                                     struct trace_event_file *file,
6428                                     char *glob, char *cmd,
6429                                     char *param_and_filter)
6430 {
6431         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6432         struct event_trigger_data *trigger_data;
6433         struct hist_trigger_attrs *attrs;
6434         struct hist_trigger_data *hist_data;
6435         char *param, *filter, *p, *start;
6436         struct synth_event *se;
6437         const char *se_name;
6438         bool remove;
6439         int ret = 0;
6440
6441         lockdep_assert_held(&event_mutex);
6442
6443         if (WARN_ON(!glob))
6444                 return -EINVAL;
6445
6446         if (glob[0]) {
6447                 hist_err_clear();
6448                 last_cmd_set(file, param_and_filter);
6449         }
6450
6451         remove = event_trigger_check_remove(glob);
6452
6453         if (event_trigger_empty_param(param_and_filter))
6454                 return -EINVAL;
6455
6456         /*
6457          * separate the trigger from the filter (k:v [if filter])
6458          * allowing for whitespace in the trigger
6459          */
6460         p = param = param_and_filter;
6461         do {
6462                 p = strstr(p, "if");
6463                 if (!p)
6464                         break;
6465                 if (p == param_and_filter)
6466                         return -EINVAL;
6467                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6468                         p++;
6469                         continue;
6470                 }
6471                 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6472                         return -EINVAL;
6473                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6474                         p++;
6475                         continue;
6476                 }
6477                 break;
6478         } while (1);
6479
6480         if (!p)
6481                 filter = NULL;
6482         else {
6483                 *(p - 1) = '\0';
6484                 filter = strstrip(p);
6485                 param = strstrip(param);
6486         }
6487
6488         /*
6489          * To simplify arithmetic expression parsing, replace occurrences of
6490          * '.sym-offset' modifier with '.symXoffset'
6491          */
6492         start = strstr(param, ".sym-offset");
6493         while (start) {
6494                 *(start + 4) = 'X';
6495                 start = strstr(start + 11, ".sym-offset");
6496         }
6497
6498         attrs = parse_hist_trigger_attrs(file->tr, param);
6499         if (IS_ERR(attrs))
6500                 return PTR_ERR(attrs);
6501
6502         if (attrs->map_bits)
6503                 hist_trigger_bits = attrs->map_bits;
6504
6505         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6506         if (IS_ERR(hist_data)) {
6507                 destroy_hist_trigger_attrs(attrs);
6508                 return PTR_ERR(hist_data);
6509         }
6510
6511         trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data);
6512         if (!trigger_data) {
6513                 ret = -ENOMEM;
6514                 goto out_free;
6515         }
6516
6517         ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6518         if (ret < 0)
6519                 goto out_free;
6520
6521         if (remove) {
6522                 if (!have_hist_trigger_match(trigger_data, file))
6523                         goto out_free;
6524
6525                 if (hist_trigger_check_refs(trigger_data, file)) {
6526                         ret = -EBUSY;
6527                         goto out_free;
6528                 }
6529
6530                 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6531                 se_name = trace_event_name(file->event_call);
6532                 se = find_synth_event(se_name);
6533                 if (se)
6534                         se->ref--;
6535                 ret = 0;
6536                 goto out_free;
6537         }
6538
6539         if (existing_hist_update_only(glob, trigger_data, file))
6540                 goto out_free;
6541
6542         ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6543         if (ret < 0)
6544                 goto out_free;
6545
6546         if (get_named_trigger_data(trigger_data))
6547                 goto enable;
6548
6549         if (has_hist_vars(hist_data))
6550                 save_hist_vars(hist_data);
6551
6552         ret = create_actions(hist_data);
6553         if (ret)
6554                 goto out_unreg;
6555
6556         ret = tracing_map_init(hist_data->map);
6557         if (ret)
6558                 goto out_unreg;
6559 enable:
6560         ret = hist_trigger_enable(trigger_data, file);
6561         if (ret)
6562                 goto out_unreg;
6563
6564         se_name = trace_event_name(file->event_call);
6565         se = find_synth_event(se_name);
6566         if (se)
6567                 se->ref++;
6568  out:
6569         if (ret == 0 && glob[0])
6570                 hist_err_clear();
6571
6572         return ret;
6573  out_unreg:
6574         event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6575  out_free:
6576         event_trigger_reset_filter(cmd_ops, trigger_data);
6577
6578         remove_hist_vars(hist_data);
6579
6580         kfree(trigger_data);
6581
6582         destroy_hist_data(hist_data);
6583         goto out;
6584 }
6585
6586 static struct event_command trigger_hist_cmd = {
6587         .name                   = "hist",
6588         .trigger_type           = ETT_EVENT_HIST,
6589         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6590         .parse                  = event_hist_trigger_parse,
6591         .reg                    = hist_register_trigger,
6592         .unreg                  = hist_unregister_trigger,
6593         .unreg_all              = hist_unreg_all,
6594         .get_trigger_ops        = event_hist_get_trigger_ops,
6595         .set_filter             = set_trigger_filter,
6596 };
6597
6598 __init int register_trigger_hist_cmd(void)
6599 {
6600         int ret;
6601
6602         ret = register_event_command(&trigger_hist_cmd);
6603         WARN_ON(ret < 0);
6604
6605         return ret;
6606 }
6607
6608 static void
6609 hist_enable_trigger(struct event_trigger_data *data,
6610                     struct trace_buffer *buffer,  void *rec,
6611                     struct ring_buffer_event *event)
6612 {
6613         struct enable_trigger_data *enable_data = data->private_data;
6614         struct event_trigger_data *test;
6615
6616         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6617                                 lockdep_is_held(&event_mutex)) {
6618                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6619                         if (enable_data->enable)
6620                                 test->paused = false;
6621                         else
6622                                 test->paused = true;
6623                 }
6624         }
6625 }
6626
6627 static void
6628 hist_enable_count_trigger(struct event_trigger_data *data,
6629                           struct trace_buffer *buffer,  void *rec,
6630                           struct ring_buffer_event *event)
6631 {
6632         if (!data->count)
6633                 return;
6634
6635         if (data->count != -1)
6636                 (data->count)--;
6637
6638         hist_enable_trigger(data, buffer, rec, event);
6639 }
6640
6641 static struct event_trigger_ops hist_enable_trigger_ops = {
6642         .trigger                = hist_enable_trigger,
6643         .print                  = event_enable_trigger_print,
6644         .init                   = event_trigger_init,
6645         .free                   = event_enable_trigger_free,
6646 };
6647
6648 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6649         .trigger                = hist_enable_count_trigger,
6650         .print                  = event_enable_trigger_print,
6651         .init                   = event_trigger_init,
6652         .free                   = event_enable_trigger_free,
6653 };
6654
6655 static struct event_trigger_ops hist_disable_trigger_ops = {
6656         .trigger                = hist_enable_trigger,
6657         .print                  = event_enable_trigger_print,
6658         .init                   = event_trigger_init,
6659         .free                   = event_enable_trigger_free,
6660 };
6661
6662 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6663         .trigger                = hist_enable_count_trigger,
6664         .print                  = event_enable_trigger_print,
6665         .init                   = event_trigger_init,
6666         .free                   = event_enable_trigger_free,
6667 };
6668
6669 static struct event_trigger_ops *
6670 hist_enable_get_trigger_ops(char *cmd, char *param)
6671 {
6672         struct event_trigger_ops *ops;
6673         bool enable;
6674
6675         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6676
6677         if (enable)
6678                 ops = param ? &hist_enable_count_trigger_ops :
6679                         &hist_enable_trigger_ops;
6680         else
6681                 ops = param ? &hist_disable_count_trigger_ops :
6682                         &hist_disable_trigger_ops;
6683
6684         return ops;
6685 }
6686
6687 static void hist_enable_unreg_all(struct trace_event_file *file)
6688 {
6689         struct event_trigger_data *test, *n;
6690
6691         list_for_each_entry_safe(test, n, &file->triggers, list) {
6692                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6693                         list_del_rcu(&test->list);
6694                         update_cond_flag(file);
6695                         trace_event_trigger_enable_disable(file, 0);
6696                         if (test->ops->free)
6697                                 test->ops->free(test);
6698                 }
6699         }
6700 }
6701
6702 static struct event_command trigger_hist_enable_cmd = {
6703         .name                   = ENABLE_HIST_STR,
6704         .trigger_type           = ETT_HIST_ENABLE,
6705         .parse                  = event_enable_trigger_parse,
6706         .reg                    = event_enable_register_trigger,
6707         .unreg                  = event_enable_unregister_trigger,
6708         .unreg_all              = hist_enable_unreg_all,
6709         .get_trigger_ops        = hist_enable_get_trigger_ops,
6710         .set_filter             = set_trigger_filter,
6711 };
6712
6713 static struct event_command trigger_hist_disable_cmd = {
6714         .name                   = DISABLE_HIST_STR,
6715         .trigger_type           = ETT_HIST_ENABLE,
6716         .parse                  = event_enable_trigger_parse,
6717         .reg                    = event_enable_register_trigger,
6718         .unreg                  = event_enable_unregister_trigger,
6719         .unreg_all              = hist_enable_unreg_all,
6720         .get_trigger_ops        = hist_enable_get_trigger_ops,
6721         .set_filter             = set_trigger_filter,
6722 };
6723
6724 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6725 {
6726         unregister_event_command(&trigger_hist_enable_cmd);
6727         unregister_event_command(&trigger_hist_disable_cmd);
6728 }
6729
6730 __init int register_trigger_hist_enable_disable_cmds(void)
6731 {
6732         int ret;
6733
6734         ret = register_event_command(&trigger_hist_enable_cmd);
6735         if (WARN_ON(ret < 0))
6736                 return ret;
6737         ret = register_event_command(&trigger_hist_disable_cmd);
6738         if (WARN_ON(ret < 0))
6739                 unregister_trigger_hist_enable_disable_cmds();
6740
6741         return ret;
6742 }