kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),
  73
  74 #undef C
  75 #define C(a, b)         HIST_ERR_##a
  76
  77 enum { ERRORS };
  78
  79 #undef C
  80 #define C(a, b)         b
  81
  82 static const char *err_text[] = { ERRORS };
  83
  84 struct hist_field;
  85
  86 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  87                                 struct tracing_map_elt *elt,
  88                                 struct trace_buffer *buffer,
  89                                 struct ring_buffer_event *rbe,
  90                                 void *event);
  91
  92 #define HIST_FIELD_OPERANDS_MAX 2
  93 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  94 #define HIST_ACTIONS_MAX        8
  95 #define HIST_CONST_DIGITS_MAX   21
  96 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  97
  98 enum field_op_id {
  99         FIELD_OP_NONE,
 100         FIELD_OP_PLUS,
 101         FIELD_OP_MINUS,
 102         FIELD_OP_UNARY_MINUS,
 103         FIELD_OP_DIV,
 104         FIELD_OP_MULT,
 105 };
 106
 107 enum hist_field_fn {
 108         HIST_FIELD_FN_NOP,
 109         HIST_FIELD_FN_VAR_REF,
 110         HIST_FIELD_FN_COUNTER,
 111         HIST_FIELD_FN_CONST,
 112         HIST_FIELD_FN_LOG2,
 113         HIST_FIELD_FN_BUCKET,
 114         HIST_FIELD_FN_TIMESTAMP,
 115         HIST_FIELD_FN_CPU,
 116         HIST_FIELD_FN_STRING,
 117         HIST_FIELD_FN_DYNSTRING,
 118         HIST_FIELD_FN_RELDYNSTRING,
 119         HIST_FIELD_FN_PSTRING,
 120         HIST_FIELD_FN_S64,
 121         HIST_FIELD_FN_U64,
 122         HIST_FIELD_FN_S32,
 123         HIST_FIELD_FN_U32,
 124         HIST_FIELD_FN_S16,
 125         HIST_FIELD_FN_U16,
 126         HIST_FIELD_FN_S8,
 127         HIST_FIELD_FN_U8,
 128         HIST_FIELD_FN_UMINUS,
 129         HIST_FIELD_FN_MINUS,
 130         HIST_FIELD_FN_PLUS,
 131         HIST_FIELD_FN_DIV,
 132         HIST_FIELD_FN_MULT,
 133         HIST_FIELD_FN_DIV_POWER2,
 134         HIST_FIELD_FN_DIV_NOT_POWER2,
 135         HIST_FIELD_FN_DIV_MULT_SHIFT,
 136         HIST_FIELD_FN_EXECNAME,
 137 };
 138
 139 /*
 140  * A hist_var (histogram variable) contains variable information for
 141  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 142  * flag set.  A hist_var has a variable name e.g. ts0, and is
 143  * associated with a given histogram trigger, as specified by
 144  * hist_data.  The hist_var idx is the unique index assigned to the
 145  * variable by the hist trigger's tracing_map.  The idx is what is
 146  * used to set a variable's value and, by a variable reference, to
 147  * retrieve it.
 148  */
 149 struct hist_var {
 150         char                            *name;
 151         struct hist_trigger_data        *hist_data;
 152         unsigned int                    idx;
 153 };
 154
 155 struct hist_field {
 156         struct ftrace_event_field       *field;
 157         unsigned long                   flags;
 158         unsigned long                   buckets;
 159         const char                      *type;
 160         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 161         struct hist_trigger_data        *hist_data;
 162         enum hist_field_fn              fn_num;
 163         unsigned int                    ref;
 164         unsigned int                    size;
 165         unsigned int                    offset;
 166         unsigned int                    is_signed;
 167
 168         /*
 169          * Variable fields contain variable-specific info in var.
 170          */
 171         struct hist_var                 var;
 172         enum field_op_id                operator;
 173         char                            *system;
 174         char                            *event_name;
 175
 176         /*
 177          * The name field is used for EXPR and VAR_REF fields.  VAR
 178          * fields contain the variable name in var.name.
 179          */
 180         char                            *name;
 181
 182         /*
 183          * When a histogram trigger is hit, if it has any references
 184          * to variables, the values of those variables are collected
 185          * into a var_ref_vals array by resolve_var_refs().  The
 186          * current value of each variable is read from the tracing_map
 187          * using the hist field's hist_var.idx and entered into the
 188          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 189          */
 190         unsigned int                    var_ref_idx;
 191         bool                            read_once;
 192
 193         unsigned int                    var_str_idx;
 194
 195         /* Numeric literals are represented as u64 */
 196         u64                             constant;
 197         /* Used to optimize division by constants */
 198         u64                             div_multiplier;
 199 };
 200
 201 static u64 hist_fn_call(struct hist_field *hist_field,
 202                         struct tracing_map_elt *elt,
 203                         struct trace_buffer *buffer,
 204                         struct ring_buffer_event *rbe,
 205                         void *event);
 206
 207 static u64 hist_field_const(struct hist_field *field,
 208                            struct tracing_map_elt *elt,
 209                            struct trace_buffer *buffer,
 210                            struct ring_buffer_event *rbe,
 211                            void *event)
 212 {
 213         return field->constant;
 214 }
 215
 216 static u64 hist_field_counter(struct hist_field *field,
 217                               struct tracing_map_elt *elt,
 218                               struct trace_buffer *buffer,
 219                               struct ring_buffer_event *rbe,
 220                               void *event)
 221 {
 222         return 1;
 223 }
 224
 225 static u64 hist_field_string(struct hist_field *hist_field,
 226                              struct tracing_map_elt *elt,
 227                              struct trace_buffer *buffer,
 228                              struct ring_buffer_event *rbe,
 229                              void *event)
 230 {
 231         char *addr = (char *)(event + hist_field->field->offset);
 232
 233         return (u64)(unsigned long)addr;
 234 }
 235
 236 static u64 hist_field_dynstring(struct hist_field *hist_field,
 237                                 struct tracing_map_elt *elt,
 238                                 struct trace_buffer *buffer,
 239                                 struct ring_buffer_event *rbe,
 240                                 void *event)
 241 {
 242         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 243         int str_loc = str_item & 0xffff;
 244         char *addr = (char *)(event + str_loc);
 245
 246         return (u64)(unsigned long)addr;
 247 }
 248
 249 static u64 hist_field_reldynstring(struct hist_field *hist_field,
 250                                    struct tracing_map_elt *elt,
 251                                    struct trace_buffer *buffer,
 252                                    struct ring_buffer_event *rbe,
 253                                    void *event)
 254 {
 255         u32 *item = event + hist_field->field->offset;
 256         u32 str_item = *item;
 257         int str_loc = str_item & 0xffff;
 258         char *addr = (char *)&item[1] + str_loc;
 259
 260         return (u64)(unsigned long)addr;
 261 }
 262
 263 static u64 hist_field_pstring(struct hist_field *hist_field,
 264                               struct tracing_map_elt *elt,
 265                               struct trace_buffer *buffer,
 266                               struct ring_buffer_event *rbe,
 267                               void *event)
 268 {
 269         char **addr = (char **)(event + hist_field->field->offset);
 270
 271         return (u64)(unsigned long)*addr;
 272 }
 273
 274 static u64 hist_field_log2(struct hist_field *hist_field,
 275                            struct tracing_map_elt *elt,
 276                            struct trace_buffer *buffer,
 277                            struct ring_buffer_event *rbe,
 278                            void *event)
 279 {
 280         struct hist_field *operand = hist_field->operands[0];
 281
 282         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 283
 284         return (u64) ilog2(roundup_pow_of_two(val));
 285 }
 286
 287 static u64 hist_field_bucket(struct hist_field *hist_field,
 288                              struct tracing_map_elt *elt,
 289                              struct trace_buffer *buffer,
 290                              struct ring_buffer_event *rbe,
 291                              void *event)
 292 {
 293         struct hist_field *operand = hist_field->operands[0];
 294         unsigned long buckets = hist_field->buckets;
 295
 296         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 297
 298         if (WARN_ON_ONCE(!buckets))
 299                 return val;
 300
 301         if (val >= LONG_MAX)
 302                 val = div64_ul(val, buckets);
 303         else
 304                 val = (u64)((unsigned long)val / buckets);
 305         return val * buckets;
 306 }
 307
 308 static u64 hist_field_plus(struct hist_field *hist_field,
 309                            struct tracing_map_elt *elt,
 310                            struct trace_buffer *buffer,
 311                            struct ring_buffer_event *rbe,
 312                            void *event)
 313 {
 314         struct hist_field *operand1 = hist_field->operands[0];
 315         struct hist_field *operand2 = hist_field->operands[1];
 316
 317         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 318         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 319
 320         return val1 + val2;
 321 }
 322
 323 static u64 hist_field_minus(struct hist_field *hist_field,
 324                             struct tracing_map_elt *elt,
 325                             struct trace_buffer *buffer,
 326                             struct ring_buffer_event *rbe,
 327                             void *event)
 328 {
 329         struct hist_field *operand1 = hist_field->operands[0];
 330         struct hist_field *operand2 = hist_field->operands[1];
 331
 332         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 333         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 334
 335         return val1 - val2;
 336 }
 337
 338 static u64 hist_field_div(struct hist_field *hist_field,
 339                            struct tracing_map_elt *elt,
 340                            struct trace_buffer *buffer,
 341                            struct ring_buffer_event *rbe,
 342                            void *event)
 343 {
 344         struct hist_field *operand1 = hist_field->operands[0];
 345         struct hist_field *operand2 = hist_field->operands[1];
 346
 347         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 348         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 349
 350         /* Return -1 for the undefined case */
 351         if (!val2)
 352                 return -1;
 353
 354         /* Use shift if the divisor is a power of 2 */
 355         if (!(val2 & (val2 - 1)))
 356                 return val1 >> __ffs64(val2);
 357
 358         return div64_u64(val1, val2);
 359 }
 360
 361 static u64 div_by_power_of_two(struct hist_field *hist_field,
 362                                 struct tracing_map_elt *elt,
 363                                 struct trace_buffer *buffer,
 364                                 struct ring_buffer_event *rbe,
 365                                 void *event)
 366 {
 367         struct hist_field *operand1 = hist_field->operands[0];
 368         struct hist_field *operand2 = hist_field->operands[1];
 369
 370         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 371
 372         return val1 >> __ffs64(operand2->constant);
 373 }
 374
 375 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 376                                 struct tracing_map_elt *elt,
 377                                 struct trace_buffer *buffer,
 378                                 struct ring_buffer_event *rbe,
 379                                 void *event)
 380 {
 381         struct hist_field *operand1 = hist_field->operands[0];
 382         struct hist_field *operand2 = hist_field->operands[1];
 383
 384         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 385
 386         return div64_u64(val1, operand2->constant);
 387 }
 388
 389 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 390                                 struct tracing_map_elt *elt,
 391                                 struct trace_buffer *buffer,
 392                                 struct ring_buffer_event *rbe,
 393                                 void *event)
 394 {
 395         struct hist_field *operand1 = hist_field->operands[0];
 396         struct hist_field *operand2 = hist_field->operands[1];
 397
 398         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 399
 400         /*
 401          * If the divisor is a constant, do a multiplication and shift instead.
 402          *
 403          * Choose Z = some power of 2. If Y <= Z, then:
 404          *     X / Y = (X * (Z / Y)) / Z
 405          *
 406          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 407          *     X / Y = (X * mult) / Z
 408          *
 409          * The division by Z can be replaced by a shift since Z is a power of 2:
 410          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 411          *
 412          * As long, as X < Z the results will not be off by more than 1.
 413          */
 414         if (val1 < (1 << HIST_DIV_SHIFT)) {
 415                 u64 mult = operand2->div_multiplier;
 416
 417                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 418         }
 419
 420         return div64_u64(val1, operand2->constant);
 421 }
 422
 423 static u64 hist_field_mult(struct hist_field *hist_field,
 424                            struct tracing_map_elt *elt,
 425                            struct trace_buffer *buffer,
 426                            struct ring_buffer_event *rbe,
 427                            void *event)
 428 {
 429         struct hist_field *operand1 = hist_field->operands[0];
 430         struct hist_field *operand2 = hist_field->operands[1];
 431
 432         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 433         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 434
 435         return val1 * val2;
 436 }
 437
 438 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 439                                   struct tracing_map_elt *elt,
 440                                   struct trace_buffer *buffer,
 441                                   struct ring_buffer_event *rbe,
 442                                   void *event)
 443 {
 444         struct hist_field *operand = hist_field->operands[0];
 445
 446         s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
 447         u64 val = (u64)-sval;
 448
 449         return val;
 450 }
 451
 452 #define DEFINE_HIST_FIELD_FN(type)                                      \
 453         static u64 hist_field_##type(struct hist_field *hist_field,     \
 454                                      struct tracing_map_elt *elt,       \
 455                                      struct trace_buffer *buffer,       \
 456                                      struct ring_buffer_event *rbe,     \
 457                                      void *event)                       \
 458 {                                                                       \
 459         type *addr = (type *)(event + hist_field->field->offset);       \
 460                                                                         \
 461         return (u64)(unsigned long)*addr;                               \
 462 }
 463
 464 DEFINE_HIST_FIELD_FN(s64);
 465 DEFINE_HIST_FIELD_FN(u64);
 466 DEFINE_HIST_FIELD_FN(s32);
 467 DEFINE_HIST_FIELD_FN(u32);
 468 DEFINE_HIST_FIELD_FN(s16);
 469 DEFINE_HIST_FIELD_FN(u16);
 470 DEFINE_HIST_FIELD_FN(s8);
 471 DEFINE_HIST_FIELD_FN(u8);
 472
 473 #define for_each_hist_field(i, hist_data)       \
 474         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 475
 476 #define for_each_hist_val_field(i, hist_data)   \
 477         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 478
 479 #define for_each_hist_key_field(i, hist_data)   \
 480         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 481
 482 #define HIST_STACKTRACE_DEPTH   16
 483 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 484 #define HIST_STACKTRACE_SKIP    5
 485
 486 #define HITCOUNT_IDX            0
 487 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 488
 489 enum hist_field_flags {
 490         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 491         HIST_FIELD_FL_KEY               = 1 << 1,
 492         HIST_FIELD_FL_STRING            = 1 << 2,
 493         HIST_FIELD_FL_HEX               = 1 << 3,
 494         HIST_FIELD_FL_SYM               = 1 << 4,
 495         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 496         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 497         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 498         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 499         HIST_FIELD_FL_LOG2              = 1 << 9,
 500         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 501         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 502         HIST_FIELD_FL_VAR               = 1 << 12,
 503         HIST_FIELD_FL_EXPR              = 1 << 13,
 504         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 505         HIST_FIELD_FL_CPU               = 1 << 15,
 506         HIST_FIELD_FL_ALIAS             = 1 << 16,
 507         HIST_FIELD_FL_BUCKET            = 1 << 17,
 508         HIST_FIELD_FL_CONST             = 1 << 18,
 509         HIST_FIELD_FL_PERCENT           = 1 << 19,
 510         HIST_FIELD_FL_GRAPH             = 1 << 20,
 511 };
 512
 513 struct var_defs {
 514         unsigned int    n_vars;
 515         char            *name[TRACING_MAP_VARS_MAX];
 516         char            *expr[TRACING_MAP_VARS_MAX];
 517 };
 518
 519 struct hist_trigger_attrs {
 520         char            *keys_str;
 521         char            *vals_str;
 522         char            *sort_key_str;
 523         char            *name;
 524         char            *clock;
 525         bool            pause;
 526         bool            cont;
 527         bool            clear;
 528         bool            ts_in_usecs;
 529         unsigned int    map_bits;
 530
 531         char            *assignment_str[TRACING_MAP_VARS_MAX];
 532         unsigned int    n_assignments;
 533
 534         char            *action_str[HIST_ACTIONS_MAX];
 535         unsigned int    n_actions;
 536
 537         struct var_defs var_defs;
 538 };
 539
 540 struct field_var {
 541         struct hist_field       *var;
 542         struct hist_field       *val;
 543 };
 544
 545 struct field_var_hist {
 546         struct hist_trigger_data        *hist_data;
 547         char                            *cmd;
 548 };
 549
 550 struct hist_trigger_data {
 551         struct hist_field               *fields[HIST_FIELDS_MAX];
 552         unsigned int                    n_vals;
 553         unsigned int                    n_keys;
 554         unsigned int                    n_fields;
 555         unsigned int                    n_vars;
 556         unsigned int                    n_var_str;
 557         unsigned int                    key_size;
 558         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 559         unsigned int                    n_sort_keys;
 560         struct trace_event_file         *event_file;
 561         struct hist_trigger_attrs       *attrs;
 562         struct tracing_map              *map;
 563         bool                            enable_timestamps;
 564         bool                            remove;
 565         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 566         unsigned int                    n_var_refs;
 567
 568         struct action_data              *actions[HIST_ACTIONS_MAX];
 569         unsigned int                    n_actions;
 570
 571         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 572         unsigned int                    n_field_vars;
 573         unsigned int                    n_field_var_str;
 574         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 575         unsigned int                    n_field_var_hists;
 576
 577         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 578         unsigned int                    n_save_vars;
 579         unsigned int                    n_save_var_str;
 580 };
 581
 582 struct action_data;
 583
 584 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 585                              struct tracing_map_elt *elt,
 586                              struct trace_buffer *buffer, void *rec,
 587                              struct ring_buffer_event *rbe, void *key,
 588                              struct action_data *data, u64 *var_ref_vals);
 589
 590 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 591
 592 enum handler_id {
 593         HANDLER_ONMATCH = 1,
 594         HANDLER_ONMAX,
 595         HANDLER_ONCHANGE,
 596 };
 597
 598 enum action_id {
 599         ACTION_SAVE = 1,
 600         ACTION_TRACE,
 601         ACTION_SNAPSHOT,
 602 };
 603
 604 struct action_data {
 605         enum handler_id         handler;
 606         enum action_id          action;
 607         char                    *action_name;
 608         action_fn_t             fn;
 609
 610         unsigned int            n_params;
 611         char                    *params[SYNTH_FIELDS_MAX];
 612
 613         /*
 614          * When a histogram trigger is hit, the values of any
 615          * references to variables, including variables being passed
 616          * as parameters to synthetic events, are collected into a
 617          * var_ref_vals array.  This var_ref_idx array is an array of
 618          * indices into the var_ref_vals array, one for each synthetic
 619          * event param, and is passed to the synthetic event
 620          * invocation.
 621          */
 622         unsigned int            var_ref_idx[SYNTH_FIELDS_MAX];
 623         struct synth_event      *synth_event;
 624         bool                    use_trace_keyword;
 625         char                    *synth_event_name;
 626
 627         union {
 628                 struct {
 629                         char                    *event;
 630                         char                    *event_system;
 631                 } match_data;
 632
 633                 struct {
 634                         /*
 635                          * var_str contains the $-unstripped variable
 636                          * name referenced by var_ref, and used when
 637                          * printing the action.  Because var_ref
 638                          * creation is deferred to create_actions(),
 639                          * we need a per-action way to save it until
 640                          * then, thus var_str.
 641                          */
 642                         char                    *var_str;
 643
 644                         /*
 645                          * var_ref refers to the variable being
 646                          * tracked e.g onmax($var).
 647                          */
 648                         struct hist_field       *var_ref;
 649
 650                         /*
 651                          * track_var contains the 'invisible' tracking
 652                          * variable created to keep the current
 653                          * e.g. max value.
 654                          */
 655                         struct hist_field       *track_var;
 656
 657                         check_track_val_fn_t    check_val;
 658                         action_fn_t             save_data;
 659                 } track_data;
 660         };
 661 };
 662
 663 struct track_data {
 664         u64                             track_val;
 665         bool                            updated;
 666
 667         unsigned int                    key_len;
 668         void                            *key;
 669         struct tracing_map_elt          elt;
 670
 671         struct action_data              *action_data;
 672         struct hist_trigger_data        *hist_data;
 673 };
 674
 675 struct hist_elt_data {
 676         char *comm;
 677         u64 *var_ref_vals;
 678         char **field_var_str;
 679         int n_field_var_str;
 680 };
 681
 682 struct snapshot_context {
 683         struct tracing_map_elt  *elt;
 684         void                    *key;
 685 };
 686
 687 /*
 688  * Returns the specific division function to use if the divisor
 689  * is constant. This avoids extra branches when the trigger is hit.
 690  */
 691 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
 692 {
 693         u64 div = divisor->constant;
 694
 695         if (!(div & (div - 1)))
 696                 return HIST_FIELD_FN_DIV_POWER2;
 697
 698         /* If the divisor is too large, do a regular division */
 699         if (div > (1 << HIST_DIV_SHIFT))
 700                 return HIST_FIELD_FN_DIV_NOT_POWER2;
 701
 702         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 703         return HIST_FIELD_FN_DIV_MULT_SHIFT;
 704 }
 705
 706 static void track_data_free(struct track_data *track_data)
 707 {
 708         struct hist_elt_data *elt_data;
 709
 710         if (!track_data)
 711                 return;
 712
 713         kfree(track_data->key);
 714
 715         elt_data = track_data->elt.private_data;
 716         if (elt_data) {
 717                 kfree(elt_data->comm);
 718                 kfree(elt_data);
 719         }
 720
 721         kfree(track_data);
 722 }
 723
 724 static struct track_data *track_data_alloc(unsigned int key_len,
 725                                            struct action_data *action_data,
 726                                            struct hist_trigger_data *hist_data)
 727 {
 728         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 729         struct hist_elt_data *elt_data;
 730
 731         if (!data)
 732                 return ERR_PTR(-ENOMEM);
 733
 734         data->key = kzalloc(key_len, GFP_KERNEL);
 735         if (!data->key) {
 736                 track_data_free(data);
 737                 return ERR_PTR(-ENOMEM);
 738         }
 739
 740         data->key_len = key_len;
 741         data->action_data = action_data;
 742         data->hist_data = hist_data;
 743
 744         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 745         if (!elt_data) {
 746                 track_data_free(data);
 747                 return ERR_PTR(-ENOMEM);
 748         }
 749
 750         data->elt.private_data = elt_data;
 751
 752         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 753         if (!elt_data->comm) {
 754                 track_data_free(data);
 755                 return ERR_PTR(-ENOMEM);
 756         }
 757
 758         return data;
 759 }
 760
 761 #define HIST_PREFIX "hist:"
 762
 763 static char *last_cmd;
 764 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 765
 766 static int errpos(char *str)
 767 {
 768         if (!str || !last_cmd)
 769                 return 0;
 770
 771         return err_pos(last_cmd, str);
 772 }
 773
 774 static void last_cmd_set(struct trace_event_file *file, char *str)
 775 {
 776         const char *system = NULL, *name = NULL;
 777         struct trace_event_call *call;
 778         int len;
 779
 780         if (!str)
 781                 return;
 782
 783         /* sizeof() contains the nul byte */
 784         len = sizeof(HIST_PREFIX) + strlen(str);
 785         kfree(last_cmd);
 786         last_cmd = kzalloc(len, GFP_KERNEL);
 787         if (!last_cmd)
 788                 return;
 789
 790         strcpy(last_cmd, HIST_PREFIX);
 791         /* Again, sizeof() contains the nul byte */
 792         len -= sizeof(HIST_PREFIX);
 793         strncat(last_cmd, str, len);
 794
 795         if (file) {
 796                 call = file->event_call;
 797                 system = call->class->system;
 798                 if (system) {
 799                         name = trace_event_name(call);
 800                         if (!name)
 801                                 system = NULL;
 802                 }
 803         }
 804
 805         if (system)
 806                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
 807 }
 808
 809 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
 810 {
 811         if (!last_cmd)
 812                 return;
 813
 814         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 815                         err_type, err_pos);
 816 }
 817
 818 static void hist_err_clear(void)
 819 {
 820         if (last_cmd)
 821                 last_cmd[0] = '\0';
 822         last_cmd_loc[0] = '\0';
 823 }
 824
 825 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 826                                     unsigned int *var_ref_idx);
 827
 828 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 829                                unsigned int *var_ref_idx)
 830 {
 831         struct tracepoint *tp = event->tp;
 832
 833         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 834                 struct tracepoint_func *probe_func_ptr;
 835                 synth_probe_func_t probe_func;
 836                 void *__data;
 837
 838                 if (!(cpu_online(raw_smp_processor_id())))
 839                         return;
 840
 841                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 842                 if (probe_func_ptr) {
 843                         do {
 844                                 probe_func = probe_func_ptr->func;
 845                                 __data = probe_func_ptr->data;
 846                                 probe_func(__data, var_ref_vals, var_ref_idx);
 847                         } while ((++probe_func_ptr)->func);
 848                 }
 849         }
 850 }
 851
 852 static void action_trace(struct hist_trigger_data *hist_data,
 853                          struct tracing_map_elt *elt,
 854                          struct trace_buffer *buffer, void *rec,
 855                          struct ring_buffer_event *rbe, void *key,
 856                          struct action_data *data, u64 *var_ref_vals)
 857 {
 858         struct synth_event *event = data->synth_event;
 859
 860         trace_synth(event, var_ref_vals, data->var_ref_idx);
 861 }
 862
 863 struct hist_var_data {
 864         struct list_head list;
 865         struct hist_trigger_data *hist_data;
 866 };
 867
 868 static u64 hist_field_timestamp(struct hist_field *hist_field,
 869                                 struct tracing_map_elt *elt,
 870                                 struct trace_buffer *buffer,
 871                                 struct ring_buffer_event *rbe,
 872                                 void *event)
 873 {
 874         struct hist_trigger_data *hist_data = hist_field->hist_data;
 875         struct trace_array *tr = hist_data->event_file->tr;
 876
 877         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 878
 879         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 880                 ts = ns2usecs(ts);
 881
 882         return ts;
 883 }
 884
 885 static u64 hist_field_cpu(struct hist_field *hist_field,
 886                           struct tracing_map_elt *elt,
 887                           struct trace_buffer *buffer,
 888                           struct ring_buffer_event *rbe,
 889                           void *event)
 890 {
 891         int cpu = smp_processor_id();
 892
 893         return cpu;
 894 }
 895
 896 /**
 897  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 898  * @hist_field: The VAR_REF field to check
 899  * @var_data: The hist trigger that owns the variable
 900  * @var_idx: The trigger variable identifier
 901  *
 902  * Check the given VAR_REF field to see whether or not it references
 903  * the given variable associated with the given trigger.
 904  *
 905  * Return: The VAR_REF field if it does reference the variable, NULL if not
 906  */
 907 static struct hist_field *
 908 check_field_for_var_ref(struct hist_field *hist_field,
 909                         struct hist_trigger_data *var_data,
 910                         unsigned int var_idx)
 911 {
 912         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 913
 914         if (hist_field && hist_field->var.idx == var_idx &&
 915             hist_field->var.hist_data == var_data)
 916                 return hist_field;
 917
 918         return NULL;
 919 }
 920
 921 /**
 922  * find_var_ref - Check if a trigger has a reference to a trigger variable
 923  * @hist_data: The hist trigger that might have a reference to the variable
 924  * @var_data: The hist trigger that owns the variable
 925  * @var_idx: The trigger variable identifier
 926  *
 927  * Check the list of var_refs[] on the first hist trigger to see
 928  * whether any of them are references to the variable on the second
 929  * trigger.
 930  *
 931  * Return: The VAR_REF field referencing the variable if so, NULL if not
 932  */
 933 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 934                                        struct hist_trigger_data *var_data,
 935                                        unsigned int var_idx)
 936 {
 937         struct hist_field *hist_field;
 938         unsigned int i;
 939
 940         for (i = 0; i < hist_data->n_var_refs; i++) {
 941                 hist_field = hist_data->var_refs[i];
 942                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 943                         return hist_field;
 944         }
 945
 946         return NULL;
 947 }
 948
 949 /**
 950  * find_any_var_ref - Check if there is a reference to a given trigger variable
 951  * @hist_data: The hist trigger
 952  * @var_idx: The trigger variable identifier
 953  *
 954  * Check to see whether the given variable is currently referenced by
 955  * any other trigger.
 956  *
 957  * The trigger the variable is defined on is explicitly excluded - the
 958  * assumption being that a self-reference doesn't prevent a trigger
 959  * from being removed.
 960  *
 961  * Return: The VAR_REF field referencing the variable if so, NULL if not
 962  */
 963 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 964                                            unsigned int var_idx)
 965 {
 966         struct trace_array *tr = hist_data->event_file->tr;
 967         struct hist_field *found = NULL;
 968         struct hist_var_data *var_data;
 969
 970         list_for_each_entry(var_data, &tr->hist_vars, list) {
 971                 if (var_data->hist_data == hist_data)
 972                         continue;
 973                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 974                 if (found)
 975                         break;
 976         }
 977
 978         return found;
 979 }
 980
 981 /**
 982  * check_var_refs - Check if there is a reference to any of trigger's variables
 983  * @hist_data: The hist trigger
 984  *
 985  * A trigger can define one or more variables.  If any one of them is
 986  * currently referenced by any other trigger, this function will
 987  * determine that.
 988  *
 989  * Typically used to determine whether or not a trigger can be removed
 990  * - if there are any references to a trigger's variables, it cannot.
 991  *
 992  * Return: True if there is a reference to any of trigger's variables
 993  */
 994 static bool check_var_refs(struct hist_trigger_data *hist_data)
 995 {
 996         struct hist_field *field;
 997         bool found = false;
 998         int i;
 999
1000         for_each_hist_field(i, hist_data) {
1001                 field = hist_data->fields[i];
1002                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1003                         if (find_any_var_ref(hist_data, field->var.idx)) {
1004                                 found = true;
1005                                 break;
1006                         }
1007                 }
1008         }
1009
1010         return found;
1011 }
1012
1013 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1014 {
1015         struct trace_array *tr = hist_data->event_file->tr;
1016         struct hist_var_data *var_data, *found = NULL;
1017
1018         list_for_each_entry(var_data, &tr->hist_vars, list) {
1019                 if (var_data->hist_data == hist_data) {
1020                         found = var_data;
1021                         break;
1022                 }
1023         }
1024
1025         return found;
1026 }
1027
1028 static bool field_has_hist_vars(struct hist_field *hist_field,
1029                                 unsigned int level)
1030 {
1031         int i;
1032
1033         if (level > 3)
1034                 return false;
1035
1036         if (!hist_field)
1037                 return false;
1038
1039         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1040             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1041                 return true;
1042
1043         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1044                 struct hist_field *operand;
1045
1046                 operand = hist_field->operands[i];
1047                 if (field_has_hist_vars(operand, level + 1))
1048                         return true;
1049         }
1050
1051         return false;
1052 }
1053
1054 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1055 {
1056         struct hist_field *hist_field;
1057         int i;
1058
1059         for_each_hist_field(i, hist_data) {
1060                 hist_field = hist_data->fields[i];
1061                 if (field_has_hist_vars(hist_field, 0))
1062                         return true;
1063         }
1064
1065         return false;
1066 }
1067
1068 static int save_hist_vars(struct hist_trigger_data *hist_data)
1069 {
1070         struct trace_array *tr = hist_data->event_file->tr;
1071         struct hist_var_data *var_data;
1072
1073         var_data = find_hist_vars(hist_data);
1074         if (var_data)
1075                 return 0;
1076
1077         if (tracing_check_open_get_tr(tr))
1078                 return -ENODEV;
1079
1080         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1081         if (!var_data) {
1082                 trace_array_put(tr);
1083                 return -ENOMEM;
1084         }
1085
1086         var_data->hist_data = hist_data;
1087         list_add(&var_data->list, &tr->hist_vars);
1088
1089         return 0;
1090 }
1091
1092 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1093 {
1094         struct trace_array *tr = hist_data->event_file->tr;
1095         struct hist_var_data *var_data;
1096
1097         var_data = find_hist_vars(hist_data);
1098         if (!var_data)
1099                 return;
1100
1101         if (WARN_ON(check_var_refs(hist_data)))
1102                 return;
1103
1104         list_del(&var_data->list);
1105
1106         kfree(var_data);
1107
1108         trace_array_put(tr);
1109 }
1110
1111 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1112                                          const char *var_name)
1113 {
1114         struct hist_field *hist_field, *found = NULL;
1115         int i;
1116
1117         for_each_hist_field(i, hist_data) {
1118                 hist_field = hist_data->fields[i];
1119                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1120                     strcmp(hist_field->var.name, var_name) == 0) {
1121                         found = hist_field;
1122                         break;
1123                 }
1124         }
1125
1126         return found;
1127 }
1128
1129 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1130                                    struct trace_event_file *file,
1131                                    const char *var_name)
1132 {
1133         struct hist_trigger_data *test_data;
1134         struct event_trigger_data *test;
1135         struct hist_field *hist_field;
1136
1137         lockdep_assert_held(&event_mutex);
1138
1139         hist_field = find_var_field(hist_data, var_name);
1140         if (hist_field)
1141                 return hist_field;
1142
1143         list_for_each_entry(test, &file->triggers, list) {
1144                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1145                         test_data = test->private_data;
1146                         hist_field = find_var_field(test_data, var_name);
1147                         if (hist_field)
1148                                 return hist_field;
1149                 }
1150         }
1151
1152         return NULL;
1153 }
1154
1155 static struct trace_event_file *find_var_file(struct trace_array *tr,
1156                                               char *system,
1157                                               char *event_name,
1158                                               char *var_name)
1159 {
1160         struct hist_trigger_data *var_hist_data;
1161         struct hist_var_data *var_data;
1162         struct trace_event_file *file, *found = NULL;
1163
1164         if (system)
1165                 return find_event_file(tr, system, event_name);
1166
1167         list_for_each_entry(var_data, &tr->hist_vars, list) {
1168                 var_hist_data = var_data->hist_data;
1169                 file = var_hist_data->event_file;
1170                 if (file == found)
1171                         continue;
1172
1173                 if (find_var_field(var_hist_data, var_name)) {
1174                         if (found) {
1175                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1176                                 return NULL;
1177                         }
1178
1179                         found = file;
1180                 }
1181         }
1182
1183         return found;
1184 }
1185
1186 static struct hist_field *find_file_var(struct trace_event_file *file,
1187                                         const char *var_name)
1188 {
1189         struct hist_trigger_data *test_data;
1190         struct event_trigger_data *test;
1191         struct hist_field *hist_field;
1192
1193         lockdep_assert_held(&event_mutex);
1194
1195         list_for_each_entry(test, &file->triggers, list) {
1196                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1197                         test_data = test->private_data;
1198                         hist_field = find_var_field(test_data, var_name);
1199                         if (hist_field)
1200                                 return hist_field;
1201                 }
1202         }
1203
1204         return NULL;
1205 }
1206
1207 static struct hist_field *
1208 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1209 {
1210         struct trace_array *tr = hist_data->event_file->tr;
1211         struct hist_field *hist_field, *found = NULL;
1212         struct trace_event_file *file;
1213         unsigned int i;
1214
1215         for (i = 0; i < hist_data->n_actions; i++) {
1216                 struct action_data *data = hist_data->actions[i];
1217
1218                 if (data->handler == HANDLER_ONMATCH) {
1219                         char *system = data->match_data.event_system;
1220                         char *event_name = data->match_data.event;
1221
1222                         file = find_var_file(tr, system, event_name, var_name);
1223                         if (!file)
1224                                 continue;
1225                         hist_field = find_file_var(file, var_name);
1226                         if (hist_field) {
1227                                 if (found) {
1228                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1229                                                  errpos(var_name));
1230                                         return ERR_PTR(-EINVAL);
1231                                 }
1232
1233                                 found = hist_field;
1234                         }
1235                 }
1236         }
1237         return found;
1238 }
1239
1240 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1241                                          char *system,
1242                                          char *event_name,
1243                                          char *var_name)
1244 {
1245         struct trace_array *tr = hist_data->event_file->tr;
1246         struct hist_field *hist_field = NULL;
1247         struct trace_event_file *file;
1248
1249         if (!system || !event_name) {
1250                 hist_field = find_match_var(hist_data, var_name);
1251                 if (IS_ERR(hist_field))
1252                         return NULL;
1253                 if (hist_field)
1254                         return hist_field;
1255         }
1256
1257         file = find_var_file(tr, system, event_name, var_name);
1258         if (!file)
1259                 return NULL;
1260
1261         hist_field = find_file_var(file, var_name);
1262
1263         return hist_field;
1264 }
1265
1266 static u64 hist_field_var_ref(struct hist_field *hist_field,
1267                               struct tracing_map_elt *elt,
1268                               struct trace_buffer *buffer,
1269                               struct ring_buffer_event *rbe,
1270                               void *event)
1271 {
1272         struct hist_elt_data *elt_data;
1273         u64 var_val = 0;
1274
1275         if (WARN_ON_ONCE(!elt))
1276                 return var_val;
1277
1278         elt_data = elt->private_data;
1279         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1280
1281         return var_val;
1282 }
1283
1284 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1285                              u64 *var_ref_vals, bool self)
1286 {
1287         struct hist_trigger_data *var_data;
1288         struct tracing_map_elt *var_elt;
1289         struct hist_field *hist_field;
1290         unsigned int i, var_idx;
1291         bool resolved = true;
1292         u64 var_val = 0;
1293
1294         for (i = 0; i < hist_data->n_var_refs; i++) {
1295                 hist_field = hist_data->var_refs[i];
1296                 var_idx = hist_field->var.idx;
1297                 var_data = hist_field->var.hist_data;
1298
1299                 if (var_data == NULL) {
1300                         resolved = false;
1301                         break;
1302                 }
1303
1304                 if ((self && var_data != hist_data) ||
1305                     (!self && var_data == hist_data))
1306                         continue;
1307
1308                 var_elt = tracing_map_lookup(var_data->map, key);
1309                 if (!var_elt) {
1310                         resolved = false;
1311                         break;
1312                 }
1313
1314                 if (!tracing_map_var_set(var_elt, var_idx)) {
1315                         resolved = false;
1316                         break;
1317                 }
1318
1319                 if (self || !hist_field->read_once)
1320                         var_val = tracing_map_read_var(var_elt, var_idx);
1321                 else
1322                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1323
1324                 var_ref_vals[i] = var_val;
1325         }
1326
1327         return resolved;
1328 }
1329
1330 static const char *hist_field_name(struct hist_field *field,
1331                                    unsigned int level)
1332 {
1333         const char *field_name = "";
1334
1335         if (WARN_ON_ONCE(!field))
1336                 return field_name;
1337
1338         if (level > 1)
1339                 return field_name;
1340
1341         if (field->field)
1342                 field_name = field->field->name;
1343         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1344                  field->flags & HIST_FIELD_FL_ALIAS ||
1345                  field->flags & HIST_FIELD_FL_BUCKET)
1346                 field_name = hist_field_name(field->operands[0], ++level);
1347         else if (field->flags & HIST_FIELD_FL_CPU)
1348                 field_name = "common_cpu";
1349         else if (field->flags & HIST_FIELD_FL_EXPR ||
1350                  field->flags & HIST_FIELD_FL_VAR_REF) {
1351                 if (field->system) {
1352                         static char full_name[MAX_FILTER_STR_VAL];
1353
1354                         strcat(full_name, field->system);
1355                         strcat(full_name, ".");
1356                         strcat(full_name, field->event_name);
1357                         strcat(full_name, ".");
1358                         strcat(full_name, field->name);
1359                         field_name = full_name;
1360                 } else
1361                         field_name = field->name;
1362         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1363                 field_name = "common_timestamp";
1364
1365         if (field_name == NULL)
1366                 field_name = "";
1367
1368         return field_name;
1369 }
1370
1371 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1372 {
1373         switch (field_size) {
1374         case 8:
1375                 if (field_is_signed)
1376                         return HIST_FIELD_FN_S64;
1377                 else
1378                         return HIST_FIELD_FN_U64;
1379         case 4:
1380                 if (field_is_signed)
1381                         return HIST_FIELD_FN_S32;
1382                 else
1383                         return HIST_FIELD_FN_U32;
1384         case 2:
1385                 if (field_is_signed)
1386                         return HIST_FIELD_FN_S16;
1387                 else
1388                         return HIST_FIELD_FN_U16;
1389         case 1:
1390                 if (field_is_signed)
1391                         return HIST_FIELD_FN_S8;
1392                 else
1393                         return HIST_FIELD_FN_U8;
1394         }
1395
1396         return HIST_FIELD_FN_NOP;
1397 }
1398
1399 static int parse_map_size(char *str)
1400 {
1401         unsigned long size, map_bits;
1402         int ret;
1403
1404         ret = kstrtoul(str, 0, &size);
1405         if (ret)
1406                 goto out;
1407
1408         map_bits = ilog2(roundup_pow_of_two(size));
1409         if (map_bits < TRACING_MAP_BITS_MIN ||
1410             map_bits > TRACING_MAP_BITS_MAX)
1411                 ret = -EINVAL;
1412         else
1413                 ret = map_bits;
1414  out:
1415         return ret;
1416 }
1417
1418 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1419 {
1420         unsigned int i;
1421
1422         if (!attrs)
1423                 return;
1424
1425         for (i = 0; i < attrs->n_assignments; i++)
1426                 kfree(attrs->assignment_str[i]);
1427
1428         for (i = 0; i < attrs->n_actions; i++)
1429                 kfree(attrs->action_str[i]);
1430
1431         kfree(attrs->name);
1432         kfree(attrs->sort_key_str);
1433         kfree(attrs->keys_str);
1434         kfree(attrs->vals_str);
1435         kfree(attrs->clock);
1436         kfree(attrs);
1437 }
1438
1439 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1440 {
1441         int ret = -EINVAL;
1442
1443         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1444                 return ret;
1445
1446         if ((str_has_prefix(str, "onmatch(")) ||
1447             (str_has_prefix(str, "onmax(")) ||
1448             (str_has_prefix(str, "onchange("))) {
1449                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1450                 if (!attrs->action_str[attrs->n_actions]) {
1451                         ret = -ENOMEM;
1452                         return ret;
1453                 }
1454                 attrs->n_actions++;
1455                 ret = 0;
1456         }
1457         return ret;
1458 }
1459
1460 static int parse_assignment(struct trace_array *tr,
1461                             char *str, struct hist_trigger_attrs *attrs)
1462 {
1463         int len, ret = 0;
1464
1465         if ((len = str_has_prefix(str, "key=")) ||
1466             (len = str_has_prefix(str, "keys="))) {
1467                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1468                 if (!attrs->keys_str) {
1469                         ret = -ENOMEM;
1470                         goto out;
1471                 }
1472         } else if ((len = str_has_prefix(str, "val=")) ||
1473                    (len = str_has_prefix(str, "vals=")) ||
1474                    (len = str_has_prefix(str, "values="))) {
1475                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1476                 if (!attrs->vals_str) {
1477                         ret = -ENOMEM;
1478                         goto out;
1479                 }
1480         } else if ((len = str_has_prefix(str, "sort="))) {
1481                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1482                 if (!attrs->sort_key_str) {
1483                         ret = -ENOMEM;
1484                         goto out;
1485                 }
1486         } else if (str_has_prefix(str, "name=")) {
1487                 attrs->name = kstrdup(str, GFP_KERNEL);
1488                 if (!attrs->name) {
1489                         ret = -ENOMEM;
1490                         goto out;
1491                 }
1492         } else if ((len = str_has_prefix(str, "clock="))) {
1493                 str += len;
1494
1495                 str = strstrip(str);
1496                 attrs->clock = kstrdup(str, GFP_KERNEL);
1497                 if (!attrs->clock) {
1498                         ret = -ENOMEM;
1499                         goto out;
1500                 }
1501         } else if ((len = str_has_prefix(str, "size="))) {
1502                 int map_bits = parse_map_size(str + len);
1503
1504                 if (map_bits < 0) {
1505                         ret = map_bits;
1506                         goto out;
1507                 }
1508                 attrs->map_bits = map_bits;
1509         } else {
1510                 char *assignment;
1511
1512                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1513                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1514                         ret = -EINVAL;
1515                         goto out;
1516                 }
1517
1518                 assignment = kstrdup(str, GFP_KERNEL);
1519                 if (!assignment) {
1520                         ret = -ENOMEM;
1521                         goto out;
1522                 }
1523
1524                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1525         }
1526  out:
1527         return ret;
1528 }
1529
1530 static struct hist_trigger_attrs *
1531 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1532 {
1533         struct hist_trigger_attrs *attrs;
1534         int ret = 0;
1535
1536         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1537         if (!attrs)
1538                 return ERR_PTR(-ENOMEM);
1539
1540         while (trigger_str) {
1541                 char *str = strsep(&trigger_str, ":");
1542                 char *rhs;
1543
1544                 rhs = strchr(str, '=');
1545                 if (rhs) {
1546                         if (!strlen(++rhs)) {
1547                                 ret = -EINVAL;
1548                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1549                                 goto free;
1550                         }
1551                         ret = parse_assignment(tr, str, attrs);
1552                         if (ret)
1553                                 goto free;
1554                 } else if (strcmp(str, "pause") == 0)
1555                         attrs->pause = true;
1556                 else if ((strcmp(str, "cont") == 0) ||
1557                          (strcmp(str, "continue") == 0))
1558                         attrs->cont = true;
1559                 else if (strcmp(str, "clear") == 0)
1560                         attrs->clear = true;
1561                 else {
1562                         ret = parse_action(str, attrs);
1563                         if (ret)
1564                                 goto free;
1565                 }
1566         }
1567
1568         if (!attrs->keys_str) {
1569                 ret = -EINVAL;
1570                 goto free;
1571         }
1572
1573         if (!attrs->clock) {
1574                 attrs->clock = kstrdup("global", GFP_KERNEL);
1575                 if (!attrs->clock) {
1576                         ret = -ENOMEM;
1577                         goto free;
1578                 }
1579         }
1580
1581         return attrs;
1582  free:
1583         destroy_hist_trigger_attrs(attrs);
1584
1585         return ERR_PTR(ret);
1586 }
1587
1588 static inline void save_comm(char *comm, struct task_struct *task)
1589 {
1590         if (!task->pid) {
1591                 strcpy(comm, "<idle>");
1592                 return;
1593         }
1594
1595         if (WARN_ON_ONCE(task->pid < 0)) {
1596                 strcpy(comm, "<XXX>");
1597                 return;
1598         }
1599
1600         strncpy(comm, task->comm, TASK_COMM_LEN);
1601 }
1602
1603 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1604 {
1605         unsigned int i;
1606
1607         for (i = 0; i < elt_data->n_field_var_str; i++)
1608                 kfree(elt_data->field_var_str[i]);
1609
1610         kfree(elt_data->field_var_str);
1611
1612         kfree(elt_data->comm);
1613         kfree(elt_data);
1614 }
1615
1616 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1617 {
1618         struct hist_elt_data *elt_data = elt->private_data;
1619
1620         hist_elt_data_free(elt_data);
1621 }
1622
1623 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1624 {
1625         struct hist_trigger_data *hist_data = elt->map->private_data;
1626         unsigned int size = TASK_COMM_LEN;
1627         struct hist_elt_data *elt_data;
1628         struct hist_field *hist_field;
1629         unsigned int i, n_str;
1630
1631         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1632         if (!elt_data)
1633                 return -ENOMEM;
1634
1635         for_each_hist_field(i, hist_data) {
1636                 hist_field = hist_data->fields[i];
1637
1638                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1639                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1640                         if (!elt_data->comm) {
1641                                 kfree(elt_data);
1642                                 return -ENOMEM;
1643                         }
1644                         break;
1645                 }
1646         }
1647
1648         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1649                 hist_data->n_var_str;
1650         if (n_str > SYNTH_FIELDS_MAX) {
1651                 hist_elt_data_free(elt_data);
1652                 return -EINVAL;
1653         }
1654
1655         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1656
1657         size = STR_VAR_LEN_MAX;
1658
1659         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1660         if (!elt_data->field_var_str) {
1661                 hist_elt_data_free(elt_data);
1662                 return -EINVAL;
1663         }
1664         elt_data->n_field_var_str = n_str;
1665
1666         for (i = 0; i < n_str; i++) {
1667                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1668                 if (!elt_data->field_var_str[i]) {
1669                         hist_elt_data_free(elt_data);
1670                         return -ENOMEM;
1671                 }
1672         }
1673
1674         elt->private_data = elt_data;
1675
1676         return 0;
1677 }
1678
1679 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1680 {
1681         struct hist_elt_data *elt_data = elt->private_data;
1682
1683         if (elt_data->comm)
1684                 save_comm(elt_data->comm, current);
1685 }
1686
1687 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1688         .elt_alloc      = hist_trigger_elt_data_alloc,
1689         .elt_free       = hist_trigger_elt_data_free,
1690         .elt_init       = hist_trigger_elt_data_init,
1691 };
1692
1693 static const char *get_hist_field_flags(struct hist_field *hist_field)
1694 {
1695         const char *flags_str = NULL;
1696
1697         if (hist_field->flags & HIST_FIELD_FL_HEX)
1698                 flags_str = "hex";
1699         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1700                 flags_str = "sym";
1701         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1702                 flags_str = "sym-offset";
1703         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1704                 flags_str = "execname";
1705         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1706                 flags_str = "syscall";
1707         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1708                 flags_str = "log2";
1709         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1710                 flags_str = "buckets";
1711         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1712                 flags_str = "usecs";
1713         else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1714                 flags_str = "percent";
1715         else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1716                 flags_str = "graph";
1717
1718         return flags_str;
1719 }
1720
1721 static void expr_field_str(struct hist_field *field, char *expr)
1722 {
1723         if (field->flags & HIST_FIELD_FL_VAR_REF)
1724                 strcat(expr, "$");
1725         else if (field->flags & HIST_FIELD_FL_CONST) {
1726                 char str[HIST_CONST_DIGITS_MAX];
1727
1728                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1729                 strcat(expr, str);
1730         }
1731
1732         strcat(expr, hist_field_name(field, 0));
1733
1734         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1735                 const char *flags_str = get_hist_field_flags(field);
1736
1737                 if (flags_str) {
1738                         strcat(expr, ".");
1739                         strcat(expr, flags_str);
1740                 }
1741         }
1742 }
1743
1744 static char *expr_str(struct hist_field *field, unsigned int level)
1745 {
1746         char *expr;
1747
1748         if (level > 1)
1749                 return NULL;
1750
1751         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1752         if (!expr)
1753                 return NULL;
1754
1755         if (!field->operands[0]) {
1756                 expr_field_str(field, expr);
1757                 return expr;
1758         }
1759
1760         if (field->operator == FIELD_OP_UNARY_MINUS) {
1761                 char *subexpr;
1762
1763                 strcat(expr, "-(");
1764                 subexpr = expr_str(field->operands[0], ++level);
1765                 if (!subexpr) {
1766                         kfree(expr);
1767                         return NULL;
1768                 }
1769                 strcat(expr, subexpr);
1770                 strcat(expr, ")");
1771
1772                 kfree(subexpr);
1773
1774                 return expr;
1775         }
1776
1777         expr_field_str(field->operands[0], expr);
1778
1779         switch (field->operator) {
1780         case FIELD_OP_MINUS:
1781                 strcat(expr, "-");
1782                 break;
1783         case FIELD_OP_PLUS:
1784                 strcat(expr, "+");
1785                 break;
1786         case FIELD_OP_DIV:
1787                 strcat(expr, "/");
1788                 break;
1789         case FIELD_OP_MULT:
1790                 strcat(expr, "*");
1791                 break;
1792         default:
1793                 kfree(expr);
1794                 return NULL;
1795         }
1796
1797         expr_field_str(field->operands[1], expr);
1798
1799         return expr;
1800 }
1801
1802 /*
1803  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1804  * of the expression tree to be evaluated.
1805  */
1806 static int contains_operator(char *str, char **sep)
1807 {
1808         enum field_op_id field_op = FIELD_OP_NONE;
1809         char *minus_op, *plus_op, *div_op, *mult_op;
1810
1811
1812         /*
1813          * Report the last occurrence of the operators first, so that the
1814          * expression is evaluated left to right. This is important since
1815          * subtraction and division are not associative.
1816          *
1817          *      e.g
1818          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1819          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1820          */
1821
1822         /*
1823          * First, find lower precedence addition and subtraction
1824          * since the expression will be evaluated recursively.
1825          */
1826         minus_op = strrchr(str, '-');
1827         if (minus_op) {
1828                 /*
1829                  * Unary minus is not supported in sub-expressions. If
1830                  * present, it is always the next root operator.
1831                  */
1832                 if (minus_op == str) {
1833                         field_op = FIELD_OP_UNARY_MINUS;
1834                         goto out;
1835                 }
1836
1837                 field_op = FIELD_OP_MINUS;
1838         }
1839
1840         plus_op = strrchr(str, '+');
1841         if (plus_op || minus_op) {
1842                 /*
1843                  * For operators of the same precedence use to rightmost as the
1844                  * root, so that the expression is evaluated left to right.
1845                  */
1846                 if (plus_op > minus_op)
1847                         field_op = FIELD_OP_PLUS;
1848                 goto out;
1849         }
1850
1851         /*
1852          * Multiplication and division have higher precedence than addition and
1853          * subtraction.
1854          */
1855         div_op = strrchr(str, '/');
1856         if (div_op)
1857                 field_op = FIELD_OP_DIV;
1858
1859         mult_op = strrchr(str, '*');
1860         /*
1861          * For operators of the same precedence use to rightmost as the
1862          * root, so that the expression is evaluated left to right.
1863          */
1864         if (mult_op > div_op)
1865                 field_op = FIELD_OP_MULT;
1866
1867 out:
1868         if (sep) {
1869                 switch (field_op) {
1870                 case FIELD_OP_UNARY_MINUS:
1871                 case FIELD_OP_MINUS:
1872                         *sep = minus_op;
1873                         break;
1874                 case FIELD_OP_PLUS:
1875                         *sep = plus_op;
1876                         break;
1877                 case FIELD_OP_DIV:
1878                         *sep = div_op;
1879                         break;
1880                 case FIELD_OP_MULT:
1881                         *sep = mult_op;
1882                         break;
1883                 case FIELD_OP_NONE:
1884                 default:
1885                         *sep = NULL;
1886                         break;
1887                 }
1888         }
1889
1890         return field_op;
1891 }
1892
1893 static void get_hist_field(struct hist_field *hist_field)
1894 {
1895         hist_field->ref++;
1896 }
1897
1898 static void __destroy_hist_field(struct hist_field *hist_field)
1899 {
1900         if (--hist_field->ref > 1)
1901                 return;
1902
1903         kfree(hist_field->var.name);
1904         kfree(hist_field->name);
1905
1906         /* Can likely be a const */
1907         kfree_const(hist_field->type);
1908
1909         kfree(hist_field->system);
1910         kfree(hist_field->event_name);
1911
1912         kfree(hist_field);
1913 }
1914
1915 static void destroy_hist_field(struct hist_field *hist_field,
1916                                unsigned int level)
1917 {
1918         unsigned int i;
1919
1920         if (level > 3)
1921                 return;
1922
1923         if (!hist_field)
1924                 return;
1925
1926         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1927                 return; /* var refs will be destroyed separately */
1928
1929         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1930                 destroy_hist_field(hist_field->operands[i], level + 1);
1931
1932         __destroy_hist_field(hist_field);
1933 }
1934
1935 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1936                                             struct ftrace_event_field *field,
1937                                             unsigned long flags,
1938                                             char *var_name)
1939 {
1940         struct hist_field *hist_field;
1941
1942         if (field && is_function_field(field))
1943                 return NULL;
1944
1945         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1946         if (!hist_field)
1947                 return NULL;
1948
1949         hist_field->ref = 1;
1950
1951         hist_field->hist_data = hist_data;
1952
1953         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1954                 goto out; /* caller will populate */
1955
1956         if (flags & HIST_FIELD_FL_VAR_REF) {
1957                 hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1958                 goto out;
1959         }
1960
1961         if (flags & HIST_FIELD_FL_HITCOUNT) {
1962                 hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1963                 hist_field->size = sizeof(u64);
1964                 hist_field->type = "u64";
1965                 goto out;
1966         }
1967
1968         if (flags & HIST_FIELD_FL_CONST) {
1969                 hist_field->fn_num = HIST_FIELD_FN_CONST;
1970                 hist_field->size = sizeof(u64);
1971                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1972                 if (!hist_field->type)
1973                         goto free;
1974                 goto out;
1975         }
1976
1977         if (flags & HIST_FIELD_FL_STACKTRACE) {
1978                 hist_field->fn_num = HIST_FIELD_FN_NOP;
1979                 goto out;
1980         }
1981
1982         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1983                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1984                 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
1985                         HIST_FIELD_FN_BUCKET;
1986                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1987                 if (!hist_field->operands[0])
1988                         goto free;
1989                 hist_field->size = hist_field->operands[0]->size;
1990                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1991                 if (!hist_field->type)
1992                         goto free;
1993                 goto out;
1994         }
1995
1996         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1997                 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
1998                 hist_field->size = sizeof(u64);
1999                 hist_field->type = "u64";
2000                 goto out;
2001         }
2002
2003         if (flags & HIST_FIELD_FL_CPU) {
2004                 hist_field->fn_num = HIST_FIELD_FN_CPU;
2005                 hist_field->size = sizeof(int);
2006                 hist_field->type = "unsigned int";
2007                 goto out;
2008         }
2009
2010         if (WARN_ON_ONCE(!field))
2011                 goto out;
2012
2013         /* Pointers to strings are just pointers and dangerous to dereference */
2014         if (is_string_field(field) &&
2015             (field->filter_type != FILTER_PTR_STRING)) {
2016                 flags |= HIST_FIELD_FL_STRING;
2017
2018                 hist_field->size = MAX_FILTER_STR_VAL;
2019                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2020                 if (!hist_field->type)
2021                         goto free;
2022
2023                 if (field->filter_type == FILTER_STATIC_STRING) {
2024                         hist_field->fn_num = HIST_FIELD_FN_STRING;
2025                         hist_field->size = field->size;
2026                 } else if (field->filter_type == FILTER_DYN_STRING) {
2027                         hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2028                 } else if (field->filter_type == FILTER_RDYN_STRING)
2029                         hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2030                 else
2031                         hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2032         } else {
2033                 hist_field->size = field->size;
2034                 hist_field->is_signed = field->is_signed;
2035                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2036                 if (!hist_field->type)
2037                         goto free;
2038
2039                 hist_field->fn_num = select_value_fn(field->size,
2040                                                      field->is_signed);
2041                 if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2042                         destroy_hist_field(hist_field, 0);
2043                         return NULL;
2044                 }
2045         }
2046  out:
2047         hist_field->field = field;
2048         hist_field->flags = flags;
2049
2050         if (var_name) {
2051                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2052                 if (!hist_field->var.name)
2053                         goto free;
2054         }
2055
2056         return hist_field;
2057  free:
2058         destroy_hist_field(hist_field, 0);
2059         return NULL;
2060 }
2061
2062 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2063 {
2064         unsigned int i;
2065
2066         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2067                 if (hist_data->fields[i]) {
2068                         destroy_hist_field(hist_data->fields[i], 0);
2069                         hist_data->fields[i] = NULL;
2070                 }
2071         }
2072
2073         for (i = 0; i < hist_data->n_var_refs; i++) {
2074                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2075                 __destroy_hist_field(hist_data->var_refs[i]);
2076                 hist_data->var_refs[i] = NULL;
2077         }
2078 }
2079
2080 static int init_var_ref(struct hist_field *ref_field,
2081                         struct hist_field *var_field,
2082                         char *system, char *event_name)
2083 {
2084         int err = 0;
2085
2086         ref_field->var.idx = var_field->var.idx;
2087         ref_field->var.hist_data = var_field->hist_data;
2088         ref_field->size = var_field->size;
2089         ref_field->is_signed = var_field->is_signed;
2090         ref_field->flags |= var_field->flags &
2091                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2092
2093         if (system) {
2094                 ref_field->system = kstrdup(system, GFP_KERNEL);
2095                 if (!ref_field->system)
2096                         return -ENOMEM;
2097         }
2098
2099         if (event_name) {
2100                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2101                 if (!ref_field->event_name) {
2102                         err = -ENOMEM;
2103                         goto free;
2104                 }
2105         }
2106
2107         if (var_field->var.name) {
2108                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2109                 if (!ref_field->name) {
2110                         err = -ENOMEM;
2111                         goto free;
2112                 }
2113         } else if (var_field->name) {
2114                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2115                 if (!ref_field->name) {
2116                         err = -ENOMEM;
2117                         goto free;
2118                 }
2119         }
2120
2121         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2122         if (!ref_field->type) {
2123                 err = -ENOMEM;
2124                 goto free;
2125         }
2126  out:
2127         return err;
2128  free:
2129         kfree(ref_field->system);
2130         ref_field->system = NULL;
2131         kfree(ref_field->event_name);
2132         ref_field->event_name = NULL;
2133         kfree(ref_field->name);
2134         ref_field->name = NULL;
2135
2136         goto out;
2137 }
2138
2139 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2140                             struct hist_field *var_field)
2141 {
2142         struct hist_field *ref_field;
2143         int i;
2144
2145         for (i = 0; i < hist_data->n_var_refs; i++) {
2146                 ref_field = hist_data->var_refs[i];
2147                 if (ref_field->var.idx == var_field->var.idx &&
2148                     ref_field->var.hist_data == var_field->hist_data)
2149                         return i;
2150         }
2151
2152         return -ENOENT;
2153 }
2154
2155 /**
2156  * create_var_ref - Create a variable reference and attach it to trigger
2157  * @hist_data: The trigger that will be referencing the variable
2158  * @var_field: The VAR field to create a reference to
2159  * @system: The optional system string
2160  * @event_name: The optional event_name string
2161  *
2162  * Given a variable hist_field, create a VAR_REF hist_field that
2163  * represents a reference to it.
2164  *
2165  * This function also adds the reference to the trigger that
2166  * now references the variable.
2167  *
2168  * Return: The VAR_REF field if successful, NULL if not
2169  */
2170 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2171                                          struct hist_field *var_field,
2172                                          char *system, char *event_name)
2173 {
2174         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2175         struct hist_field *ref_field;
2176         int i;
2177
2178         /* Check if the variable already exists */
2179         for (i = 0; i < hist_data->n_var_refs; i++) {
2180                 ref_field = hist_data->var_refs[i];
2181                 if (ref_field->var.idx == var_field->var.idx &&
2182                     ref_field->var.hist_data == var_field->hist_data) {
2183                         get_hist_field(ref_field);
2184                         return ref_field;
2185                 }
2186         }
2187         /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2188         if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2189                 return NULL;
2190         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2191         if (ref_field) {
2192                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2193                         destroy_hist_field(ref_field, 0);
2194                         return NULL;
2195                 }
2196
2197                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2198                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2199         }
2200
2201         return ref_field;
2202 }
2203
2204 static bool is_var_ref(char *var_name)
2205 {
2206         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2207                 return false;
2208
2209         return true;
2210 }
2211
2212 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2213                                  char *var_name)
2214 {
2215         char *name, *field;
2216         unsigned int i;
2217
2218         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2219                 name = hist_data->attrs->var_defs.name[i];
2220
2221                 if (strcmp(var_name, name) == 0) {
2222                         field = hist_data->attrs->var_defs.expr[i];
2223                         if (contains_operator(field, NULL) || is_var_ref(field))
2224                                 continue;
2225                         return field;
2226                 }
2227         }
2228
2229         return NULL;
2230 }
2231
2232 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2233                                  char *system, char *event_name,
2234                                  char *var_name)
2235 {
2236         struct trace_event_call *call;
2237
2238         if (system && event_name) {
2239                 call = hist_data->event_file->event_call;
2240
2241                 if (strcmp(system, call->class->system) != 0)
2242                         return NULL;
2243
2244                 if (strcmp(event_name, trace_event_name(call)) != 0)
2245                         return NULL;
2246         }
2247
2248         if (!!system != !!event_name)
2249                 return NULL;
2250
2251         if (!is_var_ref(var_name))
2252                 return NULL;
2253
2254         var_name++;
2255
2256         return field_name_from_var(hist_data, var_name);
2257 }
2258
2259 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2260                                         char *system, char *event_name,
2261                                         char *var_name)
2262 {
2263         struct hist_field *var_field = NULL, *ref_field = NULL;
2264         struct trace_array *tr = hist_data->event_file->tr;
2265
2266         if (!is_var_ref(var_name))
2267                 return NULL;
2268
2269         var_name++;
2270
2271         var_field = find_event_var(hist_data, system, event_name, var_name);
2272         if (var_field)
2273                 ref_field = create_var_ref(hist_data, var_field,
2274                                            system, event_name);
2275
2276         if (!ref_field)
2277                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2278
2279         return ref_field;
2280 }
2281
2282 static struct ftrace_event_field *
2283 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2284             char *field_str, unsigned long *flags, unsigned long *buckets)
2285 {
2286         struct ftrace_event_field *field = NULL;
2287         char *field_name, *modifier, *str;
2288         struct trace_array *tr = file->tr;
2289
2290         modifier = str = kstrdup(field_str, GFP_KERNEL);
2291         if (!modifier)
2292                 return ERR_PTR(-ENOMEM);
2293
2294         field_name = strsep(&modifier, ".");
2295         if (modifier) {
2296                 if (strcmp(modifier, "hex") == 0)
2297                         *flags |= HIST_FIELD_FL_HEX;
2298                 else if (strcmp(modifier, "sym") == 0)
2299                         *flags |= HIST_FIELD_FL_SYM;
2300                 /*
2301                  * 'sym-offset' occurrences in the trigger string are modified
2302                  * to 'symXoffset' to simplify arithmetic expression parsing.
2303                  */
2304                 else if (strcmp(modifier, "symXoffset") == 0)
2305                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2306                 else if ((strcmp(modifier, "execname") == 0) &&
2307                          (strcmp(field_name, "common_pid") == 0))
2308                         *flags |= HIST_FIELD_FL_EXECNAME;
2309                 else if (strcmp(modifier, "syscall") == 0)
2310                         *flags |= HIST_FIELD_FL_SYSCALL;
2311                 else if (strcmp(modifier, "log2") == 0)
2312                         *flags |= HIST_FIELD_FL_LOG2;
2313                 else if (strcmp(modifier, "usecs") == 0)
2314                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2315                 else if (strncmp(modifier, "bucket", 6) == 0) {
2316                         int ret;
2317
2318                         modifier += 6;
2319
2320                         if (*modifier == 's')
2321                                 modifier++;
2322                         if (*modifier != '=')
2323                                 goto error;
2324                         modifier++;
2325                         ret = kstrtoul(modifier, 0, buckets);
2326                         if (ret || !(*buckets))
2327                                 goto error;
2328                         *flags |= HIST_FIELD_FL_BUCKET;
2329                 } else if (strncmp(modifier, "percent", 7) == 0) {
2330                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2331                                 goto error;
2332                         *flags |= HIST_FIELD_FL_PERCENT;
2333                 } else if (strncmp(modifier, "graph", 5) == 0) {
2334                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2335                                 goto error;
2336                         *flags |= HIST_FIELD_FL_GRAPH;
2337                 } else {
2338  error:
2339                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2340                         field = ERR_PTR(-EINVAL);
2341                         goto out;
2342                 }
2343         }
2344
2345         if (strcmp(field_name, "common_timestamp") == 0) {
2346                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2347                 hist_data->enable_timestamps = true;
2348                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2349                         hist_data->attrs->ts_in_usecs = true;
2350         } else if (strcmp(field_name, "common_cpu") == 0)
2351                 *flags |= HIST_FIELD_FL_CPU;
2352         else {
2353                 field = trace_find_event_field(file->event_call, field_name);
2354                 if (!field || !field->size) {
2355                         /*
2356                          * For backward compatibility, if field_name
2357                          * was "cpu", then we treat this the same as
2358                          * common_cpu. This also works for "CPU".
2359                          */
2360                         if (field && field->filter_type == FILTER_CPU) {
2361                                 *flags |= HIST_FIELD_FL_CPU;
2362                         } else {
2363                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2364                                          errpos(field_name));
2365                                 field = ERR_PTR(-EINVAL);
2366                                 goto out;
2367                         }
2368                 }
2369         }
2370  out:
2371         kfree(str);
2372
2373         return field;
2374 }
2375
2376 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2377                                        struct hist_field *var_ref,
2378                                        char *var_name)
2379 {
2380         struct hist_field *alias = NULL;
2381         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2382
2383         alias = create_hist_field(hist_data, NULL, flags, var_name);
2384         if (!alias)
2385                 return NULL;
2386
2387         alias->fn_num = var_ref->fn_num;
2388         alias->operands[0] = var_ref;
2389
2390         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2391                 destroy_hist_field(alias, 0);
2392                 return NULL;
2393         }
2394
2395         alias->var_ref_idx = var_ref->var_ref_idx;
2396
2397         return alias;
2398 }
2399
2400 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2401                                       char *str, char *var_name,
2402                                       unsigned long *flags)
2403 {
2404         struct trace_array *tr = hist_data->event_file->tr;
2405         struct hist_field *field = NULL;
2406         u64 constant;
2407
2408         if (kstrtoull(str, 0, &constant)) {
2409                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2410                 return NULL;
2411         }
2412
2413         *flags |= HIST_FIELD_FL_CONST;
2414         field = create_hist_field(hist_data, NULL, *flags, var_name);
2415         if (!field)
2416                 return NULL;
2417
2418         field->constant = constant;
2419
2420         return field;
2421 }
2422
2423 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2424                                      struct trace_event_file *file, char *str,
2425                                      unsigned long *flags, char *var_name)
2426 {
2427         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2428         struct ftrace_event_field *field = NULL;
2429         struct hist_field *hist_field = NULL;
2430         unsigned long buckets = 0;
2431         int ret = 0;
2432
2433         if (isdigit(str[0])) {
2434                 hist_field = parse_const(hist_data, str, var_name, flags);
2435                 if (!hist_field) {
2436                         ret = -EINVAL;
2437                         goto out;
2438                 }
2439                 return hist_field;
2440         }
2441
2442         s = strchr(str, '.');
2443         if (s) {
2444                 s = strchr(++s, '.');
2445                 if (s) {
2446                         ref_system = strsep(&str, ".");
2447                         if (!str) {
2448                                 ret = -EINVAL;
2449                                 goto out;
2450                         }
2451                         ref_event = strsep(&str, ".");
2452                         if (!str) {
2453                                 ret = -EINVAL;
2454                                 goto out;
2455                         }
2456                         ref_var = str;
2457                 }
2458         }
2459
2460         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2461         if (!s) {
2462                 hist_field = parse_var_ref(hist_data, ref_system,
2463                                            ref_event, ref_var);
2464                 if (hist_field) {
2465                         if (var_name) {
2466                                 hist_field = create_alias(hist_data, hist_field, var_name);
2467                                 if (!hist_field) {
2468                                         ret = -ENOMEM;
2469                                         goto out;
2470                                 }
2471                         }
2472                         return hist_field;
2473                 }
2474         } else
2475                 str = s;
2476
2477         field = parse_field(hist_data, file, str, flags, &buckets);
2478         if (IS_ERR(field)) {
2479                 ret = PTR_ERR(field);
2480                 goto out;
2481         }
2482
2483         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2484         if (!hist_field) {
2485                 ret = -ENOMEM;
2486                 goto out;
2487         }
2488         hist_field->buckets = buckets;
2489
2490         return hist_field;
2491  out:
2492         return ERR_PTR(ret);
2493 }
2494
2495 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2496                                      struct trace_event_file *file,
2497                                      char *str, unsigned long flags,
2498                                      char *var_name, unsigned int *n_subexprs);
2499
2500 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2501                                       struct trace_event_file *file,
2502                                       char *str, unsigned long flags,
2503                                       char *var_name, unsigned int *n_subexprs)
2504 {
2505         struct hist_field *operand1, *expr = NULL;
2506         unsigned long operand_flags;
2507         int ret = 0;
2508         char *s;
2509
2510         /* Unary minus operator, increment n_subexprs */
2511         ++*n_subexprs;
2512
2513         /* we support only -(xxx) i.e. explicit parens required */
2514
2515         if (*n_subexprs > 3) {
2516                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2517                 ret = -EINVAL;
2518                 goto free;
2519         }
2520
2521         str++; /* skip leading '-' */
2522
2523         s = strchr(str, '(');
2524         if (s)
2525                 str++;
2526         else {
2527                 ret = -EINVAL;
2528                 goto free;
2529         }
2530
2531         s = strrchr(str, ')');
2532         if (s) {
2533                  /* unary minus not supported in sub-expressions */
2534                 if (*(s+1) != '\0') {
2535                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2536                                  errpos(str));
2537                         ret = -EINVAL;
2538                         goto free;
2539                 }
2540                 *s = '\0';
2541         }
2542         else {
2543                 ret = -EINVAL; /* no closing ')' */
2544                 goto free;
2545         }
2546
2547         flags |= HIST_FIELD_FL_EXPR;
2548         expr = create_hist_field(hist_data, NULL, flags, var_name);
2549         if (!expr) {
2550                 ret = -ENOMEM;
2551                 goto free;
2552         }
2553
2554         operand_flags = 0;
2555         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2556         if (IS_ERR(operand1)) {
2557                 ret = PTR_ERR(operand1);
2558                 goto free;
2559         }
2560         if (operand1->flags & HIST_FIELD_FL_STRING) {
2561                 /* String type can not be the operand of unary operator. */
2562                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2563                 destroy_hist_field(operand1, 0);
2564                 ret = -EINVAL;
2565                 goto free;
2566         }
2567
2568         expr->flags |= operand1->flags &
2569                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2570         expr->fn_num = HIST_FIELD_FN_UMINUS;
2571         expr->operands[0] = operand1;
2572         expr->size = operand1->size;
2573         expr->is_signed = operand1->is_signed;
2574         expr->operator = FIELD_OP_UNARY_MINUS;
2575         expr->name = expr_str(expr, 0);
2576         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2577         if (!expr->type) {
2578                 ret = -ENOMEM;
2579                 goto free;
2580         }
2581
2582         return expr;
2583  free:
2584         destroy_hist_field(expr, 0);
2585         return ERR_PTR(ret);
2586 }
2587
2588 /*
2589  * If the operands are var refs, return pointers the
2590  * variable(s) referenced in var1 and var2, else NULL.
2591  */
2592 static int check_expr_operands(struct trace_array *tr,
2593                                struct hist_field *operand1,
2594                                struct hist_field *operand2,
2595                                struct hist_field **var1,
2596                                struct hist_field **var2)
2597 {
2598         unsigned long operand1_flags = operand1->flags;
2599         unsigned long operand2_flags = operand2->flags;
2600
2601         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2602             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2603                 struct hist_field *var;
2604
2605                 var = find_var_field(operand1->var.hist_data, operand1->name);
2606                 if (!var)
2607                         return -EINVAL;
2608                 operand1_flags = var->flags;
2609                 *var1 = var;
2610         }
2611
2612         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2613             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2614                 struct hist_field *var;
2615
2616                 var = find_var_field(operand2->var.hist_data, operand2->name);
2617                 if (!var)
2618                         return -EINVAL;
2619                 operand2_flags = var->flags;
2620                 *var2 = var;
2621         }
2622
2623         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2624             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2625                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2626                 return -EINVAL;
2627         }
2628
2629         return 0;
2630 }
2631
2632 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2633                                      struct trace_event_file *file,
2634                                      char *str, unsigned long flags,
2635                                      char *var_name, unsigned int *n_subexprs)
2636 {
2637         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2638         struct hist_field *var1 = NULL, *var2 = NULL;
2639         unsigned long operand_flags, operand2_flags;
2640         int field_op, ret = -EINVAL;
2641         char *sep, *operand1_str;
2642         enum hist_field_fn op_fn;
2643         bool combine_consts;
2644
2645         if (*n_subexprs > 3) {
2646                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2647                 return ERR_PTR(-EINVAL);
2648         }
2649
2650         field_op = contains_operator(str, &sep);
2651
2652         if (field_op == FIELD_OP_NONE)
2653                 return parse_atom(hist_data, file, str, &flags, var_name);
2654
2655         if (field_op == FIELD_OP_UNARY_MINUS)
2656                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2657
2658         /* Binary operator found, increment n_subexprs */
2659         ++*n_subexprs;
2660
2661         /* Split the expression string at the root operator */
2662         if (!sep)
2663                 return ERR_PTR(-EINVAL);
2664
2665         *sep = '\0';
2666         operand1_str = str;
2667         str = sep+1;
2668
2669         /* Binary operator requires both operands */
2670         if (*operand1_str == '\0' || *str == '\0')
2671                 return ERR_PTR(-EINVAL);
2672
2673         operand_flags = 0;
2674
2675         /* LHS of string is an expression e.g. a+b in a+b+c */
2676         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2677         if (IS_ERR(operand1))
2678                 return ERR_CAST(operand1);
2679
2680         if (operand1->flags & HIST_FIELD_FL_STRING) {
2681                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2682                 ret = -EINVAL;
2683                 goto free_op1;
2684         }
2685
2686         /* RHS of string is another expression e.g. c in a+b+c */
2687         operand_flags = 0;
2688         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2689         if (IS_ERR(operand2)) {
2690                 ret = PTR_ERR(operand2);
2691                 goto free_op1;
2692         }
2693         if (operand2->flags & HIST_FIELD_FL_STRING) {
2694                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2695                 ret = -EINVAL;
2696                 goto free_operands;
2697         }
2698
2699         switch (field_op) {
2700         case FIELD_OP_MINUS:
2701                 op_fn = HIST_FIELD_FN_MINUS;
2702                 break;
2703         case FIELD_OP_PLUS:
2704                 op_fn = HIST_FIELD_FN_PLUS;
2705                 break;
2706         case FIELD_OP_DIV:
2707                 op_fn = HIST_FIELD_FN_DIV;
2708                 break;
2709         case FIELD_OP_MULT:
2710                 op_fn = HIST_FIELD_FN_MULT;
2711                 break;
2712         default:
2713                 ret = -EINVAL;
2714                 goto free_operands;
2715         }
2716
2717         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2718         if (ret)
2719                 goto free_operands;
2720
2721         operand_flags = var1 ? var1->flags : operand1->flags;
2722         operand2_flags = var2 ? var2->flags : operand2->flags;
2723
2724         /*
2725          * If both operands are constant, the expression can be
2726          * collapsed to a single constant.
2727          */
2728         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2729
2730         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2731
2732         flags |= operand1->flags &
2733                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2734
2735         expr = create_hist_field(hist_data, NULL, flags, var_name);
2736         if (!expr) {
2737                 ret = -ENOMEM;
2738                 goto free_operands;
2739         }
2740
2741         operand1->read_once = true;
2742         operand2->read_once = true;
2743
2744         /* The operands are now owned and free'd by 'expr' */
2745         expr->operands[0] = operand1;
2746         expr->operands[1] = operand2;
2747
2748         if (field_op == FIELD_OP_DIV &&
2749                         operand2_flags & HIST_FIELD_FL_CONST) {
2750                 u64 divisor = var2 ? var2->constant : operand2->constant;
2751
2752                 if (!divisor) {
2753                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2754                         ret = -EDOM;
2755                         goto free_expr;
2756                 }
2757
2758                 /*
2759                  * Copy the divisor here so we don't have to look it up
2760                  * later if this is a var ref
2761                  */
2762                 operand2->constant = divisor;
2763                 op_fn = hist_field_get_div_fn(operand2);
2764         }
2765
2766         expr->fn_num = op_fn;
2767
2768         if (combine_consts) {
2769                 if (var1)
2770                         expr->operands[0] = var1;
2771                 if (var2)
2772                         expr->operands[1] = var2;
2773
2774                 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2775                 expr->fn_num = HIST_FIELD_FN_CONST;
2776
2777                 expr->operands[0] = NULL;
2778                 expr->operands[1] = NULL;
2779
2780                 /*
2781                  * var refs won't be destroyed immediately
2782                  * See: destroy_hist_field()
2783                  */
2784                 destroy_hist_field(operand2, 0);
2785                 destroy_hist_field(operand1, 0);
2786
2787                 expr->name = expr_str(expr, 0);
2788         } else {
2789                 /* The operand sizes should be the same, so just pick one */
2790                 expr->size = operand1->size;
2791                 expr->is_signed = operand1->is_signed;
2792
2793                 expr->operator = field_op;
2794                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2795                 if (!expr->type) {
2796                         ret = -ENOMEM;
2797                         goto free_expr;
2798                 }
2799
2800                 expr->name = expr_str(expr, 0);
2801         }
2802
2803         return expr;
2804
2805 free_operands:
2806         destroy_hist_field(operand2, 0);
2807 free_op1:
2808         destroy_hist_field(operand1, 0);
2809         return ERR_PTR(ret);
2810
2811 free_expr:
2812         destroy_hist_field(expr, 0);
2813         return ERR_PTR(ret);
2814 }
2815
2816 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2817                                  struct trace_event_file *file)
2818 {
2819         struct event_trigger_data *test;
2820
2821         lockdep_assert_held(&event_mutex);
2822
2823         list_for_each_entry(test, &file->triggers, list) {
2824                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2825                         if (test->private_data == hist_data)
2826                                 return test->filter_str;
2827                 }
2828         }
2829
2830         return NULL;
2831 }
2832
2833 static struct event_command trigger_hist_cmd;
2834 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2835                                     struct trace_event_file *file,
2836                                     char *glob, char *cmd,
2837                                     char *param_and_filter);
2838
2839 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2840                             struct hist_trigger_data *hist_data,
2841                             unsigned int n_keys)
2842 {
2843         struct hist_field *target_hist_field, *hist_field;
2844         unsigned int n, i, j;
2845
2846         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2847                 return false;
2848
2849         i = hist_data->n_vals;
2850         j = target_hist_data->n_vals;
2851
2852         for (n = 0; n < n_keys; n++) {
2853                 hist_field = hist_data->fields[i + n];
2854                 target_hist_field = target_hist_data->fields[j + n];
2855
2856                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2857                         return false;
2858                 if (hist_field->size != target_hist_field->size)
2859                         return false;
2860                 if (hist_field->is_signed != target_hist_field->is_signed)
2861                         return false;
2862         }
2863
2864         return true;
2865 }
2866
2867 static struct hist_trigger_data *
2868 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2869                      struct trace_event_file *file)
2870 {
2871         struct hist_trigger_data *hist_data;
2872         struct event_trigger_data *test;
2873         unsigned int n_keys;
2874
2875         lockdep_assert_held(&event_mutex);
2876
2877         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2878
2879         list_for_each_entry(test, &file->triggers, list) {
2880                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2881                         hist_data = test->private_data;
2882
2883                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2884                                 return hist_data;
2885                 }
2886         }
2887
2888         return NULL;
2889 }
2890
2891 static struct trace_event_file *event_file(struct trace_array *tr,
2892                                            char *system, char *event_name)
2893 {
2894         struct trace_event_file *file;
2895
2896         file = __find_event_file(tr, system, event_name);
2897         if (!file)
2898                 return ERR_PTR(-EINVAL);
2899
2900         return file;
2901 }
2902
2903 static struct hist_field *
2904 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2905                          char *system, char *event_name, char *field_name)
2906 {
2907         struct hist_field *event_var;
2908         char *synthetic_name;
2909
2910         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2911         if (!synthetic_name)
2912                 return ERR_PTR(-ENOMEM);
2913
2914         strcpy(synthetic_name, "synthetic_");
2915         strcat(synthetic_name, field_name);
2916
2917         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2918
2919         kfree(synthetic_name);
2920
2921         return event_var;
2922 }
2923
2924 /**
2925  * create_field_var_hist - Automatically create a histogram and var for a field
2926  * @target_hist_data: The target hist trigger
2927  * @subsys_name: Optional subsystem name
2928  * @event_name: Optional event name
2929  * @field_name: The name of the field (and the resulting variable)
2930  *
2931  * Hist trigger actions fetch data from variables, not directly from
2932  * events.  However, for convenience, users are allowed to directly
2933  * specify an event field in an action, which will be automatically
2934  * converted into a variable on their behalf.
2935  *
2936  * If a user specifies a field on an event that isn't the event the
2937  * histogram currently being defined (the target event histogram), the
2938  * only way that can be accomplished is if a new hist trigger is
2939  * created and the field variable defined on that.
2940  *
2941  * This function creates a new histogram compatible with the target
2942  * event (meaning a histogram with the same key as the target
2943  * histogram), and creates a variable for the specified field, but
2944  * with 'synthetic_' prepended to the variable name in order to avoid
2945  * collision with normal field variables.
2946  *
2947  * Return: The variable created for the field.
2948  */
2949 static struct hist_field *
2950 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2951                       char *subsys_name, char *event_name, char *field_name)
2952 {
2953         struct trace_array *tr = target_hist_data->event_file->tr;
2954         struct hist_trigger_data *hist_data;
2955         unsigned int i, n, first = true;
2956         struct field_var_hist *var_hist;
2957         struct trace_event_file *file;
2958         struct hist_field *key_field;
2959         struct hist_field *event_var;
2960         char *saved_filter;
2961         char *cmd;
2962         int ret;
2963
2964         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2965                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2966                 return ERR_PTR(-EINVAL);
2967         }
2968
2969         file = event_file(tr, subsys_name, event_name);
2970
2971         if (IS_ERR(file)) {
2972                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2973                 ret = PTR_ERR(file);
2974                 return ERR_PTR(ret);
2975         }
2976
2977         /*
2978          * Look for a histogram compatible with target.  We'll use the
2979          * found histogram specification to create a new matching
2980          * histogram with our variable on it.  target_hist_data is not
2981          * yet a registered histogram so we can't use that.
2982          */
2983         hist_data = find_compatible_hist(target_hist_data, file);
2984         if (!hist_data) {
2985                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2986                 return ERR_PTR(-EINVAL);
2987         }
2988
2989         /* See if a synthetic field variable has already been created */
2990         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2991                                              event_name, field_name);
2992         if (!IS_ERR_OR_NULL(event_var))
2993                 return event_var;
2994
2995         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2996         if (!var_hist)
2997                 return ERR_PTR(-ENOMEM);
2998
2999         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3000         if (!cmd) {
3001                 kfree(var_hist);
3002                 return ERR_PTR(-ENOMEM);
3003         }
3004
3005         /* Use the same keys as the compatible histogram */
3006         strcat(cmd, "keys=");
3007
3008         for_each_hist_key_field(i, hist_data) {
3009                 key_field = hist_data->fields[i];
3010                 if (!first)
3011                         strcat(cmd, ",");
3012                 strcat(cmd, key_field->field->name);
3013                 first = false;
3014         }
3015
3016         /* Create the synthetic field variable specification */
3017         strcat(cmd, ":synthetic_");
3018         strcat(cmd, field_name);
3019         strcat(cmd, "=");
3020         strcat(cmd, field_name);
3021
3022         /* Use the same filter as the compatible histogram */
3023         saved_filter = find_trigger_filter(hist_data, file);
3024         if (saved_filter) {
3025                 strcat(cmd, " if ");
3026                 strcat(cmd, saved_filter);
3027         }
3028
3029         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3030         if (!var_hist->cmd) {
3031                 kfree(cmd);
3032                 kfree(var_hist);
3033                 return ERR_PTR(-ENOMEM);
3034         }
3035
3036         /* Save the compatible histogram information */
3037         var_hist->hist_data = hist_data;
3038
3039         /* Create the new histogram with our variable */
3040         ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3041                                        "", "hist", cmd);
3042         if (ret) {
3043                 kfree(cmd);
3044                 kfree(var_hist->cmd);
3045                 kfree(var_hist);
3046                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3047                 return ERR_PTR(ret);
3048         }
3049
3050         kfree(cmd);
3051
3052         /* If we can't find the variable, something went wrong */
3053         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3054                                              event_name, field_name);
3055         if (IS_ERR_OR_NULL(event_var)) {
3056                 kfree(var_hist->cmd);
3057                 kfree(var_hist);
3058                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3059                 return ERR_PTR(-EINVAL);
3060         }
3061
3062         n = target_hist_data->n_field_var_hists;
3063         target_hist_data->field_var_hists[n] = var_hist;
3064         target_hist_data->n_field_var_hists++;
3065
3066         return event_var;
3067 }
3068
3069 static struct hist_field *
3070 find_target_event_var(struct hist_trigger_data *hist_data,
3071                       char *subsys_name, char *event_name, char *var_name)
3072 {
3073         struct trace_event_file *file = hist_data->event_file;
3074         struct hist_field *hist_field = NULL;
3075
3076         if (subsys_name) {
3077                 struct trace_event_call *call;
3078
3079                 if (!event_name)
3080                         return NULL;
3081
3082                 call = file->event_call;
3083
3084                 if (strcmp(subsys_name, call->class->system) != 0)
3085                         return NULL;
3086
3087                 if (strcmp(event_name, trace_event_name(call)) != 0)
3088                         return NULL;
3089         }
3090
3091         hist_field = find_var_field(hist_data, var_name);
3092
3093         return hist_field;
3094 }
3095
3096 static inline void __update_field_vars(struct tracing_map_elt *elt,
3097                                        struct trace_buffer *buffer,
3098                                        struct ring_buffer_event *rbe,
3099                                        void *rec,
3100                                        struct field_var **field_vars,
3101                                        unsigned int n_field_vars,
3102                                        unsigned int field_var_str_start)
3103 {
3104         struct hist_elt_data *elt_data = elt->private_data;
3105         unsigned int i, j, var_idx;
3106         u64 var_val;
3107
3108         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3109                 struct field_var *field_var = field_vars[i];
3110                 struct hist_field *var = field_var->var;
3111                 struct hist_field *val = field_var->val;
3112
3113                 var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3114                 var_idx = var->var.idx;
3115
3116                 if (val->flags & HIST_FIELD_FL_STRING) {
3117                         char *str = elt_data->field_var_str[j++];
3118                         char *val_str = (char *)(uintptr_t)var_val;
3119                         unsigned int size;
3120
3121                         size = min(val->size, STR_VAR_LEN_MAX);
3122                         strscpy(str, val_str, size);
3123                         var_val = (u64)(uintptr_t)str;
3124                 }
3125                 tracing_map_set_var(elt, var_idx, var_val);
3126         }
3127 }
3128
3129 static void update_field_vars(struct hist_trigger_data *hist_data,
3130                               struct tracing_map_elt *elt,
3131                               struct trace_buffer *buffer,
3132                               struct ring_buffer_event *rbe,
3133                               void *rec)
3134 {
3135         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3136                             hist_data->n_field_vars, 0);
3137 }
3138
3139 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3140                                  struct tracing_map_elt *elt,
3141                                  struct trace_buffer *buffer,  void *rec,
3142                                  struct ring_buffer_event *rbe, void *key,
3143                                  struct action_data *data, u64 *var_ref_vals)
3144 {
3145         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3146                             hist_data->n_save_vars, hist_data->n_field_var_str);
3147 }
3148
3149 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3150                                      struct trace_event_file *file,
3151                                      char *name, int size, const char *type)
3152 {
3153         struct hist_field *var;
3154         int idx;
3155
3156         if (find_var(hist_data, file, name) && !hist_data->remove) {
3157                 var = ERR_PTR(-EINVAL);
3158                 goto out;
3159         }
3160
3161         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3162         if (!var) {
3163                 var = ERR_PTR(-ENOMEM);
3164                 goto out;
3165         }
3166
3167         idx = tracing_map_add_var(hist_data->map);
3168         if (idx < 0) {
3169                 kfree(var);
3170                 var = ERR_PTR(-EINVAL);
3171                 goto out;
3172         }
3173
3174         var->ref = 1;
3175         var->flags = HIST_FIELD_FL_VAR;
3176         var->var.idx = idx;
3177         var->var.hist_data = var->hist_data = hist_data;
3178         var->size = size;
3179         var->var.name = kstrdup(name, GFP_KERNEL);
3180         var->type = kstrdup_const(type, GFP_KERNEL);
3181         if (!var->var.name || !var->type) {
3182                 kfree_const(var->type);
3183                 kfree(var->var.name);
3184                 kfree(var);
3185                 var = ERR_PTR(-ENOMEM);
3186         }
3187  out:
3188         return var;
3189 }
3190
3191 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3192                                           struct trace_event_file *file,
3193                                           char *field_name)
3194 {
3195         struct hist_field *val = NULL, *var = NULL;
3196         unsigned long flags = HIST_FIELD_FL_VAR;
3197         struct trace_array *tr = file->tr;
3198         struct field_var *field_var;
3199         int ret = 0;
3200
3201         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3202                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3203                 ret = -EINVAL;
3204                 goto err;
3205         }
3206
3207         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3208         if (IS_ERR(val)) {
3209                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3210                 ret = PTR_ERR(val);
3211                 goto err;
3212         }
3213
3214         var = create_var(hist_data, file, field_name, val->size, val->type);
3215         if (IS_ERR(var)) {
3216                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3217                 kfree(val);
3218                 ret = PTR_ERR(var);
3219                 goto err;
3220         }
3221
3222         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3223         if (!field_var) {
3224                 kfree(val);
3225                 kfree(var);
3226                 ret =  -ENOMEM;
3227                 goto err;
3228         }
3229
3230         field_var->var = var;
3231         field_var->val = val;
3232  out:
3233         return field_var;
3234  err:
3235         field_var = ERR_PTR(ret);
3236         goto out;
3237 }
3238
3239 /**
3240  * create_target_field_var - Automatically create a variable for a field
3241  * @target_hist_data: The target hist trigger
3242  * @subsys_name: Optional subsystem name
3243  * @event_name: Optional event name
3244  * @var_name: The name of the field (and the resulting variable)
3245  *
3246  * Hist trigger actions fetch data from variables, not directly from
3247  * events.  However, for convenience, users are allowed to directly
3248  * specify an event field in an action, which will be automatically
3249  * converted into a variable on their behalf.
3250  *
3251  * This function creates a field variable with the name var_name on
3252  * the hist trigger currently being defined on the target event.  If
3253  * subsys_name and event_name are specified, this function simply
3254  * verifies that they do in fact match the target event subsystem and
3255  * event name.
3256  *
3257  * Return: The variable created for the field.
3258  */
3259 static struct field_var *
3260 create_target_field_var(struct hist_trigger_data *target_hist_data,
3261                         char *subsys_name, char *event_name, char *var_name)
3262 {
3263         struct trace_event_file *file = target_hist_data->event_file;
3264
3265         if (subsys_name) {
3266                 struct trace_event_call *call;
3267
3268                 if (!event_name)
3269                         return NULL;
3270
3271                 call = file->event_call;
3272
3273                 if (strcmp(subsys_name, call->class->system) != 0)
3274                         return NULL;
3275
3276                 if (strcmp(event_name, trace_event_name(call)) != 0)
3277                         return NULL;
3278         }
3279
3280         return create_field_var(target_hist_data, file, var_name);
3281 }
3282
3283 static bool check_track_val_max(u64 track_val, u64 var_val)
3284 {
3285         if (var_val <= track_val)
3286                 return false;
3287
3288         return true;
3289 }
3290
3291 static bool check_track_val_changed(u64 track_val, u64 var_val)
3292 {
3293         if (var_val == track_val)
3294                 return false;
3295
3296         return true;
3297 }
3298
3299 static u64 get_track_val(struct hist_trigger_data *hist_data,
3300                          struct tracing_map_elt *elt,
3301                          struct action_data *data)
3302 {
3303         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3304         u64 track_val;
3305
3306         track_val = tracing_map_read_var(elt, track_var_idx);
3307
3308         return track_val;
3309 }
3310
3311 static void save_track_val(struct hist_trigger_data *hist_data,
3312                            struct tracing_map_elt *elt,
3313                            struct action_data *data, u64 var_val)
3314 {
3315         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3316
3317         tracing_map_set_var(elt, track_var_idx, var_val);
3318 }
3319
3320 static void save_track_data(struct hist_trigger_data *hist_data,
3321                             struct tracing_map_elt *elt,
3322                             struct trace_buffer *buffer, void *rec,
3323                             struct ring_buffer_event *rbe, void *key,
3324                             struct action_data *data, u64 *var_ref_vals)
3325 {
3326         if (data->track_data.save_data)
3327                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3328                                            key, data, var_ref_vals);
3329 }
3330
3331 static bool check_track_val(struct tracing_map_elt *elt,
3332                             struct action_data *data,
3333                             u64 var_val)
3334 {
3335         struct hist_trigger_data *hist_data;
3336         u64 track_val;
3337
3338         hist_data = data->track_data.track_var->hist_data;
3339         track_val = get_track_val(hist_data, elt, data);
3340
3341         return data->track_data.check_val(track_val, var_val);
3342 }
3343
3344 #ifdef CONFIG_TRACER_SNAPSHOT
3345 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3346 {
3347         /* called with tr->max_lock held */
3348         struct track_data *track_data = tr->cond_snapshot->cond_data;
3349         struct hist_elt_data *elt_data, *track_elt_data;
3350         struct snapshot_context *context = cond_data;
3351         struct action_data *action;
3352         u64 track_val;
3353
3354         if (!track_data)
3355                 return false;
3356
3357         action = track_data->action_data;
3358
3359         track_val = get_track_val(track_data->hist_data, context->elt,
3360                                   track_data->action_data);
3361
3362         if (!action->track_data.check_val(track_data->track_val, track_val))
3363                 return false;
3364
3365         track_data->track_val = track_val;
3366         memcpy(track_data->key, context->key, track_data->key_len);
3367
3368         elt_data = context->elt->private_data;
3369         track_elt_data = track_data->elt.private_data;
3370         if (elt_data->comm)
3371                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3372
3373         track_data->updated = true;
3374
3375         return true;
3376 }
3377
3378 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3379                                      struct tracing_map_elt *elt,
3380                                      struct trace_buffer *buffer, void *rec,
3381                                      struct ring_buffer_event *rbe, void *key,
3382                                      struct action_data *data,
3383                                      u64 *var_ref_vals)
3384 {
3385         struct trace_event_file *file = hist_data->event_file;
3386         struct snapshot_context context;
3387
3388         context.elt = elt;
3389         context.key = key;
3390
3391         tracing_snapshot_cond(file->tr, &context);
3392 }
3393
3394 static void hist_trigger_print_key(struct seq_file *m,
3395                                    struct hist_trigger_data *hist_data,
3396                                    void *key,
3397                                    struct tracing_map_elt *elt);
3398
3399 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3400 {
3401         unsigned int i;
3402
3403         if (!hist_data->n_actions)
3404                 return NULL;
3405
3406         for (i = 0; i < hist_data->n_actions; i++) {
3407                 struct action_data *data = hist_data->actions[i];
3408
3409                 if (data->action == ACTION_SNAPSHOT)
3410                         return data;
3411         }
3412
3413         return NULL;
3414 }
3415
3416 static void track_data_snapshot_print(struct seq_file *m,
3417                                       struct hist_trigger_data *hist_data)
3418 {
3419         struct trace_event_file *file = hist_data->event_file;
3420         struct track_data *track_data;
3421         struct action_data *action;
3422
3423         track_data = tracing_cond_snapshot_data(file->tr);
3424         if (!track_data)
3425                 return;
3426
3427         if (!track_data->updated)
3428                 return;
3429
3430         action = snapshot_action(hist_data);
3431         if (!action)
3432                 return;
3433
3434         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3435         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3436                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3437                    action->track_data.var_str, track_data->track_val);
3438
3439         seq_puts(m, "\ttriggered by event with key: ");
3440         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3441         seq_putc(m, '\n');
3442 }
3443 #else
3444 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3445 {
3446         return false;
3447 }
3448 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3449                                      struct tracing_map_elt *elt,
3450                                      struct trace_buffer *buffer, void *rec,
3451                                      struct ring_buffer_event *rbe, void *key,
3452                                      struct action_data *data,
3453                                      u64 *var_ref_vals) {}
3454 static void track_data_snapshot_print(struct seq_file *m,
3455                                       struct hist_trigger_data *hist_data) {}
3456 #endif /* CONFIG_TRACER_SNAPSHOT */
3457
3458 static void track_data_print(struct seq_file *m,
3459                              struct hist_trigger_data *hist_data,
3460                              struct tracing_map_elt *elt,
3461                              struct action_data *data)
3462 {
3463         u64 track_val = get_track_val(hist_data, elt, data);
3464         unsigned int i, save_var_idx;
3465
3466         if (data->handler == HANDLER_ONMAX)
3467                 seq_printf(m, "\n\tmax: %10llu", track_val);
3468         else if (data->handler == HANDLER_ONCHANGE)
3469                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3470
3471         if (data->action == ACTION_SNAPSHOT)
3472                 return;
3473
3474         for (i = 0; i < hist_data->n_save_vars; i++) {
3475                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3476                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3477                 u64 val;
3478
3479                 save_var_idx = save_var->var.idx;
3480
3481                 val = tracing_map_read_var(elt, save_var_idx);
3482
3483                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3484                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3485                                    (char *)(uintptr_t)(val));
3486                 } else
3487                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3488         }
3489 }
3490
3491 static void ontrack_action(struct hist_trigger_data *hist_data,
3492                            struct tracing_map_elt *elt,
3493                            struct trace_buffer *buffer, void *rec,
3494                            struct ring_buffer_event *rbe, void *key,
3495                            struct action_data *data, u64 *var_ref_vals)
3496 {
3497         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3498
3499         if (check_track_val(elt, data, var_val)) {
3500                 save_track_val(hist_data, elt, data, var_val);
3501                 save_track_data(hist_data, elt, buffer, rec, rbe,
3502                                 key, data, var_ref_vals);
3503         }
3504 }
3505
3506 static void action_data_destroy(struct action_data *data)
3507 {
3508         unsigned int i;
3509
3510         lockdep_assert_held(&event_mutex);
3511
3512         kfree(data->action_name);
3513
3514         for (i = 0; i < data->n_params; i++)
3515                 kfree(data->params[i]);
3516
3517         if (data->synth_event)
3518                 data->synth_event->ref--;
3519
3520         kfree(data->synth_event_name);
3521
3522         kfree(data);
3523 }
3524
3525 static void track_data_destroy(struct hist_trigger_data *hist_data,
3526                                struct action_data *data)
3527 {
3528         struct trace_event_file *file = hist_data->event_file;
3529
3530         destroy_hist_field(data->track_data.track_var, 0);
3531
3532         if (data->action == ACTION_SNAPSHOT) {
3533                 struct track_data *track_data;
3534
3535                 track_data = tracing_cond_snapshot_data(file->tr);
3536                 if (track_data && track_data->hist_data == hist_data) {
3537                         tracing_snapshot_cond_disable(file->tr);
3538                         track_data_free(track_data);
3539                 }
3540         }
3541
3542         kfree(data->track_data.var_str);
3543
3544         action_data_destroy(data);
3545 }
3546
3547 static int action_create(struct hist_trigger_data *hist_data,
3548                          struct action_data *data);
3549
3550 static int track_data_create(struct hist_trigger_data *hist_data,
3551                              struct action_data *data)
3552 {
3553         struct hist_field *var_field, *ref_field, *track_var = NULL;
3554         struct trace_event_file *file = hist_data->event_file;
3555         struct trace_array *tr = file->tr;
3556         char *track_data_var_str;
3557         int ret = 0;
3558
3559         track_data_var_str = data->track_data.var_str;
3560         if (track_data_var_str[0] != '$') {
3561                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3562                 return -EINVAL;
3563         }
3564         track_data_var_str++;
3565
3566         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3567         if (!var_field) {
3568                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3569                 return -EINVAL;
3570         }
3571
3572         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3573         if (!ref_field)
3574                 return -ENOMEM;
3575
3576         data->track_data.var_ref = ref_field;
3577
3578         if (data->handler == HANDLER_ONMAX)
3579                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3580         if (IS_ERR(track_var)) {
3581                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3582                 ret = PTR_ERR(track_var);
3583                 goto out;
3584         }
3585
3586         if (data->handler == HANDLER_ONCHANGE)
3587                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3588         if (IS_ERR(track_var)) {
3589                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3590                 ret = PTR_ERR(track_var);
3591                 goto out;
3592         }
3593         data->track_data.track_var = track_var;
3594
3595         ret = action_create(hist_data, data);
3596  out:
3597         return ret;
3598 }
3599
3600 static int parse_action_params(struct trace_array *tr, char *params,
3601                                struct action_data *data)
3602 {
3603         char *param, *saved_param;
3604         bool first_param = true;
3605         int ret = 0;
3606
3607         while (params) {
3608                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3609                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3610                         ret = -EINVAL;
3611                         goto out;
3612                 }
3613
3614                 param = strsep(&params, ",");
3615                 if (!param) {
3616                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3617                         ret = -EINVAL;
3618                         goto out;
3619                 }
3620
3621                 param = strstrip(param);
3622                 if (strlen(param) < 2) {
3623                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3624                         ret = -EINVAL;
3625                         goto out;
3626                 }
3627
3628                 saved_param = kstrdup(param, GFP_KERNEL);
3629                 if (!saved_param) {
3630                         ret = -ENOMEM;
3631                         goto out;
3632                 }
3633
3634                 if (first_param && data->use_trace_keyword) {
3635                         data->synth_event_name = saved_param;
3636                         first_param = false;
3637                         continue;
3638                 }
3639                 first_param = false;
3640
3641                 data->params[data->n_params++] = saved_param;
3642         }
3643  out:
3644         return ret;
3645 }
3646
3647 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3648                         enum handler_id handler)
3649 {
3650         char *action_name;
3651         int ret = 0;
3652
3653         strsep(&str, ".");
3654         if (!str) {
3655                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3656                 ret = -EINVAL;
3657                 goto out;
3658         }
3659
3660         action_name = strsep(&str, "(");
3661         if (!action_name || !str) {
3662                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3663                 ret = -EINVAL;
3664                 goto out;
3665         }
3666
3667         if (str_has_prefix(action_name, "save")) {
3668                 char *params = strsep(&str, ")");
3669
3670                 if (!params) {
3671                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3672                         ret = -EINVAL;
3673                         goto out;
3674                 }
3675
3676                 ret = parse_action_params(tr, params, data);
3677                 if (ret)
3678                         goto out;
3679
3680                 if (handler == HANDLER_ONMAX)
3681                         data->track_data.check_val = check_track_val_max;
3682                 else if (handler == HANDLER_ONCHANGE)
3683                         data->track_data.check_val = check_track_val_changed;
3684                 else {
3685                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3686                         ret = -EINVAL;
3687                         goto out;
3688                 }
3689
3690                 data->track_data.save_data = save_track_data_vars;
3691                 data->fn = ontrack_action;
3692                 data->action = ACTION_SAVE;
3693         } else if (str_has_prefix(action_name, "snapshot")) {
3694                 char *params = strsep(&str, ")");
3695
3696                 if (!str) {
3697                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3698                         ret = -EINVAL;
3699                         goto out;
3700                 }
3701
3702                 if (handler == HANDLER_ONMAX)
3703                         data->track_data.check_val = check_track_val_max;
3704                 else if (handler == HANDLER_ONCHANGE)
3705                         data->track_data.check_val = check_track_val_changed;
3706                 else {
3707                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3708                         ret = -EINVAL;
3709                         goto out;
3710                 }
3711
3712                 data->track_data.save_data = save_track_data_snapshot;
3713                 data->fn = ontrack_action;
3714                 data->action = ACTION_SNAPSHOT;
3715         } else {
3716                 char *params = strsep(&str, ")");
3717
3718                 if (str_has_prefix(action_name, "trace"))
3719                         data->use_trace_keyword = true;
3720
3721                 if (params) {
3722                         ret = parse_action_params(tr, params, data);
3723                         if (ret)
3724                                 goto out;
3725                 }
3726
3727                 if (handler == HANDLER_ONMAX)
3728                         data->track_data.check_val = check_track_val_max;
3729                 else if (handler == HANDLER_ONCHANGE)
3730                         data->track_data.check_val = check_track_val_changed;
3731
3732                 if (handler != HANDLER_ONMATCH) {
3733                         data->track_data.save_data = action_trace;
3734                         data->fn = ontrack_action;
3735                 } else
3736                         data->fn = action_trace;
3737
3738                 data->action = ACTION_TRACE;
3739         }
3740
3741         data->action_name = kstrdup(action_name, GFP_KERNEL);
3742         if (!data->action_name) {
3743                 ret = -ENOMEM;
3744                 goto out;
3745         }
3746
3747         data->handler = handler;
3748  out:
3749         return ret;
3750 }
3751
3752 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3753                                             char *str, enum handler_id handler)
3754 {
3755         struct action_data *data;
3756         int ret = -EINVAL;
3757         char *var_str;
3758
3759         data = kzalloc(sizeof(*data), GFP_KERNEL);
3760         if (!data)
3761                 return ERR_PTR(-ENOMEM);
3762
3763         var_str = strsep(&str, ")");
3764         if (!var_str || !str) {
3765                 ret = -EINVAL;
3766                 goto free;
3767         }
3768
3769         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3770         if (!data->track_data.var_str) {
3771                 ret = -ENOMEM;
3772                 goto free;
3773         }
3774
3775         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3776         if (ret)
3777                 goto free;
3778  out:
3779         return data;
3780  free:
3781         track_data_destroy(hist_data, data);
3782         data = ERR_PTR(ret);
3783         goto out;
3784 }
3785
3786 static void onmatch_destroy(struct action_data *data)
3787 {
3788         kfree(data->match_data.event);
3789         kfree(data->match_data.event_system);
3790
3791         action_data_destroy(data);
3792 }
3793
3794 static void destroy_field_var(struct field_var *field_var)
3795 {
3796         if (!field_var)
3797                 return;
3798
3799         destroy_hist_field(field_var->var, 0);
3800         destroy_hist_field(field_var->val, 0);
3801
3802         kfree(field_var);
3803 }
3804
3805 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3806 {
3807         unsigned int i;
3808
3809         for (i = 0; i < hist_data->n_field_vars; i++)
3810                 destroy_field_var(hist_data->field_vars[i]);
3811
3812         for (i = 0; i < hist_data->n_save_vars; i++)
3813                 destroy_field_var(hist_data->save_vars[i]);
3814 }
3815
3816 static void save_field_var(struct hist_trigger_data *hist_data,
3817                            struct field_var *field_var)
3818 {
3819         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3820
3821         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3822                 hist_data->n_field_var_str++;
3823 }
3824
3825
3826 static int check_synth_field(struct synth_event *event,
3827                              struct hist_field *hist_field,
3828                              unsigned int field_pos)
3829 {
3830         struct synth_field *field;
3831
3832         if (field_pos >= event->n_fields)
3833                 return -EINVAL;
3834
3835         field = event->fields[field_pos];
3836
3837         /*
3838          * A dynamic string synth field can accept static or
3839          * dynamic. A static string synth field can only accept a
3840          * same-sized static string, which is checked for later.
3841          */
3842         if (strstr(hist_field->type, "char[") && field->is_string
3843             && field->is_dynamic)
3844                 return 0;
3845
3846         if (strcmp(field->type, hist_field->type) != 0) {
3847                 if (field->size != hist_field->size ||
3848                     (!field->is_string && field->is_signed != hist_field->is_signed))
3849                         return -EINVAL;
3850         }
3851
3852         return 0;
3853 }
3854
3855 static struct hist_field *
3856 trace_action_find_var(struct hist_trigger_data *hist_data,
3857                       struct action_data *data,
3858                       char *system, char *event, char *var)
3859 {
3860         struct trace_array *tr = hist_data->event_file->tr;
3861         struct hist_field *hist_field;
3862
3863         var++; /* skip '$' */
3864
3865         hist_field = find_target_event_var(hist_data, system, event, var);
3866         if (!hist_field) {
3867                 if (!system && data->handler == HANDLER_ONMATCH) {
3868                         system = data->match_data.event_system;
3869                         event = data->match_data.event;
3870                 }
3871
3872                 hist_field = find_event_var(hist_data, system, event, var);
3873         }
3874
3875         if (!hist_field)
3876                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3877
3878         return hist_field;
3879 }
3880
3881 static struct hist_field *
3882 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3883                               struct action_data *data, char *system,
3884                               char *event, char *var)
3885 {
3886         struct hist_field *hist_field = NULL;
3887         struct field_var *field_var;
3888
3889         /*
3890          * First try to create a field var on the target event (the
3891          * currently being defined).  This will create a variable for
3892          * unqualified fields on the target event, or if qualified,
3893          * target fields that have qualified names matching the target.
3894          */
3895         field_var = create_target_field_var(hist_data, system, event, var);
3896
3897         if (field_var && !IS_ERR(field_var)) {
3898                 save_field_var(hist_data, field_var);
3899                 hist_field = field_var->var;
3900         } else {
3901                 field_var = NULL;
3902                 /*
3903                  * If no explicit system.event is specified, default to
3904                  * looking for fields on the onmatch(system.event.xxx)
3905                  * event.
3906                  */
3907                 if (!system && data->handler == HANDLER_ONMATCH) {
3908                         system = data->match_data.event_system;
3909                         event = data->match_data.event;
3910                 }
3911
3912                 if (!event)
3913                         goto free;
3914                 /*
3915                  * At this point, we're looking at a field on another
3916                  * event.  Because we can't modify a hist trigger on
3917                  * another event to add a variable for a field, we need
3918                  * to create a new trigger on that event and create the
3919                  * variable at the same time.
3920                  */
3921                 hist_field = create_field_var_hist(hist_data, system, event, var);
3922                 if (IS_ERR(hist_field))
3923                         goto free;
3924         }
3925  out:
3926         return hist_field;
3927  free:
3928         destroy_field_var(field_var);
3929         hist_field = NULL;
3930         goto out;
3931 }
3932
3933 static int trace_action_create(struct hist_trigger_data *hist_data,
3934                                struct action_data *data)
3935 {
3936         struct trace_array *tr = hist_data->event_file->tr;
3937         char *event_name, *param, *system = NULL;
3938         struct hist_field *hist_field, *var_ref;
3939         unsigned int i;
3940         unsigned int field_pos = 0;
3941         struct synth_event *event;
3942         char *synth_event_name;
3943         int var_ref_idx, ret = 0;
3944
3945         lockdep_assert_held(&event_mutex);
3946
3947         /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
3948         if (data->n_params > SYNTH_FIELDS_MAX)
3949                 return -EINVAL;
3950
3951         if (data->use_trace_keyword)
3952                 synth_event_name = data->synth_event_name;
3953         else
3954                 synth_event_name = data->action_name;
3955
3956         event = find_synth_event(synth_event_name);
3957         if (!event) {
3958                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3959                 return -EINVAL;
3960         }
3961
3962         event->ref++;
3963
3964         for (i = 0; i < data->n_params; i++) {
3965                 char *p;
3966
3967                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3968                 if (!param) {
3969                         ret = -ENOMEM;
3970                         goto err;
3971                 }
3972
3973                 system = strsep(&param, ".");
3974                 if (!param) {
3975                         param = (char *)system;
3976                         system = event_name = NULL;
3977                 } else {
3978                         event_name = strsep(&param, ".");
3979                         if (!param) {
3980                                 kfree(p);
3981                                 ret = -EINVAL;
3982                                 goto err;
3983                         }
3984                 }
3985
3986                 if (param[0] == '$')
3987                         hist_field = trace_action_find_var(hist_data, data,
3988                                                            system, event_name,
3989                                                            param);
3990                 else
3991                         hist_field = trace_action_create_field_var(hist_data,
3992                                                                    data,
3993                                                                    system,
3994                                                                    event_name,
3995                                                                    param);
3996
3997                 if (!hist_field) {
3998                         kfree(p);
3999                         ret = -EINVAL;
4000                         goto err;
4001                 }
4002
4003                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4004                         var_ref = create_var_ref(hist_data, hist_field,
4005                                                  system, event_name);
4006                         if (!var_ref) {
4007                                 kfree(p);
4008                                 ret = -ENOMEM;
4009                                 goto err;
4010                         }
4011
4012                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4013                         if (WARN_ON(var_ref_idx < 0)) {
4014                                 kfree(p);
4015                                 ret = var_ref_idx;
4016                                 goto err;
4017                         }
4018
4019                         data->var_ref_idx[i] = var_ref_idx;
4020
4021                         field_pos++;
4022                         kfree(p);
4023                         continue;
4024                 }
4025
4026                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4027                 kfree(p);
4028                 ret = -EINVAL;
4029                 goto err;
4030         }
4031
4032         if (field_pos != event->n_fields) {
4033                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4034                 ret = -EINVAL;
4035                 goto err;
4036         }
4037
4038         data->synth_event = event;
4039  out:
4040         return ret;
4041  err:
4042         event->ref--;
4043
4044         goto out;
4045 }
4046
4047 static int action_create(struct hist_trigger_data *hist_data,
4048                          struct action_data *data)
4049 {
4050         struct trace_event_file *file = hist_data->event_file;
4051         struct trace_array *tr = file->tr;
4052         struct track_data *track_data;
4053         struct field_var *field_var;
4054         unsigned int i;
4055         char *param;
4056         int ret = 0;
4057
4058         if (data->action == ACTION_TRACE)
4059                 return trace_action_create(hist_data, data);
4060
4061         if (data->action == ACTION_SNAPSHOT) {
4062                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4063                 if (IS_ERR(track_data)) {
4064                         ret = PTR_ERR(track_data);
4065                         goto out;
4066                 }
4067
4068                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4069                                                    cond_snapshot_update);
4070                 if (ret)
4071                         track_data_free(track_data);
4072
4073                 goto out;
4074         }
4075
4076         if (data->action == ACTION_SAVE) {
4077                 if (hist_data->n_save_vars) {
4078                         ret = -EEXIST;
4079                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4080                         goto out;
4081                 }
4082
4083                 for (i = 0; i < data->n_params; i++) {
4084                         param = kstrdup(data->params[i], GFP_KERNEL);
4085                         if (!param) {
4086                                 ret = -ENOMEM;
4087                                 goto out;
4088                         }
4089
4090                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4091                         if (IS_ERR(field_var)) {
4092                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4093                                          errpos(param));
4094                                 ret = PTR_ERR(field_var);
4095                                 kfree(param);
4096                                 goto out;
4097                         }
4098
4099                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4100                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
4101                                 hist_data->n_save_var_str++;
4102                         kfree(param);
4103                 }
4104         }
4105  out:
4106         return ret;
4107 }
4108
4109 static int onmatch_create(struct hist_trigger_data *hist_data,
4110                           struct action_data *data)
4111 {
4112         return action_create(hist_data, data);
4113 }
4114
4115 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4116 {
4117         char *match_event, *match_event_system;
4118         struct action_data *data;
4119         int ret = -EINVAL;
4120
4121         data = kzalloc(sizeof(*data), GFP_KERNEL);
4122         if (!data)
4123                 return ERR_PTR(-ENOMEM);
4124
4125         match_event = strsep(&str, ")");
4126         if (!match_event || !str) {
4127                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4128                 goto free;
4129         }
4130
4131         match_event_system = strsep(&match_event, ".");
4132         if (!match_event) {
4133                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4134                 goto free;
4135         }
4136
4137         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4138                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4139                 goto free;
4140         }
4141
4142         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4143         if (!data->match_data.event) {
4144                 ret = -ENOMEM;
4145                 goto free;
4146         }
4147
4148         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4149         if (!data->match_data.event_system) {
4150                 ret = -ENOMEM;
4151                 goto free;
4152         }
4153
4154         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4155         if (ret)
4156                 goto free;
4157  out:
4158         return data;
4159  free:
4160         onmatch_destroy(data);
4161         data = ERR_PTR(ret);
4162         goto out;
4163 }
4164
4165 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4166 {
4167         hist_data->fields[HITCOUNT_IDX] =
4168                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4169         if (!hist_data->fields[HITCOUNT_IDX])
4170                 return -ENOMEM;
4171
4172         hist_data->n_vals++;
4173         hist_data->n_fields++;
4174
4175         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4176                 return -EINVAL;
4177
4178         return 0;
4179 }
4180
4181 static int __create_val_field(struct hist_trigger_data *hist_data,
4182                               unsigned int val_idx,
4183                               struct trace_event_file *file,
4184                               char *var_name, char *field_str,
4185                               unsigned long flags)
4186 {
4187         struct hist_field *hist_field;
4188         int ret = 0, n_subexprs = 0;
4189
4190         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4191         if (IS_ERR(hist_field)) {
4192                 ret = PTR_ERR(hist_field);
4193                 goto out;
4194         }
4195
4196         /* values and variables should not have some modifiers */
4197         if (hist_field->flags & HIST_FIELD_FL_VAR) {
4198                 /* Variable */
4199                 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4200                                          HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2))
4201                         goto err;
4202         } else {
4203                 /* Value */
4204                 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4205                                          HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 |
4206                                          HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET |
4207                                          HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE))
4208                         goto err;
4209         }
4210
4211         hist_data->fields[val_idx] = hist_field;
4212
4213         ++hist_data->n_vals;
4214         ++hist_data->n_fields;
4215
4216         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4217                 ret = -EINVAL;
4218  out:
4219         return ret;
4220  err:
4221         hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str));
4222         return -EINVAL;
4223 }
4224
4225 static int create_val_field(struct hist_trigger_data *hist_data,
4226                             unsigned int val_idx,
4227                             struct trace_event_file *file,
4228                             char *field_str)
4229 {
4230         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4231                 return -EINVAL;
4232
4233         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4234 }
4235
4236 static const char no_comm[] = "(no comm)";
4237
4238 static u64 hist_field_execname(struct hist_field *hist_field,
4239                                struct tracing_map_elt *elt,
4240                                struct trace_buffer *buffer,
4241                                struct ring_buffer_event *rbe,
4242                                void *event)
4243 {
4244         struct hist_elt_data *elt_data;
4245
4246         if (WARN_ON_ONCE(!elt))
4247                 return (u64)(unsigned long)no_comm;
4248
4249         elt_data = elt->private_data;
4250
4251         if (WARN_ON_ONCE(!elt_data->comm))
4252                 return (u64)(unsigned long)no_comm;
4253
4254         return (u64)(unsigned long)(elt_data->comm);
4255 }
4256
4257 static u64 hist_fn_call(struct hist_field *hist_field,
4258                         struct tracing_map_elt *elt,
4259                         struct trace_buffer *buffer,
4260                         struct ring_buffer_event *rbe,
4261                         void *event)
4262 {
4263         switch (hist_field->fn_num) {
4264         case HIST_FIELD_FN_VAR_REF:
4265                 return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4266         case HIST_FIELD_FN_COUNTER:
4267                 return hist_field_counter(hist_field, elt, buffer, rbe, event);
4268         case HIST_FIELD_FN_CONST:
4269                 return hist_field_const(hist_field, elt, buffer, rbe, event);
4270         case HIST_FIELD_FN_LOG2:
4271                 return hist_field_log2(hist_field, elt, buffer, rbe, event);
4272         case HIST_FIELD_FN_BUCKET:
4273                 return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4274         case HIST_FIELD_FN_TIMESTAMP:
4275                 return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4276         case HIST_FIELD_FN_CPU:
4277                 return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4278         case HIST_FIELD_FN_STRING:
4279                 return hist_field_string(hist_field, elt, buffer, rbe, event);
4280         case HIST_FIELD_FN_DYNSTRING:
4281                 return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4282         case HIST_FIELD_FN_RELDYNSTRING:
4283                 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4284         case HIST_FIELD_FN_PSTRING:
4285                 return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4286         case HIST_FIELD_FN_S64:
4287                 return hist_field_s64(hist_field, elt, buffer, rbe, event);
4288         case HIST_FIELD_FN_U64:
4289                 return hist_field_u64(hist_field, elt, buffer, rbe, event);
4290         case HIST_FIELD_FN_S32:
4291                 return hist_field_s32(hist_field, elt, buffer, rbe, event);
4292         case HIST_FIELD_FN_U32:
4293                 return hist_field_u32(hist_field, elt, buffer, rbe, event);
4294         case HIST_FIELD_FN_S16:
4295                 return hist_field_s16(hist_field, elt, buffer, rbe, event);
4296         case HIST_FIELD_FN_U16:
4297                 return hist_field_u16(hist_field, elt, buffer, rbe, event);
4298         case HIST_FIELD_FN_S8:
4299                 return hist_field_s8(hist_field, elt, buffer, rbe, event);
4300         case HIST_FIELD_FN_U8:
4301                 return hist_field_u8(hist_field, elt, buffer, rbe, event);
4302         case HIST_FIELD_FN_UMINUS:
4303                 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4304         case HIST_FIELD_FN_MINUS:
4305                 return hist_field_minus(hist_field, elt, buffer, rbe, event);
4306         case HIST_FIELD_FN_PLUS:
4307                 return hist_field_plus(hist_field, elt, buffer, rbe, event);
4308         case HIST_FIELD_FN_DIV:
4309                 return hist_field_div(hist_field, elt, buffer, rbe, event);
4310         case HIST_FIELD_FN_MULT:
4311                 return hist_field_mult(hist_field, elt, buffer, rbe, event);
4312         case HIST_FIELD_FN_DIV_POWER2:
4313                 return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4314         case HIST_FIELD_FN_DIV_NOT_POWER2:
4315                 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4316         case HIST_FIELD_FN_DIV_MULT_SHIFT:
4317                 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4318         case HIST_FIELD_FN_EXECNAME:
4319                 return hist_field_execname(hist_field, elt, buffer, rbe, event);
4320         default:
4321                 return 0;
4322         }
4323 }
4324
4325 /* Convert a var that points to common_pid.execname to a string */
4326 static void update_var_execname(struct hist_field *hist_field)
4327 {
4328         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4329                 HIST_FIELD_FL_EXECNAME;
4330         hist_field->size = MAX_FILTER_STR_VAL;
4331         hist_field->is_signed = 0;
4332
4333         kfree_const(hist_field->type);
4334         hist_field->type = "char[]";
4335
4336         hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4337 }
4338
4339 static int create_var_field(struct hist_trigger_data *hist_data,
4340                             unsigned int val_idx,
4341                             struct trace_event_file *file,
4342                             char *var_name, char *expr_str)
4343 {
4344         struct trace_array *tr = hist_data->event_file->tr;
4345         unsigned long flags = 0;
4346         int ret;
4347
4348         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4349                 return -EINVAL;
4350
4351         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4352                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4353                 return -EINVAL;
4354         }
4355
4356         flags |= HIST_FIELD_FL_VAR;
4357         hist_data->n_vars++;
4358         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4359                 return -EINVAL;
4360
4361         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4362
4363         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4364                 update_var_execname(hist_data->fields[val_idx]);
4365
4366         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
4367                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4368
4369         return ret;
4370 }
4371
4372 static int create_val_fields(struct hist_trigger_data *hist_data,
4373                              struct trace_event_file *file)
4374 {
4375         char *fields_str, *field_str;
4376         unsigned int i, j = 1;
4377         int ret;
4378
4379         ret = create_hitcount_val(hist_data);
4380         if (ret)
4381                 goto out;
4382
4383         fields_str = hist_data->attrs->vals_str;
4384         if (!fields_str)
4385                 goto out;
4386
4387         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4388                      j < TRACING_MAP_VALS_MAX; i++) {
4389                 field_str = strsep(&fields_str, ",");
4390                 if (!field_str)
4391                         break;
4392
4393                 if (strcmp(field_str, "hitcount") == 0)
4394                         continue;
4395
4396                 ret = create_val_field(hist_data, j++, file, field_str);
4397                 if (ret)
4398                         goto out;
4399         }
4400
4401         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4402                 ret = -EINVAL;
4403  out:
4404         return ret;
4405 }
4406
4407 static int create_key_field(struct hist_trigger_data *hist_data,
4408                             unsigned int key_idx,
4409                             unsigned int key_offset,
4410                             struct trace_event_file *file,
4411                             char *field_str)
4412 {
4413         struct trace_array *tr = hist_data->event_file->tr;
4414         struct hist_field *hist_field = NULL;
4415         unsigned long flags = 0;
4416         unsigned int key_size;
4417         int ret = 0, n_subexprs = 0;
4418
4419         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4420                 return -EINVAL;
4421
4422         flags |= HIST_FIELD_FL_KEY;
4423
4424         if (strcmp(field_str, "stacktrace") == 0) {
4425                 flags |= HIST_FIELD_FL_STACKTRACE;
4426                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4427                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4428         } else {
4429                 hist_field = parse_expr(hist_data, file, field_str, flags,
4430                                         NULL, &n_subexprs);
4431                 if (IS_ERR(hist_field)) {
4432                         ret = PTR_ERR(hist_field);
4433                         goto out;
4434                 }
4435
4436                 if (field_has_hist_vars(hist_field, 0)) {
4437                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4438                         destroy_hist_field(hist_field, 0);
4439                         ret = -EINVAL;
4440                         goto out;
4441                 }
4442
4443                 key_size = hist_field->size;
4444         }
4445
4446         hist_data->fields[key_idx] = hist_field;
4447
4448         key_size = ALIGN(key_size, sizeof(u64));
4449         hist_data->fields[key_idx]->size = key_size;
4450         hist_data->fields[key_idx]->offset = key_offset;
4451
4452         hist_data->key_size += key_size;
4453
4454         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4455                 ret = -EINVAL;
4456                 goto out;
4457         }
4458
4459         hist_data->n_keys++;
4460         hist_data->n_fields++;
4461
4462         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4463                 return -EINVAL;
4464
4465         ret = key_size;
4466  out:
4467         return ret;
4468 }
4469
4470 static int create_key_fields(struct hist_trigger_data *hist_data,
4471                              struct trace_event_file *file)
4472 {
4473         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4474         char *fields_str, *field_str;
4475         int ret = -EINVAL;
4476
4477         fields_str = hist_data->attrs->keys_str;
4478         if (!fields_str)
4479                 goto out;
4480
4481         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4482                 field_str = strsep(&fields_str, ",");
4483                 if (!field_str)
4484                         break;
4485                 ret = create_key_field(hist_data, i, key_offset,
4486                                        file, field_str);
4487                 if (ret < 0)
4488                         goto out;
4489                 key_offset += ret;
4490         }
4491         if (fields_str) {
4492                 ret = -EINVAL;
4493                 goto out;
4494         }
4495         ret = 0;
4496  out:
4497         return ret;
4498 }
4499
4500 static int create_var_fields(struct hist_trigger_data *hist_data,
4501                              struct trace_event_file *file)
4502 {
4503         unsigned int i, j = hist_data->n_vals;
4504         int ret = 0;
4505
4506         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4507
4508         for (i = 0; i < n_vars; i++) {
4509                 char *var_name = hist_data->attrs->var_defs.name[i];
4510                 char *expr = hist_data->attrs->var_defs.expr[i];
4511
4512                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4513                 if (ret)
4514                         goto out;
4515         }
4516  out:
4517         return ret;
4518 }
4519
4520 static void free_var_defs(struct hist_trigger_data *hist_data)
4521 {
4522         unsigned int i;
4523
4524         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4525                 kfree(hist_data->attrs->var_defs.name[i]);
4526                 kfree(hist_data->attrs->var_defs.expr[i]);
4527         }
4528
4529         hist_data->attrs->var_defs.n_vars = 0;
4530 }
4531
4532 static int parse_var_defs(struct hist_trigger_data *hist_data)
4533 {
4534         struct trace_array *tr = hist_data->event_file->tr;
4535         char *s, *str, *var_name, *field_str;
4536         unsigned int i, j, n_vars = 0;
4537         int ret = 0;
4538
4539         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4540                 str = hist_data->attrs->assignment_str[i];
4541                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4542                         field_str = strsep(&str, ",");
4543                         if (!field_str)
4544                                 break;
4545
4546                         var_name = strsep(&field_str, "=");
4547                         if (!var_name || !field_str) {
4548                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4549                                          errpos(var_name));
4550                                 ret = -EINVAL;
4551                                 goto free;
4552                         }
4553
4554                         if (n_vars == TRACING_MAP_VARS_MAX) {
4555                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4556                                 ret = -EINVAL;
4557                                 goto free;
4558                         }
4559
4560                         s = kstrdup(var_name, GFP_KERNEL);
4561                         if (!s) {
4562                                 ret = -ENOMEM;
4563                                 goto free;
4564                         }
4565                         hist_data->attrs->var_defs.name[n_vars] = s;
4566
4567                         s = kstrdup(field_str, GFP_KERNEL);
4568                         if (!s) {
4569                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4570                                 hist_data->attrs->var_defs.name[n_vars] = NULL;
4571                                 ret = -ENOMEM;
4572                                 goto free;
4573                         }
4574                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4575
4576                         hist_data->attrs->var_defs.n_vars = n_vars;
4577                 }
4578         }
4579
4580         return ret;
4581  free:
4582         free_var_defs(hist_data);
4583
4584         return ret;
4585 }
4586
4587 static int create_hist_fields(struct hist_trigger_data *hist_data,
4588                               struct trace_event_file *file)
4589 {
4590         int ret;
4591
4592         ret = parse_var_defs(hist_data);
4593         if (ret)
4594                 return ret;
4595
4596         ret = create_val_fields(hist_data, file);
4597         if (ret)
4598                 goto out;
4599
4600         ret = create_var_fields(hist_data, file);
4601         if (ret)
4602                 goto out;
4603
4604         ret = create_key_fields(hist_data, file);
4605
4606  out:
4607         free_var_defs(hist_data);
4608
4609         return ret;
4610 }
4611
4612 static int is_descending(struct trace_array *tr, const char *str)
4613 {
4614         if (!str)
4615                 return 0;
4616
4617         if (strcmp(str, "descending") == 0)
4618                 return 1;
4619
4620         if (strcmp(str, "ascending") == 0)
4621                 return 0;
4622
4623         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4624
4625         return -EINVAL;
4626 }
4627
4628 static int create_sort_keys(struct hist_trigger_data *hist_data)
4629 {
4630         struct trace_array *tr = hist_data->event_file->tr;
4631         char *fields_str = hist_data->attrs->sort_key_str;
4632         struct tracing_map_sort_key *sort_key;
4633         int descending, ret = 0;
4634         unsigned int i, j, k;
4635
4636         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4637
4638         if (!fields_str)
4639                 goto out;
4640
4641         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4642                 struct hist_field *hist_field;
4643                 char *field_str, *field_name;
4644                 const char *test_name;
4645
4646                 sort_key = &hist_data->sort_keys[i];
4647
4648                 field_str = strsep(&fields_str, ",");
4649                 if (!field_str)
4650                         break;
4651
4652                 if (!*field_str) {
4653                         ret = -EINVAL;
4654                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4655                         break;
4656                 }
4657
4658                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4659                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4660                         ret = -EINVAL;
4661                         break;
4662                 }
4663
4664                 field_name = strsep(&field_str, ".");
4665                 if (!field_name || !*field_name) {
4666                         ret = -EINVAL;
4667                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4668                         break;
4669                 }
4670
4671                 if (strcmp(field_name, "hitcount") == 0) {
4672                         descending = is_descending(tr, field_str);
4673                         if (descending < 0) {
4674                                 ret = descending;
4675                                 break;
4676                         }
4677                         sort_key->descending = descending;
4678                         continue;
4679                 }
4680
4681                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4682                         unsigned int idx;
4683
4684                         hist_field = hist_data->fields[j];
4685                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4686                                 continue;
4687
4688                         idx = k++;
4689
4690                         test_name = hist_field_name(hist_field, 0);
4691
4692                         if (strcmp(field_name, test_name) == 0) {
4693                                 sort_key->field_idx = idx;
4694                                 descending = is_descending(tr, field_str);
4695                                 if (descending < 0) {
4696                                         ret = descending;
4697                                         goto out;
4698                                 }
4699                                 sort_key->descending = descending;
4700                                 break;
4701                         }
4702                 }
4703                 if (j == hist_data->n_fields) {
4704                         ret = -EINVAL;
4705                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4706                         break;
4707                 }
4708         }
4709
4710         hist_data->n_sort_keys = i;
4711  out:
4712         return ret;
4713 }
4714
4715 static void destroy_actions(struct hist_trigger_data *hist_data)
4716 {
4717         unsigned int i;
4718
4719         for (i = 0; i < hist_data->n_actions; i++) {
4720                 struct action_data *data = hist_data->actions[i];
4721
4722                 if (data->handler == HANDLER_ONMATCH)
4723                         onmatch_destroy(data);
4724                 else if (data->handler == HANDLER_ONMAX ||
4725                          data->handler == HANDLER_ONCHANGE)
4726                         track_data_destroy(hist_data, data);
4727                 else
4728                         kfree(data);
4729         }
4730 }
4731
4732 static int parse_actions(struct hist_trigger_data *hist_data)
4733 {
4734         struct trace_array *tr = hist_data->event_file->tr;
4735         struct action_data *data;
4736         unsigned int i;
4737         int ret = 0;
4738         char *str;
4739         int len;
4740
4741         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4742                 str = hist_data->attrs->action_str[i];
4743
4744                 if ((len = str_has_prefix(str, "onmatch("))) {
4745                         char *action_str = str + len;
4746
4747                         data = onmatch_parse(tr, action_str);
4748                         if (IS_ERR(data)) {
4749                                 ret = PTR_ERR(data);
4750                                 break;
4751                         }
4752                 } else if ((len = str_has_prefix(str, "onmax("))) {
4753                         char *action_str = str + len;
4754
4755                         data = track_data_parse(hist_data, action_str,
4756                                                 HANDLER_ONMAX);
4757                         if (IS_ERR(data)) {
4758                                 ret = PTR_ERR(data);
4759                                 break;
4760                         }
4761                 } else if ((len = str_has_prefix(str, "onchange("))) {
4762                         char *action_str = str + len;
4763
4764                         data = track_data_parse(hist_data, action_str,
4765                                                 HANDLER_ONCHANGE);
4766                         if (IS_ERR(data)) {
4767                                 ret = PTR_ERR(data);
4768                                 break;
4769                         }
4770                 } else {
4771                         ret = -EINVAL;
4772                         break;
4773                 }
4774
4775                 hist_data->actions[hist_data->n_actions++] = data;
4776         }
4777
4778         return ret;
4779 }
4780
4781 static int create_actions(struct hist_trigger_data *hist_data)
4782 {
4783         struct action_data *data;
4784         unsigned int i;
4785         int ret = 0;
4786
4787         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4788                 data = hist_data->actions[i];
4789
4790                 if (data->handler == HANDLER_ONMATCH) {
4791                         ret = onmatch_create(hist_data, data);
4792                         if (ret)
4793                                 break;
4794                 } else if (data->handler == HANDLER_ONMAX ||
4795                            data->handler == HANDLER_ONCHANGE) {
4796                         ret = track_data_create(hist_data, data);
4797                         if (ret)
4798                                 break;
4799                 } else {
4800                         ret = -EINVAL;
4801                         break;
4802                 }
4803         }
4804
4805         return ret;
4806 }
4807
4808 static void print_actions(struct seq_file *m,
4809                           struct hist_trigger_data *hist_data,
4810                           struct tracing_map_elt *elt)
4811 {
4812         unsigned int i;
4813
4814         for (i = 0; i < hist_data->n_actions; i++) {
4815                 struct action_data *data = hist_data->actions[i];
4816
4817                 if (data->action == ACTION_SNAPSHOT)
4818                         continue;
4819
4820                 if (data->handler == HANDLER_ONMAX ||
4821                     data->handler == HANDLER_ONCHANGE)
4822                         track_data_print(m, hist_data, elt, data);
4823         }
4824 }
4825
4826 static void print_action_spec(struct seq_file *m,
4827                               struct hist_trigger_data *hist_data,
4828                               struct action_data *data)
4829 {
4830         unsigned int i;
4831
4832         if (data->action == ACTION_SAVE) {
4833                 for (i = 0; i < hist_data->n_save_vars; i++) {
4834                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4835                         if (i < hist_data->n_save_vars - 1)
4836                                 seq_puts(m, ",");
4837                 }
4838         } else if (data->action == ACTION_TRACE) {
4839                 if (data->use_trace_keyword)
4840                         seq_printf(m, "%s", data->synth_event_name);
4841                 for (i = 0; i < data->n_params; i++) {
4842                         if (i || data->use_trace_keyword)
4843                                 seq_puts(m, ",");
4844                         seq_printf(m, "%s", data->params[i]);
4845                 }
4846         }
4847 }
4848
4849 static void print_track_data_spec(struct seq_file *m,
4850                                   struct hist_trigger_data *hist_data,
4851                                   struct action_data *data)
4852 {
4853         if (data->handler == HANDLER_ONMAX)
4854                 seq_puts(m, ":onmax(");
4855         else if (data->handler == HANDLER_ONCHANGE)
4856                 seq_puts(m, ":onchange(");
4857         seq_printf(m, "%s", data->track_data.var_str);
4858         seq_printf(m, ").%s(", data->action_name);
4859
4860         print_action_spec(m, hist_data, data);
4861
4862         seq_puts(m, ")");
4863 }
4864
4865 static void print_onmatch_spec(struct seq_file *m,
4866                                struct hist_trigger_data *hist_data,
4867                                struct action_data *data)
4868 {
4869         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4870                    data->match_data.event);
4871
4872         seq_printf(m, "%s(", data->action_name);
4873
4874         print_action_spec(m, hist_data, data);
4875
4876         seq_puts(m, ")");
4877 }
4878
4879 static bool actions_match(struct hist_trigger_data *hist_data,
4880                           struct hist_trigger_data *hist_data_test)
4881 {
4882         unsigned int i, j;
4883
4884         if (hist_data->n_actions != hist_data_test->n_actions)
4885                 return false;
4886
4887         for (i = 0; i < hist_data->n_actions; i++) {
4888                 struct action_data *data = hist_data->actions[i];
4889                 struct action_data *data_test = hist_data_test->actions[i];
4890                 char *action_name, *action_name_test;
4891
4892                 if (data->handler != data_test->handler)
4893                         return false;
4894                 if (data->action != data_test->action)
4895                         return false;
4896
4897                 if (data->n_params != data_test->n_params)
4898                         return false;
4899
4900                 for (j = 0; j < data->n_params; j++) {
4901                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4902                                 return false;
4903                 }
4904
4905                 if (data->use_trace_keyword)
4906                         action_name = data->synth_event_name;
4907                 else
4908                         action_name = data->action_name;
4909
4910                 if (data_test->use_trace_keyword)
4911                         action_name_test = data_test->synth_event_name;
4912                 else
4913                         action_name_test = data_test->action_name;
4914
4915                 if (strcmp(action_name, action_name_test) != 0)
4916                         return false;
4917
4918                 if (data->handler == HANDLER_ONMATCH) {
4919                         if (strcmp(data->match_data.event_system,
4920                                    data_test->match_data.event_system) != 0)
4921                                 return false;
4922                         if (strcmp(data->match_data.event,
4923                                    data_test->match_data.event) != 0)
4924                                 return false;
4925                 } else if (data->handler == HANDLER_ONMAX ||
4926                            data->handler == HANDLER_ONCHANGE) {
4927                         if (strcmp(data->track_data.var_str,
4928                                    data_test->track_data.var_str) != 0)
4929                                 return false;
4930                 }
4931         }
4932
4933         return true;
4934 }
4935
4936
4937 static void print_actions_spec(struct seq_file *m,
4938                                struct hist_trigger_data *hist_data)
4939 {
4940         unsigned int i;
4941
4942         for (i = 0; i < hist_data->n_actions; i++) {
4943                 struct action_data *data = hist_data->actions[i];
4944
4945                 if (data->handler == HANDLER_ONMATCH)
4946                         print_onmatch_spec(m, hist_data, data);
4947                 else if (data->handler == HANDLER_ONMAX ||
4948                          data->handler == HANDLER_ONCHANGE)
4949                         print_track_data_spec(m, hist_data, data);
4950         }
4951 }
4952
4953 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4954 {
4955         unsigned int i;
4956
4957         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4958                 kfree(hist_data->field_var_hists[i]->cmd);
4959                 kfree(hist_data->field_var_hists[i]);
4960         }
4961 }
4962
4963 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4964 {
4965         if (!hist_data)
4966                 return;
4967
4968         destroy_hist_trigger_attrs(hist_data->attrs);
4969         destroy_hist_fields(hist_data);
4970         tracing_map_destroy(hist_data->map);
4971
4972         destroy_actions(hist_data);
4973         destroy_field_vars(hist_data);
4974         destroy_field_var_hists(hist_data);
4975
4976         kfree(hist_data);
4977 }
4978
4979 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4980 {
4981         struct tracing_map *map = hist_data->map;
4982         struct ftrace_event_field *field;
4983         struct hist_field *hist_field;
4984         int i, idx = 0;
4985
4986         for_each_hist_field(i, hist_data) {
4987                 hist_field = hist_data->fields[i];
4988                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4989                         tracing_map_cmp_fn_t cmp_fn;
4990
4991                         field = hist_field->field;
4992
4993                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4994                                 cmp_fn = tracing_map_cmp_none;
4995                         else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
4996                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4997                                                              hist_field->is_signed);
4998                         else if (is_string_field(field))
4999                                 cmp_fn = tracing_map_cmp_string;
5000                         else
5001                                 cmp_fn = tracing_map_cmp_num(field->size,
5002                                                              field->is_signed);
5003                         idx = tracing_map_add_key_field(map,
5004                                                         hist_field->offset,
5005                                                         cmp_fn);
5006                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5007                         idx = tracing_map_add_sum_field(map);
5008
5009                 if (idx < 0)
5010                         return idx;
5011
5012                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5013                         idx = tracing_map_add_var(map);
5014                         if (idx < 0)
5015                                 return idx;
5016                         hist_field->var.idx = idx;
5017                         hist_field->var.hist_data = hist_data;
5018                 }
5019         }
5020
5021         return 0;
5022 }
5023
5024 static struct hist_trigger_data *
5025 create_hist_data(unsigned int map_bits,
5026                  struct hist_trigger_attrs *attrs,
5027                  struct trace_event_file *file,
5028                  bool remove)
5029 {
5030         const struct tracing_map_ops *map_ops = NULL;
5031         struct hist_trigger_data *hist_data;
5032         int ret = 0;
5033
5034         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5035         if (!hist_data)
5036                 return ERR_PTR(-ENOMEM);
5037
5038         hist_data->attrs = attrs;
5039         hist_data->remove = remove;
5040         hist_data->event_file = file;
5041
5042         ret = parse_actions(hist_data);
5043         if (ret)
5044                 goto free;
5045
5046         ret = create_hist_fields(hist_data, file);
5047         if (ret)
5048                 goto free;
5049
5050         ret = create_sort_keys(hist_data);
5051         if (ret)
5052                 goto free;
5053
5054         map_ops = &hist_trigger_elt_data_ops;
5055
5056         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5057                                             map_ops, hist_data);
5058         if (IS_ERR(hist_data->map)) {
5059                 ret = PTR_ERR(hist_data->map);
5060                 hist_data->map = NULL;
5061                 goto free;
5062         }
5063
5064         ret = create_tracing_map_fields(hist_data);
5065         if (ret)
5066                 goto free;
5067  out:
5068         return hist_data;
5069  free:
5070         hist_data->attrs = NULL;
5071
5072         destroy_hist_data(hist_data);
5073
5074         hist_data = ERR_PTR(ret);
5075
5076         goto out;
5077 }
5078
5079 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5080                                     struct tracing_map_elt *elt,
5081                                     struct trace_buffer *buffer, void *rec,
5082                                     struct ring_buffer_event *rbe,
5083                                     u64 *var_ref_vals)
5084 {
5085         struct hist_elt_data *elt_data;
5086         struct hist_field *hist_field;
5087         unsigned int i, var_idx;
5088         u64 hist_val;
5089
5090         elt_data = elt->private_data;
5091         elt_data->var_ref_vals = var_ref_vals;
5092
5093         for_each_hist_val_field(i, hist_data) {
5094                 hist_field = hist_data->fields[i];
5095                 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5096                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5097                         var_idx = hist_field->var.idx;
5098
5099                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
5100                                 unsigned int str_start, var_str_idx, idx;
5101                                 char *str, *val_str;
5102                                 unsigned int size;
5103
5104                                 str_start = hist_data->n_field_var_str +
5105                                         hist_data->n_save_var_str;
5106                                 var_str_idx = hist_field->var_str_idx;
5107                                 idx = str_start + var_str_idx;
5108
5109                                 str = elt_data->field_var_str[idx];
5110                                 val_str = (char *)(uintptr_t)hist_val;
5111
5112                                 size = min(hist_field->size, STR_VAR_LEN_MAX);
5113                                 strscpy(str, val_str, size);
5114
5115                                 hist_val = (u64)(uintptr_t)str;
5116                         }
5117                         tracing_map_set_var(elt, var_idx, hist_val);
5118                         continue;
5119                 }
5120                 tracing_map_update_sum(elt, i, hist_val);
5121         }
5122
5123         for_each_hist_key_field(i, hist_data) {
5124                 hist_field = hist_data->fields[i];
5125                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5126                         hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5127                         var_idx = hist_field->var.idx;
5128                         tracing_map_set_var(elt, var_idx, hist_val);
5129                 }
5130         }
5131
5132         update_field_vars(hist_data, elt, buffer, rbe, rec);
5133 }
5134
5135 static inline void add_to_key(char *compound_key, void *key,
5136                               struct hist_field *key_field, void *rec)
5137 {
5138         size_t size = key_field->size;
5139
5140         if (key_field->flags & HIST_FIELD_FL_STRING) {
5141                 struct ftrace_event_field *field;
5142
5143                 field = key_field->field;
5144                 if (field->filter_type == FILTER_DYN_STRING ||
5145                     field->filter_type == FILTER_RDYN_STRING)
5146                         size = *(u32 *)(rec + field->offset) >> 16;
5147                 else if (field->filter_type == FILTER_STATIC_STRING)
5148                         size = field->size;
5149
5150                 /* ensure NULL-termination */
5151                 if (size > key_field->size - 1)
5152                         size = key_field->size - 1;
5153
5154                 strncpy(compound_key + key_field->offset, (char *)key, size);
5155         } else
5156                 memcpy(compound_key + key_field->offset, key, size);
5157 }
5158
5159 static void
5160 hist_trigger_actions(struct hist_trigger_data *hist_data,
5161                      struct tracing_map_elt *elt,
5162                      struct trace_buffer *buffer, void *rec,
5163                      struct ring_buffer_event *rbe, void *key,
5164                      u64 *var_ref_vals)
5165 {
5166         struct action_data *data;
5167         unsigned int i;
5168
5169         for (i = 0; i < hist_data->n_actions; i++) {
5170                 data = hist_data->actions[i];
5171                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5172         }
5173 }
5174
5175 static void event_hist_trigger(struct event_trigger_data *data,
5176                                struct trace_buffer *buffer, void *rec,
5177                                struct ring_buffer_event *rbe)
5178 {
5179         struct hist_trigger_data *hist_data = data->private_data;
5180         bool use_compound_key = (hist_data->n_keys > 1);
5181         unsigned long entries[HIST_STACKTRACE_DEPTH];
5182         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5183         char compound_key[HIST_KEY_SIZE_MAX];
5184         struct tracing_map_elt *elt = NULL;
5185         struct hist_field *key_field;
5186         u64 field_contents;
5187         void *key = NULL;
5188         unsigned int i;
5189
5190         if (unlikely(!rbe))
5191                 return;
5192
5193         memset(compound_key, 0, hist_data->key_size);
5194
5195         for_each_hist_key_field(i, hist_data) {
5196                 key_field = hist_data->fields[i];
5197
5198                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5199                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5200                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5201                                          HIST_STACKTRACE_SKIP);
5202                         key = entries;
5203                 } else {
5204                         field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5205                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5206                                 key = (void *)(unsigned long)field_contents;
5207                                 use_compound_key = true;
5208                         } else
5209                                 key = (void *)&field_contents;
5210                 }
5211
5212                 if (use_compound_key)
5213                         add_to_key(compound_key, key, key_field, rec);
5214         }
5215
5216         if (use_compound_key)
5217                 key = compound_key;
5218
5219         if (hist_data->n_var_refs &&
5220             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5221                 return;
5222
5223         elt = tracing_map_insert(hist_data->map, key);
5224         if (!elt)
5225                 return;
5226
5227         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5228
5229         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5230                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5231 }
5232
5233 static void hist_trigger_stacktrace_print(struct seq_file *m,
5234                                           unsigned long *stacktrace_entries,
5235                                           unsigned int max_entries)
5236 {
5237         unsigned int spaces = 8;
5238         unsigned int i;
5239
5240         for (i = 0; i < max_entries; i++) {
5241                 if (!stacktrace_entries[i])
5242                         return;
5243
5244                 seq_printf(m, "%*c", 1 + spaces, ' ');
5245                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5246         }
5247 }
5248
5249 static void hist_trigger_print_key(struct seq_file *m,
5250                                    struct hist_trigger_data *hist_data,
5251                                    void *key,
5252                                    struct tracing_map_elt *elt)
5253 {
5254         struct hist_field *key_field;
5255         bool multiline = false;
5256         const char *field_name;
5257         unsigned int i;
5258         u64 uval;
5259
5260         seq_puts(m, "{ ");
5261
5262         for_each_hist_key_field(i, hist_data) {
5263                 key_field = hist_data->fields[i];
5264
5265                 if (i > hist_data->n_vals)
5266                         seq_puts(m, ", ");
5267
5268                 field_name = hist_field_name(key_field, 0);
5269
5270                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5271                         uval = *(u64 *)(key + key_field->offset);
5272                         seq_printf(m, "%s: %llx", field_name, uval);
5273                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5274                         uval = *(u64 *)(key + key_field->offset);
5275                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5276                                    uval, (void *)(uintptr_t)uval);
5277                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5278                         uval = *(u64 *)(key + key_field->offset);
5279                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5280                                    uval, (void *)(uintptr_t)uval);
5281                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5282                         struct hist_elt_data *elt_data = elt->private_data;
5283                         char *comm;
5284
5285                         if (WARN_ON_ONCE(!elt_data))
5286                                 return;
5287
5288                         comm = elt_data->comm;
5289
5290                         uval = *(u64 *)(key + key_field->offset);
5291                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5292                                    comm, uval);
5293                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5294                         const char *syscall_name;
5295
5296                         uval = *(u64 *)(key + key_field->offset);
5297                         syscall_name = get_syscall_name(uval);
5298                         if (!syscall_name)
5299                                 syscall_name = "unknown_syscall";
5300
5301                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5302                                    syscall_name, uval);
5303                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5304                         seq_puts(m, "stacktrace:\n");
5305                         hist_trigger_stacktrace_print(m,
5306                                                       key + key_field->offset,
5307                                                       HIST_STACKTRACE_DEPTH);
5308                         multiline = true;
5309                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5310                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5311                                    *(u64 *)(key + key_field->offset));
5312                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5313                         unsigned long buckets = key_field->buckets;
5314                         uval = *(u64 *)(key + key_field->offset);
5315                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5316                                    uval, uval + buckets -1);
5317                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5318                         seq_printf(m, "%s: %-50s", field_name,
5319                                    (char *)(key + key_field->offset));
5320                 } else {
5321                         uval = *(u64 *)(key + key_field->offset);
5322                         seq_printf(m, "%s: %10llu", field_name, uval);
5323                 }
5324         }
5325
5326         if (!multiline)
5327                 seq_puts(m, " ");
5328
5329         seq_puts(m, "}");
5330 }
5331
5332 /* Get the 100 times of the percentage of @val in @total */
5333 static inline unsigned int __get_percentage(u64 val, u64 total)
5334 {
5335         if (!total)
5336                 goto div0;
5337
5338         if (val < (U64_MAX / 10000))
5339                 return (unsigned int)div64_ul(val * 10000, total);
5340
5341         total = div64_u64(total, 10000);
5342         if (!total)
5343                 goto div0;
5344
5345         return (unsigned int)div64_ul(val, total);
5346 div0:
5347         return val ? UINT_MAX : 0;
5348 }
5349
5350 #define BAR_CHAR '#'
5351
5352 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5353 {
5354         unsigned int len = __get_percentage(val, max);
5355         int i;
5356
5357         if (len == UINT_MAX) {
5358                 snprintf(buf, size, "[ERROR]");
5359                 return buf;
5360         }
5361
5362         len = len * size / 10000;
5363         for (i = 0; i < len && i < size; i++)
5364                 buf[i] = BAR_CHAR;
5365         while (i < size)
5366                 buf[i++] = ' ';
5367         buf[size] = '\0';
5368
5369         return buf;
5370 }
5371
5372 struct hist_val_stat {
5373         u64 max;
5374         u64 total;
5375 };
5376
5377 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5378                                    const char *field_name, unsigned long flags,
5379                                    struct hist_val_stat *stats,
5380                                    struct tracing_map_elt *elt)
5381 {
5382         u64 val = tracing_map_read_sum(elt, idx);
5383         unsigned int pc;
5384         char bar[21];
5385
5386         if (flags & HIST_FIELD_FL_PERCENT) {
5387                 pc = __get_percentage(val, stats[idx].total);
5388                 if (pc == UINT_MAX)
5389                         seq_printf(m, " %s (%%):[ERROR]", field_name);
5390                 else
5391                         seq_printf(m, " %s (%%): %3u.%02u", field_name,
5392                                         pc / 100, pc % 100);
5393         } else if (flags & HIST_FIELD_FL_GRAPH) {
5394                 seq_printf(m, " %s: %20s", field_name,
5395                            __fill_bar_str(bar, 20, val, stats[idx].max));
5396         } else if (flags & HIST_FIELD_FL_HEX) {
5397                 seq_printf(m, " %s: %10llx", field_name, val);
5398         } else {
5399                 seq_printf(m, " %s: %10llu", field_name, val);
5400         }
5401 }
5402
5403 static void hist_trigger_entry_print(struct seq_file *m,
5404                                      struct hist_trigger_data *hist_data,
5405                                      struct hist_val_stat *stats,
5406                                      void *key,
5407                                      struct tracing_map_elt *elt)
5408 {
5409         const char *field_name;
5410         unsigned int i = HITCOUNT_IDX;
5411         unsigned long flags;
5412
5413         hist_trigger_print_key(m, hist_data, key, elt);
5414
5415         /* At first, show the raw hitcount always */
5416         hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5417
5418         for (i = 1; i < hist_data->n_vals; i++) {
5419                 field_name = hist_field_name(hist_data->fields[i], 0);
5420                 flags = hist_data->fields[i]->flags;
5421
5422                 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
5423                         continue;
5424
5425                 seq_puts(m, " ");
5426                 hist_trigger_print_val(m, i, field_name, flags, stats, elt);
5427         }
5428
5429         print_actions(m, hist_data, elt);
5430
5431         seq_puts(m, "\n");
5432 }
5433
5434 static int print_entries(struct seq_file *m,
5435                          struct hist_trigger_data *hist_data)
5436 {
5437         struct tracing_map_sort_entry **sort_entries = NULL;
5438         struct tracing_map *map = hist_data->map;
5439         int i, j, n_entries;
5440         struct hist_val_stat *stats = NULL;
5441         u64 val;
5442
5443         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5444                                              hist_data->n_sort_keys,
5445                                              &sort_entries);
5446         if (n_entries < 0)
5447                 return n_entries;
5448
5449         /* Calculate the max and the total for each field if needed. */
5450         for (j = 0; j < hist_data->n_vals; j++) {
5451                 if (!(hist_data->fields[j]->flags &
5452                         (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5453                         continue;
5454                 if (!stats) {
5455                         stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5456                                        GFP_KERNEL);
5457                         if (!stats) {
5458                                 n_entries = -ENOMEM;
5459                                 goto out;
5460                         }
5461                 }
5462                 for (i = 0; i < n_entries; i++) {
5463                         val = tracing_map_read_sum(sort_entries[i]->elt, j);
5464                         stats[j].total += val;
5465                         if (stats[j].max < val)
5466                                 stats[j].max = val;
5467                 }
5468         }
5469
5470         for (i = 0; i < n_entries; i++)
5471                 hist_trigger_entry_print(m, hist_data, stats,
5472                                          sort_entries[i]->key,
5473                                          sort_entries[i]->elt);
5474
5475         kfree(stats);
5476 out:
5477         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5478
5479         return n_entries;
5480 }
5481
5482 static void hist_trigger_show(struct seq_file *m,
5483                               struct event_trigger_data *data, int n)
5484 {
5485         struct hist_trigger_data *hist_data;
5486         int n_entries;
5487
5488         if (n > 0)
5489                 seq_puts(m, "\n\n");
5490
5491         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5492         data->ops->print(m, data);
5493         seq_puts(m, "#\n\n");
5494
5495         hist_data = data->private_data;
5496         n_entries = print_entries(m, hist_data);
5497         if (n_entries < 0)
5498                 n_entries = 0;
5499
5500         track_data_snapshot_print(m, hist_data);
5501
5502         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5503                    (u64)atomic64_read(&hist_data->map->hits),
5504                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5505 }
5506
5507 static int hist_show(struct seq_file *m, void *v)
5508 {
5509         struct event_trigger_data *data;
5510         struct trace_event_file *event_file;
5511         int n = 0, ret = 0;
5512
5513         mutex_lock(&event_mutex);
5514
5515         event_file = event_file_data(m->private);
5516         if (unlikely(!event_file)) {
5517                 ret = -ENODEV;
5518                 goto out_unlock;
5519         }
5520
5521         list_for_each_entry(data, &event_file->triggers, list) {
5522                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5523                         hist_trigger_show(m, data, n++);
5524         }
5525
5526  out_unlock:
5527         mutex_unlock(&event_mutex);
5528
5529         return ret;
5530 }
5531
5532 static int event_hist_open(struct inode *inode, struct file *file)
5533 {
5534         int ret;
5535
5536         ret = security_locked_down(LOCKDOWN_TRACEFS);
5537         if (ret)
5538                 return ret;
5539
5540         return single_open(file, hist_show, file);
5541 }
5542
5543 const struct file_operations event_hist_fops = {
5544         .open = event_hist_open,
5545         .read = seq_read,
5546         .llseek = seq_lseek,
5547         .release = single_release,
5548 };
5549
5550 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5551 static void hist_field_debug_show_flags(struct seq_file *m,
5552                                         unsigned long flags)
5553 {
5554         seq_puts(m, "      flags:\n");
5555
5556         if (flags & HIST_FIELD_FL_KEY)
5557                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5558         else if (flags & HIST_FIELD_FL_HITCOUNT)
5559                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5560         else if (flags & HIST_FIELD_FL_VAR)
5561                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5562         else if (flags & HIST_FIELD_FL_VAR_REF)
5563                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5564         else
5565                 seq_puts(m, "        VAL: normal u64 value\n");
5566
5567         if (flags & HIST_FIELD_FL_ALIAS)
5568                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5569         else if (flags & HIST_FIELD_FL_CONST)
5570                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5571 }
5572
5573 static int hist_field_debug_show(struct seq_file *m,
5574                                  struct hist_field *field, unsigned long flags)
5575 {
5576         if ((field->flags & flags) != flags) {
5577                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5578                 return -EINVAL;
5579         }
5580
5581         hist_field_debug_show_flags(m, field->flags);
5582         if (field->field)
5583                 seq_printf(m, "      ftrace_event_field name: %s\n",
5584                            field->field->name);
5585
5586         if (field->flags & HIST_FIELD_FL_VAR) {
5587                 seq_printf(m, "      var.name: %s\n", field->var.name);
5588                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5589                            field->var.idx);
5590         }
5591
5592         if (field->flags & HIST_FIELD_FL_CONST)
5593                 seq_printf(m, "      constant: %llu\n", field->constant);
5594
5595         if (field->flags & HIST_FIELD_FL_ALIAS)
5596                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5597                            field->var_ref_idx);
5598
5599         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5600                 seq_printf(m, "      name: %s\n", field->name);
5601                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5602                            field->var.idx);
5603                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5604                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5605                            field->var_ref_idx);
5606                 if (field->system)
5607                         seq_printf(m, "      system: %s\n", field->system);
5608                 if (field->event_name)
5609                         seq_printf(m, "      event_name: %s\n", field->event_name);
5610         }
5611
5612         seq_printf(m, "      type: %s\n", field->type);
5613         seq_printf(m, "      size: %u\n", field->size);
5614         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5615
5616         return 0;
5617 }
5618
5619 static int field_var_debug_show(struct seq_file *m,
5620                                 struct field_var *field_var, unsigned int i,
5621                                 bool save_vars)
5622 {
5623         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5624         struct hist_field *field;
5625         int ret = 0;
5626
5627         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5628
5629         field = field_var->var;
5630
5631         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5632
5633         hist_field_debug_show_flags(m, field->flags);
5634         seq_printf(m, "      var.name: %s\n", field->var.name);
5635         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5636                    field->var.idx);
5637
5638         field = field_var->val;
5639
5640         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5641         if (field->field)
5642                 seq_printf(m, "      ftrace_event_field name: %s\n",
5643                            field->field->name);
5644         else {
5645                 ret = -EINVAL;
5646                 goto out;
5647         }
5648
5649         seq_printf(m, "      type: %s\n", field->type);
5650         seq_printf(m, "      size: %u\n", field->size);
5651         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5652 out:
5653         return ret;
5654 }
5655
5656 static int hist_action_debug_show(struct seq_file *m,
5657                                   struct action_data *data, int i)
5658 {
5659         int ret = 0;
5660
5661         if (data->handler == HANDLER_ONMAX ||
5662             data->handler == HANDLER_ONCHANGE) {
5663                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5664                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5665                                             HIST_FIELD_FL_VAR_REF);
5666                 if (ret)
5667                         goto out;
5668
5669                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5670                 ret = hist_field_debug_show(m, data->track_data.track_var,
5671                                             HIST_FIELD_FL_VAR);
5672                 if (ret)
5673                         goto out;
5674         }
5675
5676         if (data->handler == HANDLER_ONMATCH) {
5677                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5678                            i, data->match_data.event_system);
5679                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5680                            i, data->match_data.event);
5681         }
5682 out:
5683         return ret;
5684 }
5685
5686 static int hist_actions_debug_show(struct seq_file *m,
5687                                    struct hist_trigger_data *hist_data)
5688 {
5689         int i, ret = 0;
5690
5691         if (hist_data->n_actions)
5692                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5693
5694         for (i = 0; i < hist_data->n_actions; i++) {
5695                 struct action_data *action = hist_data->actions[i];
5696
5697                 ret = hist_action_debug_show(m, action, i);
5698                 if (ret)
5699                         goto out;
5700         }
5701
5702         if (hist_data->n_save_vars)
5703                 seq_puts(m, "\n  save action variables (save() params):\n");
5704
5705         for (i = 0; i < hist_data->n_save_vars; i++) {
5706                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5707                 if (ret)
5708                         goto out;
5709         }
5710 out:
5711         return ret;
5712 }
5713
5714 static void hist_trigger_debug_show(struct seq_file *m,
5715                                     struct event_trigger_data *data, int n)
5716 {
5717         struct hist_trigger_data *hist_data;
5718         int i, ret;
5719
5720         if (n > 0)
5721                 seq_puts(m, "\n\n");
5722
5723         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5724         data->ops->print(m, data);
5725         seq_puts(m, "#\n\n");
5726
5727         hist_data = data->private_data;
5728
5729         seq_printf(m, "hist_data: %p\n\n", hist_data);
5730         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5731         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5732         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5733
5734         seq_puts(m, "\n  val fields:\n\n");
5735
5736         seq_puts(m, "    hist_data->fields[0]:\n");
5737         ret = hist_field_debug_show(m, hist_data->fields[0],
5738                                     HIST_FIELD_FL_HITCOUNT);
5739         if (ret)
5740                 return;
5741
5742         for (i = 1; i < hist_data->n_vals; i++) {
5743                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5744                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5745                 if (ret)
5746                         return;
5747         }
5748
5749         seq_puts(m, "\n  key fields:\n");
5750
5751         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5752                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5753                 ret = hist_field_debug_show(m, hist_data->fields[i],
5754                                             HIST_FIELD_FL_KEY);
5755                 if (ret)
5756                         return;
5757         }
5758
5759         if (hist_data->n_var_refs)
5760                 seq_puts(m, "\n  variable reference fields:\n");
5761
5762         for (i = 0; i < hist_data->n_var_refs; i++) {
5763                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5764                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5765                                             HIST_FIELD_FL_VAR_REF);
5766                 if (ret)
5767                         return;
5768         }
5769
5770         if (hist_data->n_field_vars)
5771                 seq_puts(m, "\n  field variables:\n");
5772
5773         for (i = 0; i < hist_data->n_field_vars; i++) {
5774                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5775                 if (ret)
5776                         return;
5777         }
5778
5779         ret = hist_actions_debug_show(m, hist_data);
5780         if (ret)
5781                 return;
5782 }
5783
5784 static int hist_debug_show(struct seq_file *m, void *v)
5785 {
5786         struct event_trigger_data *data;
5787         struct trace_event_file *event_file;
5788         int n = 0, ret = 0;
5789
5790         mutex_lock(&event_mutex);
5791
5792         event_file = event_file_data(m->private);
5793         if (unlikely(!event_file)) {
5794                 ret = -ENODEV;
5795                 goto out_unlock;
5796         }
5797
5798         list_for_each_entry(data, &event_file->triggers, list) {
5799                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5800                         hist_trigger_debug_show(m, data, n++);
5801         }
5802
5803  out_unlock:
5804         mutex_unlock(&event_mutex);
5805
5806         return ret;
5807 }
5808
5809 static int event_hist_debug_open(struct inode *inode, struct file *file)
5810 {
5811         int ret;
5812
5813         ret = security_locked_down(LOCKDOWN_TRACEFS);
5814         if (ret)
5815                 return ret;
5816
5817         return single_open(file, hist_debug_show, file);
5818 }
5819
5820 const struct file_operations event_hist_debug_fops = {
5821         .open = event_hist_debug_open,
5822         .read = seq_read,
5823         .llseek = seq_lseek,
5824         .release = single_release,
5825 };
5826 #endif
5827
5828 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5829 {
5830         const char *field_name = hist_field_name(hist_field, 0);
5831
5832         if (hist_field->var.name)
5833                 seq_printf(m, "%s=", hist_field->var.name);
5834
5835         if (hist_field->flags & HIST_FIELD_FL_CPU)
5836                 seq_puts(m, "common_cpu");
5837         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5838                 seq_printf(m, "%llu", hist_field->constant);
5839         else if (field_name) {
5840                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5841                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5842                         seq_putc(m, '$');
5843                 seq_printf(m, "%s", field_name);
5844         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5845                 seq_puts(m, "common_timestamp");
5846
5847         if (hist_field->flags) {
5848                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5849                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5850                         const char *flags = get_hist_field_flags(hist_field);
5851
5852                         if (flags)
5853                                 seq_printf(m, ".%s", flags);
5854                 }
5855         }
5856         if (hist_field->buckets)
5857                 seq_printf(m, "=%ld", hist_field->buckets);
5858 }
5859
5860 static int event_hist_trigger_print(struct seq_file *m,
5861                                     struct event_trigger_data *data)
5862 {
5863         struct hist_trigger_data *hist_data = data->private_data;
5864         struct hist_field *field;
5865         bool have_var = false;
5866         unsigned int i;
5867
5868         seq_puts(m, HIST_PREFIX);
5869
5870         if (data->name)
5871                 seq_printf(m, "%s:", data->name);
5872
5873         seq_puts(m, "keys=");
5874
5875         for_each_hist_key_field(i, hist_data) {
5876                 field = hist_data->fields[i];
5877
5878                 if (i > hist_data->n_vals)
5879                         seq_puts(m, ",");
5880
5881                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5882                         seq_puts(m, "stacktrace");
5883                 else
5884                         hist_field_print(m, field);
5885         }
5886
5887         seq_puts(m, ":vals=");
5888
5889         for_each_hist_val_field(i, hist_data) {
5890                 field = hist_data->fields[i];
5891                 if (field->flags & HIST_FIELD_FL_VAR) {
5892                         have_var = true;
5893                         continue;
5894                 }
5895
5896                 if (i == HITCOUNT_IDX)
5897                         seq_puts(m, "hitcount");
5898                 else {
5899                         seq_puts(m, ",");
5900                         hist_field_print(m, field);
5901                 }
5902         }
5903
5904         if (have_var) {
5905                 unsigned int n = 0;
5906
5907                 seq_puts(m, ":");
5908
5909                 for_each_hist_val_field(i, hist_data) {
5910                         field = hist_data->fields[i];
5911
5912                         if (field->flags & HIST_FIELD_FL_VAR) {
5913                                 if (n++)
5914                                         seq_puts(m, ",");
5915                                 hist_field_print(m, field);
5916                         }
5917                 }
5918         }
5919
5920         seq_puts(m, ":sort=");
5921
5922         for (i = 0; i < hist_data->n_sort_keys; i++) {
5923                 struct tracing_map_sort_key *sort_key;
5924                 unsigned int idx, first_key_idx;
5925
5926                 /* skip VAR vals */
5927                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5928
5929                 sort_key = &hist_data->sort_keys[i];
5930                 idx = sort_key->field_idx;
5931
5932                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5933                         return -EINVAL;
5934
5935                 if (i > 0)
5936                         seq_puts(m, ",");
5937
5938                 if (idx == HITCOUNT_IDX)
5939                         seq_puts(m, "hitcount");
5940                 else {
5941                         if (idx >= first_key_idx)
5942                                 idx += hist_data->n_vars;
5943                         hist_field_print(m, hist_data->fields[idx]);
5944                 }
5945
5946                 if (sort_key->descending)
5947                         seq_puts(m, ".descending");
5948         }
5949         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5950         if (hist_data->enable_timestamps)
5951                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5952
5953         print_actions_spec(m, hist_data);
5954
5955         if (data->filter_str)
5956                 seq_printf(m, " if %s", data->filter_str);
5957
5958         if (data->paused)
5959                 seq_puts(m, " [paused]");
5960         else
5961                 seq_puts(m, " [active]");
5962
5963         seq_putc(m, '\n');
5964
5965         return 0;
5966 }
5967
5968 static int event_hist_trigger_init(struct event_trigger_data *data)
5969 {
5970         struct hist_trigger_data *hist_data = data->private_data;
5971
5972         if (!data->ref && hist_data->attrs->name)
5973                 save_named_trigger(hist_data->attrs->name, data);
5974
5975         data->ref++;
5976
5977         return 0;
5978 }
5979
5980 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5981 {
5982         struct trace_event_file *file;
5983         unsigned int i;
5984         char *cmd;
5985         int ret;
5986
5987         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5988                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5989                 cmd = hist_data->field_var_hists[i]->cmd;
5990                 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
5991                                                "!hist", "hist", cmd);
5992                 WARN_ON_ONCE(ret < 0);
5993         }
5994 }
5995
5996 static void event_hist_trigger_free(struct event_trigger_data *data)
5997 {
5998         struct hist_trigger_data *hist_data = data->private_data;
5999
6000         if (WARN_ON_ONCE(data->ref <= 0))
6001                 return;
6002
6003         data->ref--;
6004         if (!data->ref) {
6005                 if (data->name)
6006                         del_named_trigger(data);
6007
6008                 trigger_data_free(data);
6009
6010                 remove_hist_vars(hist_data);
6011
6012                 unregister_field_var_hists(hist_data);
6013
6014                 destroy_hist_data(hist_data);
6015         }
6016 }
6017
6018 static struct event_trigger_ops event_hist_trigger_ops = {
6019         .trigger                = event_hist_trigger,
6020         .print                  = event_hist_trigger_print,
6021         .init                   = event_hist_trigger_init,
6022         .free                   = event_hist_trigger_free,
6023 };
6024
6025 static int event_hist_trigger_named_init(struct event_trigger_data *data)
6026 {
6027         data->ref++;
6028
6029         save_named_trigger(data->named_data->name, data);
6030
6031         event_hist_trigger_init(data->named_data);
6032
6033         return 0;
6034 }
6035
6036 static void event_hist_trigger_named_free(struct event_trigger_data *data)
6037 {
6038         if (WARN_ON_ONCE(data->ref <= 0))
6039                 return;
6040
6041         event_hist_trigger_free(data->named_data);
6042
6043         data->ref--;
6044         if (!data->ref) {
6045                 del_named_trigger(data);
6046                 trigger_data_free(data);
6047         }
6048 }
6049
6050 static struct event_trigger_ops event_hist_trigger_named_ops = {
6051         .trigger                = event_hist_trigger,
6052         .print                  = event_hist_trigger_print,
6053         .init                   = event_hist_trigger_named_init,
6054         .free                   = event_hist_trigger_named_free,
6055 };
6056
6057 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6058                                                             char *param)
6059 {
6060         return &event_hist_trigger_ops;
6061 }
6062
6063 static void hist_clear(struct event_trigger_data *data)
6064 {
6065         struct hist_trigger_data *hist_data = data->private_data;
6066
6067         if (data->name)
6068                 pause_named_trigger(data);
6069
6070         tracepoint_synchronize_unregister();
6071
6072         tracing_map_clear(hist_data->map);
6073
6074         if (data->name)
6075                 unpause_named_trigger(data);
6076 }
6077
6078 static bool compatible_field(struct ftrace_event_field *field,
6079                              struct ftrace_event_field *test_field)
6080 {
6081         if (field == test_field)
6082                 return true;
6083         if (field == NULL || test_field == NULL)
6084                 return false;
6085         if (strcmp(field->name, test_field->name) != 0)
6086                 return false;
6087         if (strcmp(field->type, test_field->type) != 0)
6088                 return false;
6089         if (field->size != test_field->size)
6090                 return false;
6091         if (field->is_signed != test_field->is_signed)
6092                 return false;
6093
6094         return true;
6095 }
6096
6097 static bool hist_trigger_match(struct event_trigger_data *data,
6098                                struct event_trigger_data *data_test,
6099                                struct event_trigger_data *named_data,
6100                                bool ignore_filter)
6101 {
6102         struct tracing_map_sort_key *sort_key, *sort_key_test;
6103         struct hist_trigger_data *hist_data, *hist_data_test;
6104         struct hist_field *key_field, *key_field_test;
6105         unsigned int i;
6106
6107         if (named_data && (named_data != data_test) &&
6108             (named_data != data_test->named_data))
6109                 return false;
6110
6111         if (!named_data && is_named_trigger(data_test))
6112                 return false;
6113
6114         hist_data = data->private_data;
6115         hist_data_test = data_test->private_data;
6116
6117         if (hist_data->n_vals != hist_data_test->n_vals ||
6118             hist_data->n_fields != hist_data_test->n_fields ||
6119             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6120                 return false;
6121
6122         if (!ignore_filter) {
6123                 if ((data->filter_str && !data_test->filter_str) ||
6124                    (!data->filter_str && data_test->filter_str))
6125                         return false;
6126         }
6127
6128         for_each_hist_field(i, hist_data) {
6129                 key_field = hist_data->fields[i];
6130                 key_field_test = hist_data_test->fields[i];
6131
6132                 if (key_field->flags != key_field_test->flags)
6133                         return false;
6134                 if (!compatible_field(key_field->field, key_field_test->field))
6135                         return false;
6136                 if (key_field->offset != key_field_test->offset)
6137                         return false;
6138                 if (key_field->size != key_field_test->size)
6139                         return false;
6140                 if (key_field->is_signed != key_field_test->is_signed)
6141                         return false;
6142                 if (!!key_field->var.name != !!key_field_test->var.name)
6143                         return false;
6144                 if (key_field->var.name &&
6145                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
6146                         return false;
6147         }
6148
6149         for (i = 0; i < hist_data->n_sort_keys; i++) {
6150                 sort_key = &hist_data->sort_keys[i];
6151                 sort_key_test = &hist_data_test->sort_keys[i];
6152
6153                 if (sort_key->field_idx != sort_key_test->field_idx ||
6154                     sort_key->descending != sort_key_test->descending)
6155                         return false;
6156         }
6157
6158         if (!ignore_filter && data->filter_str &&
6159             (strcmp(data->filter_str, data_test->filter_str) != 0))
6160                 return false;
6161
6162         if (!actions_match(hist_data, hist_data_test))
6163                 return false;
6164
6165         return true;
6166 }
6167
6168 static bool existing_hist_update_only(char *glob,
6169                                       struct event_trigger_data *data,
6170                                       struct trace_event_file *file)
6171 {
6172         struct hist_trigger_data *hist_data = data->private_data;
6173         struct event_trigger_data *test, *named_data = NULL;
6174         bool updated = false;
6175
6176         if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6177             !hist_data->attrs->clear)
6178                 goto out;
6179
6180         if (hist_data->attrs->name) {
6181                 named_data = find_named_trigger(hist_data->attrs->name);
6182                 if (named_data) {
6183                         if (!hist_trigger_match(data, named_data, named_data,
6184                                                 true))
6185                                 goto out;
6186                 }
6187         }
6188
6189         if (hist_data->attrs->name && !named_data)
6190                 goto out;
6191
6192         list_for_each_entry(test, &file->triggers, list) {
6193                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6194                         if (!hist_trigger_match(data, test, named_data, false))
6195                                 continue;
6196                         if (hist_data->attrs->pause)
6197                                 test->paused = true;
6198                         else if (hist_data->attrs->cont)
6199                                 test->paused = false;
6200                         else if (hist_data->attrs->clear)
6201                                 hist_clear(test);
6202                         updated = true;
6203                         goto out;
6204                 }
6205         }
6206  out:
6207         return updated;
6208 }
6209
6210 static int hist_register_trigger(char *glob,
6211                                  struct event_trigger_data *data,
6212                                  struct trace_event_file *file)
6213 {
6214         struct hist_trigger_data *hist_data = data->private_data;
6215         struct event_trigger_data *test, *named_data = NULL;
6216         struct trace_array *tr = file->tr;
6217         int ret = 0;
6218
6219         if (hist_data->attrs->name) {
6220                 named_data = find_named_trigger(hist_data->attrs->name);
6221                 if (named_data) {
6222                         if (!hist_trigger_match(data, named_data, named_data,
6223                                                 true)) {
6224                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6225                                 ret = -EINVAL;
6226                                 goto out;
6227                         }
6228                 }
6229         }
6230
6231         if (hist_data->attrs->name && !named_data)
6232                 goto new;
6233
6234         lockdep_assert_held(&event_mutex);
6235
6236         list_for_each_entry(test, &file->triggers, list) {
6237                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6238                         if (hist_trigger_match(data, test, named_data, false)) {
6239                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6240                                 ret = -EEXIST;
6241                                 goto out;
6242                         }
6243                 }
6244         }
6245  new:
6246         if (hist_data->attrs->cont || hist_data->attrs->clear) {
6247                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6248                 ret = -ENOENT;
6249                 goto out;
6250         }
6251
6252         if (hist_data->attrs->pause)
6253                 data->paused = true;
6254
6255         if (named_data) {
6256                 data->private_data = named_data->private_data;
6257                 set_named_trigger_data(data, named_data);
6258                 data->ops = &event_hist_trigger_named_ops;
6259         }
6260
6261         if (data->ops->init) {
6262                 ret = data->ops->init(data);
6263                 if (ret < 0)
6264                         goto out;
6265         }
6266
6267         if (hist_data->enable_timestamps) {
6268                 char *clock = hist_data->attrs->clock;
6269
6270                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6271                 if (ret) {
6272                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6273                         goto out;
6274                 }
6275
6276                 tracing_set_filter_buffering(file->tr, true);
6277         }
6278
6279         if (named_data)
6280                 destroy_hist_data(hist_data);
6281  out:
6282         return ret;
6283 }
6284
6285 static int hist_trigger_enable(struct event_trigger_data *data,
6286                                struct trace_event_file *file)
6287 {
6288         int ret = 0;
6289
6290         list_add_tail_rcu(&data->list, &file->triggers);
6291
6292         update_cond_flag(file);
6293
6294         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6295                 list_del_rcu(&data->list);
6296                 update_cond_flag(file);
6297                 ret--;
6298         }
6299
6300         return ret;
6301 }
6302
6303 static bool have_hist_trigger_match(struct event_trigger_data *data,
6304                                     struct trace_event_file *file)
6305 {
6306         struct hist_trigger_data *hist_data = data->private_data;
6307         struct event_trigger_data *test, *named_data = NULL;
6308         bool match = false;
6309
6310         lockdep_assert_held(&event_mutex);
6311
6312         if (hist_data->attrs->name)
6313                 named_data = find_named_trigger(hist_data->attrs->name);
6314
6315         list_for_each_entry(test, &file->triggers, list) {
6316                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6317                         if (hist_trigger_match(data, test, named_data, false)) {
6318                                 match = true;
6319                                 break;
6320                         }
6321                 }
6322         }
6323
6324         return match;
6325 }
6326
6327 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6328                                     struct trace_event_file *file)
6329 {
6330         struct hist_trigger_data *hist_data = data->private_data;
6331         struct event_trigger_data *test, *named_data = NULL;
6332
6333         lockdep_assert_held(&event_mutex);
6334
6335         if (hist_data->attrs->name)
6336                 named_data = find_named_trigger(hist_data->attrs->name);
6337
6338         list_for_each_entry(test, &file->triggers, list) {
6339                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6340                         if (!hist_trigger_match(data, test, named_data, false))
6341                                 continue;
6342                         hist_data = test->private_data;
6343                         if (check_var_refs(hist_data))
6344                                 return true;
6345                         break;
6346                 }
6347         }
6348
6349         return false;
6350 }
6351
6352 static void hist_unregister_trigger(char *glob,
6353                                     struct event_trigger_data *data,
6354                                     struct trace_event_file *file)
6355 {
6356         struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6357         struct hist_trigger_data *hist_data = data->private_data;
6358
6359         lockdep_assert_held(&event_mutex);
6360
6361         if (hist_data->attrs->name)
6362                 named_data = find_named_trigger(hist_data->attrs->name);
6363
6364         list_for_each_entry(iter, &file->triggers, list) {
6365                 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6366                         if (!hist_trigger_match(data, iter, named_data, false))
6367                                 continue;
6368                         test = iter;
6369                         list_del_rcu(&test->list);
6370                         trace_event_trigger_enable_disable(file, 0);
6371                         update_cond_flag(file);
6372                         break;
6373                 }
6374         }
6375
6376         if (test && test->ops->free)
6377                 test->ops->free(test);
6378
6379         if (hist_data->enable_timestamps) {
6380                 if (!hist_data->remove || test)
6381                         tracing_set_filter_buffering(file->tr, false);
6382         }
6383 }
6384
6385 static bool hist_file_check_refs(struct trace_event_file *file)
6386 {
6387         struct hist_trigger_data *hist_data;
6388         struct event_trigger_data *test;
6389
6390         lockdep_assert_held(&event_mutex);
6391
6392         list_for_each_entry(test, &file->triggers, list) {
6393                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6394                         hist_data = test->private_data;
6395                         if (check_var_refs(hist_data))
6396                                 return true;
6397                 }
6398         }
6399
6400         return false;
6401 }
6402
6403 static void hist_unreg_all(struct trace_event_file *file)
6404 {
6405         struct event_trigger_data *test, *n;
6406         struct hist_trigger_data *hist_data;
6407         struct synth_event *se;
6408         const char *se_name;
6409
6410         lockdep_assert_held(&event_mutex);
6411
6412         if (hist_file_check_refs(file))
6413                 return;
6414
6415         list_for_each_entry_safe(test, n, &file->triggers, list) {
6416                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6417                         hist_data = test->private_data;
6418                         list_del_rcu(&test->list);
6419                         trace_event_trigger_enable_disable(file, 0);
6420
6421                         se_name = trace_event_name(file->event_call);
6422                         se = find_synth_event(se_name);
6423                         if (se)
6424                                 se->ref--;
6425
6426                         update_cond_flag(file);
6427                         if (hist_data->enable_timestamps)
6428                                 tracing_set_filter_buffering(file->tr, false);
6429                         if (test->ops->free)
6430                                 test->ops->free(test);
6431                 }
6432         }
6433 }
6434
6435 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6436                                     struct trace_event_file *file,
6437                                     char *glob, char *cmd,
6438                                     char *param_and_filter)
6439 {
6440         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6441         struct event_trigger_data *trigger_data;
6442         struct hist_trigger_attrs *attrs;
6443         struct hist_trigger_data *hist_data;
6444         char *param, *filter, *p, *start;
6445         struct synth_event *se;
6446         const char *se_name;
6447         bool remove;
6448         int ret = 0;
6449
6450         lockdep_assert_held(&event_mutex);
6451
6452         if (WARN_ON(!glob))
6453                 return -EINVAL;
6454
6455         if (glob[0]) {
6456                 hist_err_clear();
6457                 last_cmd_set(file, param_and_filter);
6458         }
6459
6460         remove = event_trigger_check_remove(glob);
6461
6462         if (event_trigger_empty_param(param_and_filter))
6463                 return -EINVAL;
6464
6465         /*
6466          * separate the trigger from the filter (k:v [if filter])
6467          * allowing for whitespace in the trigger
6468          */
6469         p = param = param_and_filter;
6470         do {
6471                 p = strstr(p, "if");
6472                 if (!p)
6473                         break;
6474                 if (p == param_and_filter)
6475                         return -EINVAL;
6476                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6477                         p++;
6478                         continue;
6479                 }
6480                 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6481                         return -EINVAL;
6482                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6483                         p++;
6484                         continue;
6485                 }
6486                 break;
6487         } while (1);
6488
6489         if (!p)
6490                 filter = NULL;
6491         else {
6492                 *(p - 1) = '\0';
6493                 filter = strstrip(p);
6494                 param = strstrip(param);
6495         }
6496
6497         /*
6498          * To simplify arithmetic expression parsing, replace occurrences of
6499          * '.sym-offset' modifier with '.symXoffset'
6500          */
6501         start = strstr(param, ".sym-offset");
6502         while (start) {
6503                 *(start + 4) = 'X';
6504                 start = strstr(start + 11, ".sym-offset");
6505         }
6506
6507         attrs = parse_hist_trigger_attrs(file->tr, param);
6508         if (IS_ERR(attrs))
6509                 return PTR_ERR(attrs);
6510
6511         if (attrs->map_bits)
6512                 hist_trigger_bits = attrs->map_bits;
6513
6514         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6515         if (IS_ERR(hist_data)) {
6516                 destroy_hist_trigger_attrs(attrs);
6517                 return PTR_ERR(hist_data);
6518         }
6519
6520         trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data);
6521         if (!trigger_data) {
6522                 ret = -ENOMEM;
6523                 goto out_free;
6524         }
6525
6526         ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6527         if (ret < 0)
6528                 goto out_free;
6529
6530         if (remove) {
6531                 if (!have_hist_trigger_match(trigger_data, file))
6532                         goto out_free;
6533
6534                 if (hist_trigger_check_refs(trigger_data, file)) {
6535                         ret = -EBUSY;
6536                         goto out_free;
6537                 }
6538
6539                 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6540                 se_name = trace_event_name(file->event_call);
6541                 se = find_synth_event(se_name);
6542                 if (se)
6543                         se->ref--;
6544                 ret = 0;
6545                 goto out_free;
6546         }
6547
6548         if (existing_hist_update_only(glob, trigger_data, file))
6549                 goto out_free;
6550
6551         ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6552         if (ret < 0)
6553                 goto out_free;
6554
6555         if (get_named_trigger_data(trigger_data))
6556                 goto enable;
6557
6558         if (has_hist_vars(hist_data))
6559                 save_hist_vars(hist_data);
6560
6561         ret = create_actions(hist_data);
6562         if (ret)
6563                 goto out_unreg;
6564
6565         ret = tracing_map_init(hist_data->map);
6566         if (ret)
6567                 goto out_unreg;
6568 enable:
6569         ret = hist_trigger_enable(trigger_data, file);
6570         if (ret)
6571                 goto out_unreg;
6572
6573         se_name = trace_event_name(file->event_call);
6574         se = find_synth_event(se_name);
6575         if (se)
6576                 se->ref++;
6577  out:
6578         if (ret == 0 && glob[0])
6579                 hist_err_clear();
6580
6581         return ret;
6582  out_unreg:
6583         event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6584  out_free:
6585         event_trigger_reset_filter(cmd_ops, trigger_data);
6586
6587         remove_hist_vars(hist_data);
6588
6589         kfree(trigger_data);
6590
6591         destroy_hist_data(hist_data);
6592         goto out;
6593 }
6594
6595 static struct event_command trigger_hist_cmd = {
6596         .name                   = "hist",
6597         .trigger_type           = ETT_EVENT_HIST,
6598         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6599         .parse                  = event_hist_trigger_parse,
6600         .reg                    = hist_register_trigger,
6601         .unreg                  = hist_unregister_trigger,
6602         .unreg_all              = hist_unreg_all,
6603         .get_trigger_ops        = event_hist_get_trigger_ops,
6604         .set_filter             = set_trigger_filter,
6605 };
6606
6607 __init int register_trigger_hist_cmd(void)
6608 {
6609         int ret;
6610
6611         ret = register_event_command(&trigger_hist_cmd);
6612         WARN_ON(ret < 0);
6613
6614         return ret;
6615 }
6616
6617 static void
6618 hist_enable_trigger(struct event_trigger_data *data,
6619                     struct trace_buffer *buffer,  void *rec,
6620                     struct ring_buffer_event *event)
6621 {
6622         struct enable_trigger_data *enable_data = data->private_data;
6623         struct event_trigger_data *test;
6624
6625         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6626                                 lockdep_is_held(&event_mutex)) {
6627                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6628                         if (enable_data->enable)
6629                                 test->paused = false;
6630                         else
6631                                 test->paused = true;
6632                 }
6633         }
6634 }
6635
6636 static void
6637 hist_enable_count_trigger(struct event_trigger_data *data,
6638                           struct trace_buffer *buffer,  void *rec,
6639                           struct ring_buffer_event *event)
6640 {
6641         if (!data->count)
6642                 return;
6643
6644         if (data->count != -1)
6645                 (data->count)--;
6646
6647         hist_enable_trigger(data, buffer, rec, event);
6648 }
6649
6650 static struct event_trigger_ops hist_enable_trigger_ops = {
6651         .trigger                = hist_enable_trigger,
6652         .print                  = event_enable_trigger_print,
6653         .init                   = event_trigger_init,
6654         .free                   = event_enable_trigger_free,
6655 };
6656
6657 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6658         .trigger                = hist_enable_count_trigger,
6659         .print                  = event_enable_trigger_print,
6660         .init                   = event_trigger_init,
6661         .free                   = event_enable_trigger_free,
6662 };
6663
6664 static struct event_trigger_ops hist_disable_trigger_ops = {
6665         .trigger                = hist_enable_trigger,
6666         .print                  = event_enable_trigger_print,
6667         .init                   = event_trigger_init,
6668         .free                   = event_enable_trigger_free,
6669 };
6670
6671 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6672         .trigger                = hist_enable_count_trigger,
6673         .print                  = event_enable_trigger_print,
6674         .init                   = event_trigger_init,
6675         .free                   = event_enable_trigger_free,
6676 };
6677
6678 static struct event_trigger_ops *
6679 hist_enable_get_trigger_ops(char *cmd, char *param)
6680 {
6681         struct event_trigger_ops *ops;
6682         bool enable;
6683
6684         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6685
6686         if (enable)
6687                 ops = param ? &hist_enable_count_trigger_ops :
6688                         &hist_enable_trigger_ops;
6689         else
6690                 ops = param ? &hist_disable_count_trigger_ops :
6691                         &hist_disable_trigger_ops;
6692
6693         return ops;
6694 }
6695
6696 static void hist_enable_unreg_all(struct trace_event_file *file)
6697 {
6698         struct event_trigger_data *test, *n;
6699
6700         list_for_each_entry_safe(test, n, &file->triggers, list) {
6701                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6702                         list_del_rcu(&test->list);
6703                         update_cond_flag(file);
6704                         trace_event_trigger_enable_disable(file, 0);
6705                         if (test->ops->free)
6706                                 test->ops->free(test);
6707                 }
6708         }
6709 }
6710
6711 static struct event_command trigger_hist_enable_cmd = {
6712         .name                   = ENABLE_HIST_STR,
6713         .trigger_type           = ETT_HIST_ENABLE,
6714         .parse                  = event_enable_trigger_parse,
6715         .reg                    = event_enable_register_trigger,
6716         .unreg                  = event_enable_unregister_trigger,
6717         .unreg_all              = hist_enable_unreg_all,
6718         .get_trigger_ops        = hist_enable_get_trigger_ops,
6719         .set_filter             = set_trigger_filter,
6720 };
6721
6722 static struct event_command trigger_hist_disable_cmd = {
6723         .name                   = DISABLE_HIST_STR,
6724         .trigger_type           = ETT_HIST_ENABLE,
6725         .parse                  = event_enable_trigger_parse,
6726         .reg                    = event_enable_register_trigger,
6727         .unreg                  = event_enable_unregister_trigger,
6728         .unreg_all              = hist_enable_unreg_all,
6729         .get_trigger_ops        = hist_enable_get_trigger_ops,
6730         .set_filter             = set_trigger_filter,
6731 };
6732
6733 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6734 {
6735         unregister_event_command(&trigger_hist_enable_cmd);
6736         unregister_event_command(&trigger_hist_disable_cmd);
6737 }
6738
6739 __init int register_trigger_hist_enable_disable_cmds(void)
6740 {
6741         int ret;
6742
6743         ret = register_event_command(&trigger_hist_enable_cmd);
6744         if (WARN_ON(ret < 0))
6745                 return ret;
6746         ret = register_event_command(&trigger_hist_disable_cmd);
6747         if (WARN_ON(ret < 0))
6748                 unregister_trigger_hist_enable_disable_cmds();
6749
6750         return ret;
6751 }