kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),                    \
  73         C(NEED_NOHC_VAL,        "Non-hitcount value is required for 'nohitcount'"),
  74
  75 #undef C
  76 #define C(a, b)         HIST_ERR_##a
  77
  78 enum { ERRORS };
  79
  80 #undef C
  81 #define C(a, b)         b
  82
  83 static const char *err_text[] = { ERRORS };
  84
  85 struct hist_field;
  86
  87 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  88                                 struct tracing_map_elt *elt,
  89                                 struct trace_buffer *buffer,
  90                                 struct ring_buffer_event *rbe,
  91                                 void *event);
  92
  93 #define HIST_FIELD_OPERANDS_MAX 2
  94 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95 #define HIST_ACTIONS_MAX        8
  96 #define HIST_CONST_DIGITS_MAX   21
  97 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  98
  99 enum field_op_id {
 100         FIELD_OP_NONE,
 101         FIELD_OP_PLUS,
 102         FIELD_OP_MINUS,
 103         FIELD_OP_UNARY_MINUS,
 104         FIELD_OP_DIV,
 105         FIELD_OP_MULT,
 106 };
 107
 108 enum hist_field_fn {
 109         HIST_FIELD_FN_NOP,
 110         HIST_FIELD_FN_VAR_REF,
 111         HIST_FIELD_FN_COUNTER,
 112         HIST_FIELD_FN_CONST,
 113         HIST_FIELD_FN_LOG2,
 114         HIST_FIELD_FN_BUCKET,
 115         HIST_FIELD_FN_TIMESTAMP,
 116         HIST_FIELD_FN_CPU,
 117         HIST_FIELD_FN_STRING,
 118         HIST_FIELD_FN_DYNSTRING,
 119         HIST_FIELD_FN_RELDYNSTRING,
 120         HIST_FIELD_FN_PSTRING,
 121         HIST_FIELD_FN_S64,
 122         HIST_FIELD_FN_U64,
 123         HIST_FIELD_FN_S32,
 124         HIST_FIELD_FN_U32,
 125         HIST_FIELD_FN_S16,
 126         HIST_FIELD_FN_U16,
 127         HIST_FIELD_FN_S8,
 128         HIST_FIELD_FN_U8,
 129         HIST_FIELD_FN_UMINUS,
 130         HIST_FIELD_FN_MINUS,
 131         HIST_FIELD_FN_PLUS,
 132         HIST_FIELD_FN_DIV,
 133         HIST_FIELD_FN_MULT,
 134         HIST_FIELD_FN_DIV_POWER2,
 135         HIST_FIELD_FN_DIV_NOT_POWER2,
 136         HIST_FIELD_FN_DIV_MULT_SHIFT,
 137         HIST_FIELD_FN_EXECNAME,
 138         HIST_FIELD_FN_STACK,
 139 };
 140
 141 /*
 142  * A hist_var (histogram variable) contains variable information for
 143  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 144  * flag set.  A hist_var has a variable name e.g. ts0, and is
 145  * associated with a given histogram trigger, as specified by
 146  * hist_data.  The hist_var idx is the unique index assigned to the
 147  * variable by the hist trigger's tracing_map.  The idx is what is
 148  * used to set a variable's value and, by a variable reference, to
 149  * retrieve it.
 150  */
 151 struct hist_var {
 152         char                            *name;
 153         struct hist_trigger_data        *hist_data;
 154         unsigned int                    idx;
 155 };
 156
 157 struct hist_field {
 158         struct ftrace_event_field       *field;
 159         unsigned long                   flags;
 160         unsigned long                   buckets;
 161         const char                      *type;
 162         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 163         struct hist_trigger_data        *hist_data;
 164         enum hist_field_fn              fn_num;
 165         unsigned int                    ref;
 166         unsigned int                    size;
 167         unsigned int                    offset;
 168         unsigned int                    is_signed;
 169
 170         /*
 171          * Variable fields contain variable-specific info in var.
 172          */
 173         struct hist_var                 var;
 174         enum field_op_id                operator;
 175         char                            *system;
 176         char                            *event_name;
 177
 178         /*
 179          * The name field is used for EXPR and VAR_REF fields.  VAR
 180          * fields contain the variable name in var.name.
 181          */
 182         char                            *name;
 183
 184         /*
 185          * When a histogram trigger is hit, if it has any references
 186          * to variables, the values of those variables are collected
 187          * into a var_ref_vals array by resolve_var_refs().  The
 188          * current value of each variable is read from the tracing_map
 189          * using the hist field's hist_var.idx and entered into the
 190          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 191          */
 192         unsigned int                    var_ref_idx;
 193         bool                            read_once;
 194
 195         unsigned int                    var_str_idx;
 196
 197         /* Numeric literals are represented as u64 */
 198         u64                             constant;
 199         /* Used to optimize division by constants */
 200         u64                             div_multiplier;
 201 };
 202
 203 static u64 hist_fn_call(struct hist_field *hist_field,
 204                         struct tracing_map_elt *elt,
 205                         struct trace_buffer *buffer,
 206                         struct ring_buffer_event *rbe,
 207                         void *event);
 208
 209 static u64 hist_field_const(struct hist_field *field,
 210                            struct tracing_map_elt *elt,
 211                            struct trace_buffer *buffer,
 212                            struct ring_buffer_event *rbe,
 213                            void *event)
 214 {
 215         return field->constant;
 216 }
 217
 218 static u64 hist_field_counter(struct hist_field *field,
 219                               struct tracing_map_elt *elt,
 220                               struct trace_buffer *buffer,
 221                               struct ring_buffer_event *rbe,
 222                               void *event)
 223 {
 224         return 1;
 225 }
 226
 227 static u64 hist_field_string(struct hist_field *hist_field,
 228                              struct tracing_map_elt *elt,
 229                              struct trace_buffer *buffer,
 230                              struct ring_buffer_event *rbe,
 231                              void *event)
 232 {
 233         char *addr = (char *)(event + hist_field->field->offset);
 234
 235         return (u64)(unsigned long)addr;
 236 }
 237
 238 static u64 hist_field_dynstring(struct hist_field *hist_field,
 239                                 struct tracing_map_elt *elt,
 240                                 struct trace_buffer *buffer,
 241                                 struct ring_buffer_event *rbe,
 242                                 void *event)
 243 {
 244         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 245         int str_loc = str_item & 0xffff;
 246         char *addr = (char *)(event + str_loc);
 247
 248         return (u64)(unsigned long)addr;
 249 }
 250
 251 static u64 hist_field_reldynstring(struct hist_field *hist_field,
 252                                    struct tracing_map_elt *elt,
 253                                    struct trace_buffer *buffer,
 254                                    struct ring_buffer_event *rbe,
 255                                    void *event)
 256 {
 257         u32 *item = event + hist_field->field->offset;
 258         u32 str_item = *item;
 259         int str_loc = str_item & 0xffff;
 260         char *addr = (char *)&item[1] + str_loc;
 261
 262         return (u64)(unsigned long)addr;
 263 }
 264
 265 static u64 hist_field_pstring(struct hist_field *hist_field,
 266                               struct tracing_map_elt *elt,
 267                               struct trace_buffer *buffer,
 268                               struct ring_buffer_event *rbe,
 269                               void *event)
 270 {
 271         char **addr = (char **)(event + hist_field->field->offset);
 272
 273         return (u64)(unsigned long)*addr;
 274 }
 275
 276 static u64 hist_field_log2(struct hist_field *hist_field,
 277                            struct tracing_map_elt *elt,
 278                            struct trace_buffer *buffer,
 279                            struct ring_buffer_event *rbe,
 280                            void *event)
 281 {
 282         struct hist_field *operand = hist_field->operands[0];
 283
 284         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 285
 286         return (u64) ilog2(roundup_pow_of_two(val));
 287 }
 288
 289 static u64 hist_field_bucket(struct hist_field *hist_field,
 290                              struct tracing_map_elt *elt,
 291                              struct trace_buffer *buffer,
 292                              struct ring_buffer_event *rbe,
 293                              void *event)
 294 {
 295         struct hist_field *operand = hist_field->operands[0];
 296         unsigned long buckets = hist_field->buckets;
 297
 298         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 299
 300         if (WARN_ON_ONCE(!buckets))
 301                 return val;
 302
 303         if (val >= LONG_MAX)
 304                 val = div64_ul(val, buckets);
 305         else
 306                 val = (u64)((unsigned long)val / buckets);
 307         return val * buckets;
 308 }
 309
 310 static u64 hist_field_plus(struct hist_field *hist_field,
 311                            struct tracing_map_elt *elt,
 312                            struct trace_buffer *buffer,
 313                            struct ring_buffer_event *rbe,
 314                            void *event)
 315 {
 316         struct hist_field *operand1 = hist_field->operands[0];
 317         struct hist_field *operand2 = hist_field->operands[1];
 318
 319         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 320         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 321
 322         return val1 + val2;
 323 }
 324
 325 static u64 hist_field_minus(struct hist_field *hist_field,
 326                             struct tracing_map_elt *elt,
 327                             struct trace_buffer *buffer,
 328                             struct ring_buffer_event *rbe,
 329                             void *event)
 330 {
 331         struct hist_field *operand1 = hist_field->operands[0];
 332         struct hist_field *operand2 = hist_field->operands[1];
 333
 334         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 335         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 336
 337         return val1 - val2;
 338 }
 339
 340 static u64 hist_field_div(struct hist_field *hist_field,
 341                            struct tracing_map_elt *elt,
 342                            struct trace_buffer *buffer,
 343                            struct ring_buffer_event *rbe,
 344                            void *event)
 345 {
 346         struct hist_field *operand1 = hist_field->operands[0];
 347         struct hist_field *operand2 = hist_field->operands[1];
 348
 349         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 350         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 351
 352         /* Return -1 for the undefined case */
 353         if (!val2)
 354                 return -1;
 355
 356         /* Use shift if the divisor is a power of 2 */
 357         if (!(val2 & (val2 - 1)))
 358                 return val1 >> __ffs64(val2);
 359
 360         return div64_u64(val1, val2);
 361 }
 362
 363 static u64 div_by_power_of_two(struct hist_field *hist_field,
 364                                 struct tracing_map_elt *elt,
 365                                 struct trace_buffer *buffer,
 366                                 struct ring_buffer_event *rbe,
 367                                 void *event)
 368 {
 369         struct hist_field *operand1 = hist_field->operands[0];
 370         struct hist_field *operand2 = hist_field->operands[1];
 371
 372         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 373
 374         return val1 >> __ffs64(operand2->constant);
 375 }
 376
 377 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 378                                 struct tracing_map_elt *elt,
 379                                 struct trace_buffer *buffer,
 380                                 struct ring_buffer_event *rbe,
 381                                 void *event)
 382 {
 383         struct hist_field *operand1 = hist_field->operands[0];
 384         struct hist_field *operand2 = hist_field->operands[1];
 385
 386         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 387
 388         return div64_u64(val1, operand2->constant);
 389 }
 390
 391 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 392                                 struct tracing_map_elt *elt,
 393                                 struct trace_buffer *buffer,
 394                                 struct ring_buffer_event *rbe,
 395                                 void *event)
 396 {
 397         struct hist_field *operand1 = hist_field->operands[0];
 398         struct hist_field *operand2 = hist_field->operands[1];
 399
 400         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 401
 402         /*
 403          * If the divisor is a constant, do a multiplication and shift instead.
 404          *
 405          * Choose Z = some power of 2. If Y <= Z, then:
 406          *     X / Y = (X * (Z / Y)) / Z
 407          *
 408          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 409          *     X / Y = (X * mult) / Z
 410          *
 411          * The division by Z can be replaced by a shift since Z is a power of 2:
 412          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 413          *
 414          * As long, as X < Z the results will not be off by more than 1.
 415          */
 416         if (val1 < (1 << HIST_DIV_SHIFT)) {
 417                 u64 mult = operand2->div_multiplier;
 418
 419                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 420         }
 421
 422         return div64_u64(val1, operand2->constant);
 423 }
 424
 425 static u64 hist_field_mult(struct hist_field *hist_field,
 426                            struct tracing_map_elt *elt,
 427                            struct trace_buffer *buffer,
 428                            struct ring_buffer_event *rbe,
 429                            void *event)
 430 {
 431         struct hist_field *operand1 = hist_field->operands[0];
 432         struct hist_field *operand2 = hist_field->operands[1];
 433
 434         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 435         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 436
 437         return val1 * val2;
 438 }
 439
 440 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 441                                   struct tracing_map_elt *elt,
 442                                   struct trace_buffer *buffer,
 443                                   struct ring_buffer_event *rbe,
 444                                   void *event)
 445 {
 446         struct hist_field *operand = hist_field->operands[0];
 447
 448         s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
 449         u64 val = (u64)-sval;
 450
 451         return val;
 452 }
 453
 454 #define DEFINE_HIST_FIELD_FN(type)                                      \
 455         static u64 hist_field_##type(struct hist_field *hist_field,     \
 456                                      struct tracing_map_elt *elt,       \
 457                                      struct trace_buffer *buffer,       \
 458                                      struct ring_buffer_event *rbe,     \
 459                                      void *event)                       \
 460 {                                                                       \
 461         type *addr = (type *)(event + hist_field->field->offset);       \
 462                                                                         \
 463         return (u64)(unsigned long)*addr;                               \
 464 }
 465
 466 DEFINE_HIST_FIELD_FN(s64);
 467 DEFINE_HIST_FIELD_FN(u64);
 468 DEFINE_HIST_FIELD_FN(s32);
 469 DEFINE_HIST_FIELD_FN(u32);
 470 DEFINE_HIST_FIELD_FN(s16);
 471 DEFINE_HIST_FIELD_FN(u16);
 472 DEFINE_HIST_FIELD_FN(s8);
 473 DEFINE_HIST_FIELD_FN(u8);
 474
 475 #define for_each_hist_field(i, hist_data)       \
 476         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 477
 478 #define for_each_hist_val_field(i, hist_data)   \
 479         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 480
 481 #define for_each_hist_key_field(i, hist_data)   \
 482         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 483
 484 #define HITCOUNT_IDX            0
 485 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 486
 487 enum hist_field_flags {
 488         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 489         HIST_FIELD_FL_KEY               = 1 << 1,
 490         HIST_FIELD_FL_STRING            = 1 << 2,
 491         HIST_FIELD_FL_HEX               = 1 << 3,
 492         HIST_FIELD_FL_SYM               = 1 << 4,
 493         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 494         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 495         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 496         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 497         HIST_FIELD_FL_LOG2              = 1 << 9,
 498         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 499         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 500         HIST_FIELD_FL_VAR               = 1 << 12,
 501         HIST_FIELD_FL_EXPR              = 1 << 13,
 502         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 503         HIST_FIELD_FL_CPU               = 1 << 15,
 504         HIST_FIELD_FL_ALIAS             = 1 << 16,
 505         HIST_FIELD_FL_BUCKET            = 1 << 17,
 506         HIST_FIELD_FL_CONST             = 1 << 18,
 507         HIST_FIELD_FL_PERCENT           = 1 << 19,
 508         HIST_FIELD_FL_GRAPH             = 1 << 20,
 509 };
 510
 511 struct var_defs {
 512         unsigned int    n_vars;
 513         char            *name[TRACING_MAP_VARS_MAX];
 514         char            *expr[TRACING_MAP_VARS_MAX];
 515 };
 516
 517 struct hist_trigger_attrs {
 518         char            *keys_str;
 519         char            *vals_str;
 520         char            *sort_key_str;
 521         char            *name;
 522         char            *clock;
 523         bool            pause;
 524         bool            cont;
 525         bool            clear;
 526         bool            ts_in_usecs;
 527         bool            no_hitcount;
 528         unsigned int    map_bits;
 529
 530         char            *assignment_str[TRACING_MAP_VARS_MAX];
 531         unsigned int    n_assignments;
 532
 533         char            *action_str[HIST_ACTIONS_MAX];
 534         unsigned int    n_actions;
 535
 536         struct var_defs var_defs;
 537 };
 538
 539 struct field_var {
 540         struct hist_field       *var;
 541         struct hist_field       *val;
 542 };
 543
 544 struct field_var_hist {
 545         struct hist_trigger_data        *hist_data;
 546         char                            *cmd;
 547 };
 548
 549 struct hist_trigger_data {
 550         struct hist_field               *fields[HIST_FIELDS_MAX];
 551         unsigned int                    n_vals;
 552         unsigned int                    n_keys;
 553         unsigned int                    n_fields;
 554         unsigned int                    n_vars;
 555         unsigned int                    n_var_str;
 556         unsigned int                    key_size;
 557         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 558         unsigned int                    n_sort_keys;
 559         struct trace_event_file         *event_file;
 560         struct hist_trigger_attrs       *attrs;
 561         struct tracing_map              *map;
 562         bool                            enable_timestamps;
 563         bool                            remove;
 564         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 565         unsigned int                    n_var_refs;
 566
 567         struct action_data              *actions[HIST_ACTIONS_MAX];
 568         unsigned int                    n_actions;
 569
 570         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 571         unsigned int                    n_field_vars;
 572         unsigned int                    n_field_var_str;
 573         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 574         unsigned int                    n_field_var_hists;
 575
 576         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 577         unsigned int                    n_save_vars;
 578         unsigned int                    n_save_var_str;
 579 };
 580
 581 struct action_data;
 582
 583 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 584                              struct tracing_map_elt *elt,
 585                              struct trace_buffer *buffer, void *rec,
 586                              struct ring_buffer_event *rbe, void *key,
 587                              struct action_data *data, u64 *var_ref_vals);
 588
 589 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 590
 591 enum handler_id {
 592         HANDLER_ONMATCH = 1,
 593         HANDLER_ONMAX,
 594         HANDLER_ONCHANGE,
 595 };
 596
 597 enum action_id {
 598         ACTION_SAVE = 1,
 599         ACTION_TRACE,
 600         ACTION_SNAPSHOT,
 601 };
 602
 603 struct action_data {
 604         enum handler_id         handler;
 605         enum action_id          action;
 606         char                    *action_name;
 607         action_fn_t             fn;
 608
 609         unsigned int            n_params;
 610         char                    *params[SYNTH_FIELDS_MAX];
 611
 612         /*
 613          * When a histogram trigger is hit, the values of any
 614          * references to variables, including variables being passed
 615          * as parameters to synthetic events, are collected into a
 616          * var_ref_vals array.  This var_ref_idx array is an array of
 617          * indices into the var_ref_vals array, one for each synthetic
 618          * event param, and is passed to the synthetic event
 619          * invocation.
 620          */
 621         unsigned int            var_ref_idx[SYNTH_FIELDS_MAX];
 622         struct synth_event      *synth_event;
 623         bool                    use_trace_keyword;
 624         char                    *synth_event_name;
 625
 626         union {
 627                 struct {
 628                         char                    *event;
 629                         char                    *event_system;
 630                 } match_data;
 631
 632                 struct {
 633                         /*
 634                          * var_str contains the $-unstripped variable
 635                          * name referenced by var_ref, and used when
 636                          * printing the action.  Because var_ref
 637                          * creation is deferred to create_actions(),
 638                          * we need a per-action way to save it until
 639                          * then, thus var_str.
 640                          */
 641                         char                    *var_str;
 642
 643                         /*
 644                          * var_ref refers to the variable being
 645                          * tracked e.g onmax($var).
 646                          */
 647                         struct hist_field       *var_ref;
 648
 649                         /*
 650                          * track_var contains the 'invisible' tracking
 651                          * variable created to keep the current
 652                          * e.g. max value.
 653                          */
 654                         struct hist_field       *track_var;
 655
 656                         check_track_val_fn_t    check_val;
 657                         action_fn_t             save_data;
 658                 } track_data;
 659         };
 660 };
 661
 662 struct track_data {
 663         u64                             track_val;
 664         bool                            updated;
 665
 666         unsigned int                    key_len;
 667         void                            *key;
 668         struct tracing_map_elt          elt;
 669
 670         struct action_data              *action_data;
 671         struct hist_trigger_data        *hist_data;
 672 };
 673
 674 struct hist_elt_data {
 675         char *comm;
 676         u64 *var_ref_vals;
 677         char **field_var_str;
 678         int n_field_var_str;
 679 };
 680
 681 struct snapshot_context {
 682         struct tracing_map_elt  *elt;
 683         void                    *key;
 684 };
 685
 686 /*
 687  * Returns the specific division function to use if the divisor
 688  * is constant. This avoids extra branches when the trigger is hit.
 689  */
 690 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
 691 {
 692         u64 div = divisor->constant;
 693
 694         if (!(div & (div - 1)))
 695                 return HIST_FIELD_FN_DIV_POWER2;
 696
 697         /* If the divisor is too large, do a regular division */
 698         if (div > (1 << HIST_DIV_SHIFT))
 699                 return HIST_FIELD_FN_DIV_NOT_POWER2;
 700
 701         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 702         return HIST_FIELD_FN_DIV_MULT_SHIFT;
 703 }
 704
 705 static void track_data_free(struct track_data *track_data)
 706 {
 707         struct hist_elt_data *elt_data;
 708
 709         if (!track_data)
 710                 return;
 711
 712         kfree(track_data->key);
 713
 714         elt_data = track_data->elt.private_data;
 715         if (elt_data) {
 716                 kfree(elt_data->comm);
 717                 kfree(elt_data);
 718         }
 719
 720         kfree(track_data);
 721 }
 722
 723 static struct track_data *track_data_alloc(unsigned int key_len,
 724                                            struct action_data *action_data,
 725                                            struct hist_trigger_data *hist_data)
 726 {
 727         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 728         struct hist_elt_data *elt_data;
 729
 730         if (!data)
 731                 return ERR_PTR(-ENOMEM);
 732
 733         data->key = kzalloc(key_len, GFP_KERNEL);
 734         if (!data->key) {
 735                 track_data_free(data);
 736                 return ERR_PTR(-ENOMEM);
 737         }
 738
 739         data->key_len = key_len;
 740         data->action_data = action_data;
 741         data->hist_data = hist_data;
 742
 743         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 744         if (!elt_data) {
 745                 track_data_free(data);
 746                 return ERR_PTR(-ENOMEM);
 747         }
 748
 749         data->elt.private_data = elt_data;
 750
 751         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 752         if (!elt_data->comm) {
 753                 track_data_free(data);
 754                 return ERR_PTR(-ENOMEM);
 755         }
 756
 757         return data;
 758 }
 759
 760 #define HIST_PREFIX "hist:"
 761
 762 static char *last_cmd;
 763 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 764
 765 static int errpos(char *str)
 766 {
 767         if (!str || !last_cmd)
 768                 return 0;
 769
 770         return err_pos(last_cmd, str);
 771 }
 772
 773 static void last_cmd_set(struct trace_event_file *file, char *str)
 774 {
 775         const char *system = NULL, *name = NULL;
 776         struct trace_event_call *call;
 777         int len;
 778
 779         if (!str)
 780                 return;
 781
 782         /* sizeof() contains the nul byte */
 783         len = sizeof(HIST_PREFIX) + strlen(str);
 784         kfree(last_cmd);
 785         last_cmd = kzalloc(len, GFP_KERNEL);
 786         if (!last_cmd)
 787                 return;
 788
 789         strcpy(last_cmd, HIST_PREFIX);
 790         /* Again, sizeof() contains the nul byte */
 791         len -= sizeof(HIST_PREFIX);
 792         strncat(last_cmd, str, len);
 793
 794         if (file) {
 795                 call = file->event_call;
 796                 system = call->class->system;
 797                 if (system) {
 798                         name = trace_event_name(call);
 799                         if (!name)
 800                                 system = NULL;
 801                 }
 802         }
 803
 804         if (system)
 805                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
 806 }
 807
 808 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
 809 {
 810         if (!last_cmd)
 811                 return;
 812
 813         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 814                         err_type, err_pos);
 815 }
 816
 817 static void hist_err_clear(void)
 818 {
 819         if (last_cmd)
 820                 last_cmd[0] = '\0';
 821         last_cmd_loc[0] = '\0';
 822 }
 823
 824 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 825                                     unsigned int *var_ref_idx);
 826
 827 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 828                                unsigned int *var_ref_idx)
 829 {
 830         struct tracepoint *tp = event->tp;
 831
 832         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 833                 struct tracepoint_func *probe_func_ptr;
 834                 synth_probe_func_t probe_func;
 835                 void *__data;
 836
 837                 if (!(cpu_online(raw_smp_processor_id())))
 838                         return;
 839
 840                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 841                 if (probe_func_ptr) {
 842                         do {
 843                                 probe_func = probe_func_ptr->func;
 844                                 __data = probe_func_ptr->data;
 845                                 probe_func(__data, var_ref_vals, var_ref_idx);
 846                         } while ((++probe_func_ptr)->func);
 847                 }
 848         }
 849 }
 850
 851 static void action_trace(struct hist_trigger_data *hist_data,
 852                          struct tracing_map_elt *elt,
 853                          struct trace_buffer *buffer, void *rec,
 854                          struct ring_buffer_event *rbe, void *key,
 855                          struct action_data *data, u64 *var_ref_vals)
 856 {
 857         struct synth_event *event = data->synth_event;
 858
 859         trace_synth(event, var_ref_vals, data->var_ref_idx);
 860 }
 861
 862 struct hist_var_data {
 863         struct list_head list;
 864         struct hist_trigger_data *hist_data;
 865 };
 866
 867 static u64 hist_field_timestamp(struct hist_field *hist_field,
 868                                 struct tracing_map_elt *elt,
 869                                 struct trace_buffer *buffer,
 870                                 struct ring_buffer_event *rbe,
 871                                 void *event)
 872 {
 873         struct hist_trigger_data *hist_data = hist_field->hist_data;
 874         struct trace_array *tr = hist_data->event_file->tr;
 875
 876         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 877
 878         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 879                 ts = ns2usecs(ts);
 880
 881         return ts;
 882 }
 883
 884 static u64 hist_field_cpu(struct hist_field *hist_field,
 885                           struct tracing_map_elt *elt,
 886                           struct trace_buffer *buffer,
 887                           struct ring_buffer_event *rbe,
 888                           void *event)
 889 {
 890         int cpu = smp_processor_id();
 891
 892         return cpu;
 893 }
 894
 895 /**
 896  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 897  * @hist_field: The VAR_REF field to check
 898  * @var_data: The hist trigger that owns the variable
 899  * @var_idx: The trigger variable identifier
 900  *
 901  * Check the given VAR_REF field to see whether or not it references
 902  * the given variable associated with the given trigger.
 903  *
 904  * Return: The VAR_REF field if it does reference the variable, NULL if not
 905  */
 906 static struct hist_field *
 907 check_field_for_var_ref(struct hist_field *hist_field,
 908                         struct hist_trigger_data *var_data,
 909                         unsigned int var_idx)
 910 {
 911         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 912
 913         if (hist_field && hist_field->var.idx == var_idx &&
 914             hist_field->var.hist_data == var_data)
 915                 return hist_field;
 916
 917         return NULL;
 918 }
 919
 920 /**
 921  * find_var_ref - Check if a trigger has a reference to a trigger variable
 922  * @hist_data: The hist trigger that might have a reference to the variable
 923  * @var_data: The hist trigger that owns the variable
 924  * @var_idx: The trigger variable identifier
 925  *
 926  * Check the list of var_refs[] on the first hist trigger to see
 927  * whether any of them are references to the variable on the second
 928  * trigger.
 929  *
 930  * Return: The VAR_REF field referencing the variable if so, NULL if not
 931  */
 932 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 933                                        struct hist_trigger_data *var_data,
 934                                        unsigned int var_idx)
 935 {
 936         struct hist_field *hist_field;
 937         unsigned int i;
 938
 939         for (i = 0; i < hist_data->n_var_refs; i++) {
 940                 hist_field = hist_data->var_refs[i];
 941                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 942                         return hist_field;
 943         }
 944
 945         return NULL;
 946 }
 947
 948 /**
 949  * find_any_var_ref - Check if there is a reference to a given trigger variable
 950  * @hist_data: The hist trigger
 951  * @var_idx: The trigger variable identifier
 952  *
 953  * Check to see whether the given variable is currently referenced by
 954  * any other trigger.
 955  *
 956  * The trigger the variable is defined on is explicitly excluded - the
 957  * assumption being that a self-reference doesn't prevent a trigger
 958  * from being removed.
 959  *
 960  * Return: The VAR_REF field referencing the variable if so, NULL if not
 961  */
 962 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 963                                            unsigned int var_idx)
 964 {
 965         struct trace_array *tr = hist_data->event_file->tr;
 966         struct hist_field *found = NULL;
 967         struct hist_var_data *var_data;
 968
 969         list_for_each_entry(var_data, &tr->hist_vars, list) {
 970                 if (var_data->hist_data == hist_data)
 971                         continue;
 972                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 973                 if (found)
 974                         break;
 975         }
 976
 977         return found;
 978 }
 979
 980 /**
 981  * check_var_refs - Check if there is a reference to any of trigger's variables
 982  * @hist_data: The hist trigger
 983  *
 984  * A trigger can define one or more variables.  If any one of them is
 985  * currently referenced by any other trigger, this function will
 986  * determine that.
 987  *
 988  * Typically used to determine whether or not a trigger can be removed
 989  * - if there are any references to a trigger's variables, it cannot.
 990  *
 991  * Return: True if there is a reference to any of trigger's variables
 992  */
 993 static bool check_var_refs(struct hist_trigger_data *hist_data)
 994 {
 995         struct hist_field *field;
 996         bool found = false;
 997         int i;
 998
 999         for_each_hist_field(i, hist_data) {
1000                 field = hist_data->fields[i];
1001                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1002                         if (find_any_var_ref(hist_data, field->var.idx)) {
1003                                 found = true;
1004                                 break;
1005                         }
1006                 }
1007         }
1008
1009         return found;
1010 }
1011
1012 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1013 {
1014         struct trace_array *tr = hist_data->event_file->tr;
1015         struct hist_var_data *var_data, *found = NULL;
1016
1017         list_for_each_entry(var_data, &tr->hist_vars, list) {
1018                 if (var_data->hist_data == hist_data) {
1019                         found = var_data;
1020                         break;
1021                 }
1022         }
1023
1024         return found;
1025 }
1026
1027 static bool field_has_hist_vars(struct hist_field *hist_field,
1028                                 unsigned int level)
1029 {
1030         int i;
1031
1032         if (level > 3)
1033                 return false;
1034
1035         if (!hist_field)
1036                 return false;
1037
1038         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1039             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1040                 return true;
1041
1042         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1043                 struct hist_field *operand;
1044
1045                 operand = hist_field->operands[i];
1046                 if (field_has_hist_vars(operand, level + 1))
1047                         return true;
1048         }
1049
1050         return false;
1051 }
1052
1053 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1054 {
1055         struct hist_field *hist_field;
1056         int i;
1057
1058         for_each_hist_field(i, hist_data) {
1059                 hist_field = hist_data->fields[i];
1060                 if (field_has_hist_vars(hist_field, 0))
1061                         return true;
1062         }
1063
1064         return false;
1065 }
1066
1067 static int save_hist_vars(struct hist_trigger_data *hist_data)
1068 {
1069         struct trace_array *tr = hist_data->event_file->tr;
1070         struct hist_var_data *var_data;
1071
1072         var_data = find_hist_vars(hist_data);
1073         if (var_data)
1074                 return 0;
1075
1076         if (tracing_check_open_get_tr(tr))
1077                 return -ENODEV;
1078
1079         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1080         if (!var_data) {
1081                 trace_array_put(tr);
1082                 return -ENOMEM;
1083         }
1084
1085         var_data->hist_data = hist_data;
1086         list_add(&var_data->list, &tr->hist_vars);
1087
1088         return 0;
1089 }
1090
1091 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1092 {
1093         struct trace_array *tr = hist_data->event_file->tr;
1094         struct hist_var_data *var_data;
1095
1096         var_data = find_hist_vars(hist_data);
1097         if (!var_data)
1098                 return;
1099
1100         if (WARN_ON(check_var_refs(hist_data)))
1101                 return;
1102
1103         list_del(&var_data->list);
1104
1105         kfree(var_data);
1106
1107         trace_array_put(tr);
1108 }
1109
1110 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1111                                          const char *var_name)
1112 {
1113         struct hist_field *hist_field, *found = NULL;
1114         int i;
1115
1116         for_each_hist_field(i, hist_data) {
1117                 hist_field = hist_data->fields[i];
1118                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1119                     strcmp(hist_field->var.name, var_name) == 0) {
1120                         found = hist_field;
1121                         break;
1122                 }
1123         }
1124
1125         return found;
1126 }
1127
1128 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1129                                    struct trace_event_file *file,
1130                                    const char *var_name)
1131 {
1132         struct hist_trigger_data *test_data;
1133         struct event_trigger_data *test;
1134         struct hist_field *hist_field;
1135
1136         lockdep_assert_held(&event_mutex);
1137
1138         hist_field = find_var_field(hist_data, var_name);
1139         if (hist_field)
1140                 return hist_field;
1141
1142         list_for_each_entry(test, &file->triggers, list) {
1143                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1144                         test_data = test->private_data;
1145                         hist_field = find_var_field(test_data, var_name);
1146                         if (hist_field)
1147                                 return hist_field;
1148                 }
1149         }
1150
1151         return NULL;
1152 }
1153
1154 static struct trace_event_file *find_var_file(struct trace_array *tr,
1155                                               char *system,
1156                                               char *event_name,
1157                                               char *var_name)
1158 {
1159         struct hist_trigger_data *var_hist_data;
1160         struct hist_var_data *var_data;
1161         struct trace_event_file *file, *found = NULL;
1162
1163         if (system)
1164                 return find_event_file(tr, system, event_name);
1165
1166         list_for_each_entry(var_data, &tr->hist_vars, list) {
1167                 var_hist_data = var_data->hist_data;
1168                 file = var_hist_data->event_file;
1169                 if (file == found)
1170                         continue;
1171
1172                 if (find_var_field(var_hist_data, var_name)) {
1173                         if (found) {
1174                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1175                                 return NULL;
1176                         }
1177
1178                         found = file;
1179                 }
1180         }
1181
1182         return found;
1183 }
1184
1185 static struct hist_field *find_file_var(struct trace_event_file *file,
1186                                         const char *var_name)
1187 {
1188         struct hist_trigger_data *test_data;
1189         struct event_trigger_data *test;
1190         struct hist_field *hist_field;
1191
1192         lockdep_assert_held(&event_mutex);
1193
1194         list_for_each_entry(test, &file->triggers, list) {
1195                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1196                         test_data = test->private_data;
1197                         hist_field = find_var_field(test_data, var_name);
1198                         if (hist_field)
1199                                 return hist_field;
1200                 }
1201         }
1202
1203         return NULL;
1204 }
1205
1206 static struct hist_field *
1207 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1208 {
1209         struct trace_array *tr = hist_data->event_file->tr;
1210         struct hist_field *hist_field, *found = NULL;
1211         struct trace_event_file *file;
1212         unsigned int i;
1213
1214         for (i = 0; i < hist_data->n_actions; i++) {
1215                 struct action_data *data = hist_data->actions[i];
1216
1217                 if (data->handler == HANDLER_ONMATCH) {
1218                         char *system = data->match_data.event_system;
1219                         char *event_name = data->match_data.event;
1220
1221                         file = find_var_file(tr, system, event_name, var_name);
1222                         if (!file)
1223                                 continue;
1224                         hist_field = find_file_var(file, var_name);
1225                         if (hist_field) {
1226                                 if (found) {
1227                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1228                                                  errpos(var_name));
1229                                         return ERR_PTR(-EINVAL);
1230                                 }
1231
1232                                 found = hist_field;
1233                         }
1234                 }
1235         }
1236         return found;
1237 }
1238
1239 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1240                                          char *system,
1241                                          char *event_name,
1242                                          char *var_name)
1243 {
1244         struct trace_array *tr = hist_data->event_file->tr;
1245         struct hist_field *hist_field = NULL;
1246         struct trace_event_file *file;
1247
1248         if (!system || !event_name) {
1249                 hist_field = find_match_var(hist_data, var_name);
1250                 if (IS_ERR(hist_field))
1251                         return NULL;
1252                 if (hist_field)
1253                         return hist_field;
1254         }
1255
1256         file = find_var_file(tr, system, event_name, var_name);
1257         if (!file)
1258                 return NULL;
1259
1260         hist_field = find_file_var(file, var_name);
1261
1262         return hist_field;
1263 }
1264
1265 static u64 hist_field_var_ref(struct hist_field *hist_field,
1266                               struct tracing_map_elt *elt,
1267                               struct trace_buffer *buffer,
1268                               struct ring_buffer_event *rbe,
1269                               void *event)
1270 {
1271         struct hist_elt_data *elt_data;
1272         u64 var_val = 0;
1273
1274         if (WARN_ON_ONCE(!elt))
1275                 return var_val;
1276
1277         elt_data = elt->private_data;
1278         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1279
1280         return var_val;
1281 }
1282
1283 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1284                              u64 *var_ref_vals, bool self)
1285 {
1286         struct hist_trigger_data *var_data;
1287         struct tracing_map_elt *var_elt;
1288         struct hist_field *hist_field;
1289         unsigned int i, var_idx;
1290         bool resolved = true;
1291         u64 var_val = 0;
1292
1293         for (i = 0; i < hist_data->n_var_refs; i++) {
1294                 hist_field = hist_data->var_refs[i];
1295                 var_idx = hist_field->var.idx;
1296                 var_data = hist_field->var.hist_data;
1297
1298                 if (var_data == NULL) {
1299                         resolved = false;
1300                         break;
1301                 }
1302
1303                 if ((self && var_data != hist_data) ||
1304                     (!self && var_data == hist_data))
1305                         continue;
1306
1307                 var_elt = tracing_map_lookup(var_data->map, key);
1308                 if (!var_elt) {
1309                         resolved = false;
1310                         break;
1311                 }
1312
1313                 if (!tracing_map_var_set(var_elt, var_idx)) {
1314                         resolved = false;
1315                         break;
1316                 }
1317
1318                 if (self || !hist_field->read_once)
1319                         var_val = tracing_map_read_var(var_elt, var_idx);
1320                 else
1321                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1322
1323                 var_ref_vals[i] = var_val;
1324         }
1325
1326         return resolved;
1327 }
1328
1329 static const char *hist_field_name(struct hist_field *field,
1330                                    unsigned int level)
1331 {
1332         const char *field_name = "";
1333
1334         if (WARN_ON_ONCE(!field))
1335                 return field_name;
1336
1337         if (level > 1)
1338                 return field_name;
1339
1340         if (field->field)
1341                 field_name = field->field->name;
1342         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1343                  field->flags & HIST_FIELD_FL_ALIAS ||
1344                  field->flags & HIST_FIELD_FL_BUCKET)
1345                 field_name = hist_field_name(field->operands[0], ++level);
1346         else if (field->flags & HIST_FIELD_FL_CPU)
1347                 field_name = "common_cpu";
1348         else if (field->flags & HIST_FIELD_FL_EXPR ||
1349                  field->flags & HIST_FIELD_FL_VAR_REF) {
1350                 if (field->system) {
1351                         static char full_name[MAX_FILTER_STR_VAL];
1352
1353                         strcat(full_name, field->system);
1354                         strcat(full_name, ".");
1355                         strcat(full_name, field->event_name);
1356                         strcat(full_name, ".");
1357                         strcat(full_name, field->name);
1358                         field_name = full_name;
1359                 } else
1360                         field_name = field->name;
1361         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1362                 field_name = "common_timestamp";
1363         else if (field->flags & HIST_FIELD_FL_STACKTRACE) {
1364                 if (field->field)
1365                         field_name = field->field->name;
1366                 else
1367                         field_name = "common_stacktrace";
1368         } else if (field->flags & HIST_FIELD_FL_HITCOUNT)
1369                 field_name = "hitcount";
1370
1371         if (field_name == NULL)
1372                 field_name = "";
1373
1374         return field_name;
1375 }
1376
1377 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1378 {
1379         switch (field_size) {
1380         case 8:
1381                 if (field_is_signed)
1382                         return HIST_FIELD_FN_S64;
1383                 else
1384                         return HIST_FIELD_FN_U64;
1385         case 4:
1386                 if (field_is_signed)
1387                         return HIST_FIELD_FN_S32;
1388                 else
1389                         return HIST_FIELD_FN_U32;
1390         case 2:
1391                 if (field_is_signed)
1392                         return HIST_FIELD_FN_S16;
1393                 else
1394                         return HIST_FIELD_FN_U16;
1395         case 1:
1396                 if (field_is_signed)
1397                         return HIST_FIELD_FN_S8;
1398                 else
1399                         return HIST_FIELD_FN_U8;
1400         }
1401
1402         return HIST_FIELD_FN_NOP;
1403 }
1404
1405 static int parse_map_size(char *str)
1406 {
1407         unsigned long size, map_bits;
1408         int ret;
1409
1410         ret = kstrtoul(str, 0, &size);
1411         if (ret)
1412                 goto out;
1413
1414         map_bits = ilog2(roundup_pow_of_two(size));
1415         if (map_bits < TRACING_MAP_BITS_MIN ||
1416             map_bits > TRACING_MAP_BITS_MAX)
1417                 ret = -EINVAL;
1418         else
1419                 ret = map_bits;
1420  out:
1421         return ret;
1422 }
1423
1424 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1425 {
1426         unsigned int i;
1427
1428         if (!attrs)
1429                 return;
1430
1431         for (i = 0; i < attrs->n_assignments; i++)
1432                 kfree(attrs->assignment_str[i]);
1433
1434         for (i = 0; i < attrs->n_actions; i++)
1435                 kfree(attrs->action_str[i]);
1436
1437         kfree(attrs->name);
1438         kfree(attrs->sort_key_str);
1439         kfree(attrs->keys_str);
1440         kfree(attrs->vals_str);
1441         kfree(attrs->clock);
1442         kfree(attrs);
1443 }
1444
1445 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1446 {
1447         int ret = -EINVAL;
1448
1449         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1450                 return ret;
1451
1452         if ((str_has_prefix(str, "onmatch(")) ||
1453             (str_has_prefix(str, "onmax(")) ||
1454             (str_has_prefix(str, "onchange("))) {
1455                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1456                 if (!attrs->action_str[attrs->n_actions]) {
1457                         ret = -ENOMEM;
1458                         return ret;
1459                 }
1460                 attrs->n_actions++;
1461                 ret = 0;
1462         }
1463         return ret;
1464 }
1465
1466 static int parse_assignment(struct trace_array *tr,
1467                             char *str, struct hist_trigger_attrs *attrs)
1468 {
1469         int len, ret = 0;
1470
1471         if ((len = str_has_prefix(str, "key=")) ||
1472             (len = str_has_prefix(str, "keys="))) {
1473                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1474                 if (!attrs->keys_str) {
1475                         ret = -ENOMEM;
1476                         goto out;
1477                 }
1478         } else if ((len = str_has_prefix(str, "val=")) ||
1479                    (len = str_has_prefix(str, "vals=")) ||
1480                    (len = str_has_prefix(str, "values="))) {
1481                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1482                 if (!attrs->vals_str) {
1483                         ret = -ENOMEM;
1484                         goto out;
1485                 }
1486         } else if ((len = str_has_prefix(str, "sort="))) {
1487                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1488                 if (!attrs->sort_key_str) {
1489                         ret = -ENOMEM;
1490                         goto out;
1491                 }
1492         } else if (str_has_prefix(str, "name=")) {
1493                 attrs->name = kstrdup(str, GFP_KERNEL);
1494                 if (!attrs->name) {
1495                         ret = -ENOMEM;
1496                         goto out;
1497                 }
1498         } else if ((len = str_has_prefix(str, "clock="))) {
1499                 str += len;
1500
1501                 str = strstrip(str);
1502                 attrs->clock = kstrdup(str, GFP_KERNEL);
1503                 if (!attrs->clock) {
1504                         ret = -ENOMEM;
1505                         goto out;
1506                 }
1507         } else if ((len = str_has_prefix(str, "size="))) {
1508                 int map_bits = parse_map_size(str + len);
1509
1510                 if (map_bits < 0) {
1511                         ret = map_bits;
1512                         goto out;
1513                 }
1514                 attrs->map_bits = map_bits;
1515         } else {
1516                 char *assignment;
1517
1518                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1519                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1520                         ret = -EINVAL;
1521                         goto out;
1522                 }
1523
1524                 assignment = kstrdup(str, GFP_KERNEL);
1525                 if (!assignment) {
1526                         ret = -ENOMEM;
1527                         goto out;
1528                 }
1529
1530                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1531         }
1532  out:
1533         return ret;
1534 }
1535
1536 static struct hist_trigger_attrs *
1537 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1538 {
1539         struct hist_trigger_attrs *attrs;
1540         int ret = 0;
1541
1542         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1543         if (!attrs)
1544                 return ERR_PTR(-ENOMEM);
1545
1546         while (trigger_str) {
1547                 char *str = strsep(&trigger_str, ":");
1548                 char *rhs;
1549
1550                 rhs = strchr(str, '=');
1551                 if (rhs) {
1552                         if (!strlen(++rhs)) {
1553                                 ret = -EINVAL;
1554                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1555                                 goto free;
1556                         }
1557                         ret = parse_assignment(tr, str, attrs);
1558                         if (ret)
1559                                 goto free;
1560                 } else if (strcmp(str, "nohitcount") == 0 ||
1561                            strcmp(str, "NOHC") == 0)
1562                         attrs->no_hitcount = true;
1563                 else if (strcmp(str, "pause") == 0)
1564                         attrs->pause = true;
1565                 else if ((strcmp(str, "cont") == 0) ||
1566                          (strcmp(str, "continue") == 0))
1567                         attrs->cont = true;
1568                 else if (strcmp(str, "clear") == 0)
1569                         attrs->clear = true;
1570                 else {
1571                         ret = parse_action(str, attrs);
1572                         if (ret)
1573                                 goto free;
1574                 }
1575         }
1576
1577         if (!attrs->keys_str) {
1578                 ret = -EINVAL;
1579                 goto free;
1580         }
1581
1582         if (!attrs->clock) {
1583                 attrs->clock = kstrdup("global", GFP_KERNEL);
1584                 if (!attrs->clock) {
1585                         ret = -ENOMEM;
1586                         goto free;
1587                 }
1588         }
1589
1590         return attrs;
1591  free:
1592         destroy_hist_trigger_attrs(attrs);
1593
1594         return ERR_PTR(ret);
1595 }
1596
1597 static inline void save_comm(char *comm, struct task_struct *task)
1598 {
1599         if (!task->pid) {
1600                 strcpy(comm, "<idle>");
1601                 return;
1602         }
1603
1604         if (WARN_ON_ONCE(task->pid < 0)) {
1605                 strcpy(comm, "<XXX>");
1606                 return;
1607         }
1608
1609         strncpy(comm, task->comm, TASK_COMM_LEN);
1610 }
1611
1612 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1613 {
1614         unsigned int i;
1615
1616         for (i = 0; i < elt_data->n_field_var_str; i++)
1617                 kfree(elt_data->field_var_str[i]);
1618
1619         kfree(elt_data->field_var_str);
1620
1621         kfree(elt_data->comm);
1622         kfree(elt_data);
1623 }
1624
1625 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1626 {
1627         struct hist_elt_data *elt_data = elt->private_data;
1628
1629         hist_elt_data_free(elt_data);
1630 }
1631
1632 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1633 {
1634         struct hist_trigger_data *hist_data = elt->map->private_data;
1635         unsigned int size = TASK_COMM_LEN;
1636         struct hist_elt_data *elt_data;
1637         struct hist_field *hist_field;
1638         unsigned int i, n_str;
1639
1640         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1641         if (!elt_data)
1642                 return -ENOMEM;
1643
1644         for_each_hist_field(i, hist_data) {
1645                 hist_field = hist_data->fields[i];
1646
1647                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1648                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1649                         if (!elt_data->comm) {
1650                                 kfree(elt_data);
1651                                 return -ENOMEM;
1652                         }
1653                         break;
1654                 }
1655         }
1656
1657         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1658                 hist_data->n_var_str;
1659         if (n_str > SYNTH_FIELDS_MAX) {
1660                 hist_elt_data_free(elt_data);
1661                 return -EINVAL;
1662         }
1663
1664         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1665
1666         size = STR_VAR_LEN_MAX;
1667
1668         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1669         if (!elt_data->field_var_str) {
1670                 hist_elt_data_free(elt_data);
1671                 return -EINVAL;
1672         }
1673         elt_data->n_field_var_str = n_str;
1674
1675         for (i = 0; i < n_str; i++) {
1676                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1677                 if (!elt_data->field_var_str[i]) {
1678                         hist_elt_data_free(elt_data);
1679                         return -ENOMEM;
1680                 }
1681         }
1682
1683         elt->private_data = elt_data;
1684
1685         return 0;
1686 }
1687
1688 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1689 {
1690         struct hist_elt_data *elt_data = elt->private_data;
1691
1692         if (elt_data->comm)
1693                 save_comm(elt_data->comm, current);
1694 }
1695
1696 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1697         .elt_alloc      = hist_trigger_elt_data_alloc,
1698         .elt_free       = hist_trigger_elt_data_free,
1699         .elt_init       = hist_trigger_elt_data_init,
1700 };
1701
1702 static const char *get_hist_field_flags(struct hist_field *hist_field)
1703 {
1704         const char *flags_str = NULL;
1705
1706         if (hist_field->flags & HIST_FIELD_FL_HEX)
1707                 flags_str = "hex";
1708         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1709                 flags_str = "sym";
1710         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1711                 flags_str = "sym-offset";
1712         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1713                 flags_str = "execname";
1714         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1715                 flags_str = "syscall";
1716         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1717                 flags_str = "log2";
1718         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1719                 flags_str = "buckets";
1720         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1721                 flags_str = "usecs";
1722         else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1723                 flags_str = "percent";
1724         else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1725                 flags_str = "graph";
1726         else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1727                 flags_str = "stacktrace";
1728
1729         return flags_str;
1730 }
1731
1732 static void expr_field_str(struct hist_field *field, char *expr)
1733 {
1734         if (field->flags & HIST_FIELD_FL_VAR_REF)
1735                 strcat(expr, "$");
1736         else if (field->flags & HIST_FIELD_FL_CONST) {
1737                 char str[HIST_CONST_DIGITS_MAX];
1738
1739                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1740                 strcat(expr, str);
1741         }
1742
1743         strcat(expr, hist_field_name(field, 0));
1744
1745         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1746                 const char *flags_str = get_hist_field_flags(field);
1747
1748                 if (flags_str) {
1749                         strcat(expr, ".");
1750                         strcat(expr, flags_str);
1751                 }
1752         }
1753 }
1754
1755 static char *expr_str(struct hist_field *field, unsigned int level)
1756 {
1757         char *expr;
1758
1759         if (level > 1)
1760                 return NULL;
1761
1762         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1763         if (!expr)
1764                 return NULL;
1765
1766         if (!field->operands[0]) {
1767                 expr_field_str(field, expr);
1768                 return expr;
1769         }
1770
1771         if (field->operator == FIELD_OP_UNARY_MINUS) {
1772                 char *subexpr;
1773
1774                 strcat(expr, "-(");
1775                 subexpr = expr_str(field->operands[0], ++level);
1776                 if (!subexpr) {
1777                         kfree(expr);
1778                         return NULL;
1779                 }
1780                 strcat(expr, subexpr);
1781                 strcat(expr, ")");
1782
1783                 kfree(subexpr);
1784
1785                 return expr;
1786         }
1787
1788         expr_field_str(field->operands[0], expr);
1789
1790         switch (field->operator) {
1791         case FIELD_OP_MINUS:
1792                 strcat(expr, "-");
1793                 break;
1794         case FIELD_OP_PLUS:
1795                 strcat(expr, "+");
1796                 break;
1797         case FIELD_OP_DIV:
1798                 strcat(expr, "/");
1799                 break;
1800         case FIELD_OP_MULT:
1801                 strcat(expr, "*");
1802                 break;
1803         default:
1804                 kfree(expr);
1805                 return NULL;
1806         }
1807
1808         expr_field_str(field->operands[1], expr);
1809
1810         return expr;
1811 }
1812
1813 /*
1814  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1815  * of the expression tree to be evaluated.
1816  */
1817 static int contains_operator(char *str, char **sep)
1818 {
1819         enum field_op_id field_op = FIELD_OP_NONE;
1820         char *minus_op, *plus_op, *div_op, *mult_op;
1821
1822
1823         /*
1824          * Report the last occurrence of the operators first, so that the
1825          * expression is evaluated left to right. This is important since
1826          * subtraction and division are not associative.
1827          *
1828          *      e.g
1829          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1830          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1831          */
1832
1833         /*
1834          * First, find lower precedence addition and subtraction
1835          * since the expression will be evaluated recursively.
1836          */
1837         minus_op = strrchr(str, '-');
1838         if (minus_op) {
1839                 /*
1840                  * Unary minus is not supported in sub-expressions. If
1841                  * present, it is always the next root operator.
1842                  */
1843                 if (minus_op == str) {
1844                         field_op = FIELD_OP_UNARY_MINUS;
1845                         goto out;
1846                 }
1847
1848                 field_op = FIELD_OP_MINUS;
1849         }
1850
1851         plus_op = strrchr(str, '+');
1852         if (plus_op || minus_op) {
1853                 /*
1854                  * For operators of the same precedence use to rightmost as the
1855                  * root, so that the expression is evaluated left to right.
1856                  */
1857                 if (plus_op > minus_op)
1858                         field_op = FIELD_OP_PLUS;
1859                 goto out;
1860         }
1861
1862         /*
1863          * Multiplication and division have higher precedence than addition and
1864          * subtraction.
1865          */
1866         div_op = strrchr(str, '/');
1867         if (div_op)
1868                 field_op = FIELD_OP_DIV;
1869
1870         mult_op = strrchr(str, '*');
1871         /*
1872          * For operators of the same precedence use to rightmost as the
1873          * root, so that the expression is evaluated left to right.
1874          */
1875         if (mult_op > div_op)
1876                 field_op = FIELD_OP_MULT;
1877
1878 out:
1879         if (sep) {
1880                 switch (field_op) {
1881                 case FIELD_OP_UNARY_MINUS:
1882                 case FIELD_OP_MINUS:
1883                         *sep = minus_op;
1884                         break;
1885                 case FIELD_OP_PLUS:
1886                         *sep = plus_op;
1887                         break;
1888                 case FIELD_OP_DIV:
1889                         *sep = div_op;
1890                         break;
1891                 case FIELD_OP_MULT:
1892                         *sep = mult_op;
1893                         break;
1894                 case FIELD_OP_NONE:
1895                 default:
1896                         *sep = NULL;
1897                         break;
1898                 }
1899         }
1900
1901         return field_op;
1902 }
1903
1904 static void get_hist_field(struct hist_field *hist_field)
1905 {
1906         hist_field->ref++;
1907 }
1908
1909 static void __destroy_hist_field(struct hist_field *hist_field)
1910 {
1911         if (--hist_field->ref > 1)
1912                 return;
1913
1914         kfree(hist_field->var.name);
1915         kfree(hist_field->name);
1916
1917         /* Can likely be a const */
1918         kfree_const(hist_field->type);
1919
1920         kfree(hist_field->system);
1921         kfree(hist_field->event_name);
1922
1923         kfree(hist_field);
1924 }
1925
1926 static void destroy_hist_field(struct hist_field *hist_field,
1927                                unsigned int level)
1928 {
1929         unsigned int i;
1930
1931         if (level > 3)
1932                 return;
1933
1934         if (!hist_field)
1935                 return;
1936
1937         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1938                 return; /* var refs will be destroyed separately */
1939
1940         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1941                 destroy_hist_field(hist_field->operands[i], level + 1);
1942
1943         __destroy_hist_field(hist_field);
1944 }
1945
1946 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1947                                             struct ftrace_event_field *field,
1948                                             unsigned long flags,
1949                                             char *var_name)
1950 {
1951         struct hist_field *hist_field;
1952
1953         if (field && is_function_field(field))
1954                 return NULL;
1955
1956         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1957         if (!hist_field)
1958                 return NULL;
1959
1960         hist_field->ref = 1;
1961
1962         hist_field->hist_data = hist_data;
1963
1964         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1965                 goto out; /* caller will populate */
1966
1967         if (flags & HIST_FIELD_FL_VAR_REF) {
1968                 hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1969                 goto out;
1970         }
1971
1972         if (flags & HIST_FIELD_FL_HITCOUNT) {
1973                 hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1974                 hist_field->size = sizeof(u64);
1975                 hist_field->type = "u64";
1976                 goto out;
1977         }
1978
1979         if (flags & HIST_FIELD_FL_CONST) {
1980                 hist_field->fn_num = HIST_FIELD_FN_CONST;
1981                 hist_field->size = sizeof(u64);
1982                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1983                 if (!hist_field->type)
1984                         goto free;
1985                 goto out;
1986         }
1987
1988         if (flags & HIST_FIELD_FL_STACKTRACE) {
1989                 if (field)
1990                         hist_field->fn_num = HIST_FIELD_FN_STACK;
1991                 else
1992                         hist_field->fn_num = HIST_FIELD_FN_NOP;
1993                 hist_field->size = HIST_STACKTRACE_SIZE;
1994                 hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL);
1995                 if (!hist_field->type)
1996                         goto free;
1997                 goto out;
1998         }
1999
2000         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
2001                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
2002                 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
2003                         HIST_FIELD_FN_BUCKET;
2004                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2005                 if (!hist_field->operands[0])
2006                         goto free;
2007                 hist_field->size = hist_field->operands[0]->size;
2008                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
2009                 if (!hist_field->type)
2010                         goto free;
2011                 goto out;
2012         }
2013
2014         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2015                 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
2016                 hist_field->size = sizeof(u64);
2017                 hist_field->type = "u64";
2018                 goto out;
2019         }
2020
2021         if (flags & HIST_FIELD_FL_CPU) {
2022                 hist_field->fn_num = HIST_FIELD_FN_CPU;
2023                 hist_field->size = sizeof(int);
2024                 hist_field->type = "unsigned int";
2025                 goto out;
2026         }
2027
2028         if (WARN_ON_ONCE(!field))
2029                 goto out;
2030
2031         /* Pointers to strings are just pointers and dangerous to dereference */
2032         if (is_string_field(field) &&
2033             (field->filter_type != FILTER_PTR_STRING)) {
2034                 flags |= HIST_FIELD_FL_STRING;
2035
2036                 hist_field->size = MAX_FILTER_STR_VAL;
2037                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2038                 if (!hist_field->type)
2039                         goto free;
2040
2041                 if (field->filter_type == FILTER_STATIC_STRING) {
2042                         hist_field->fn_num = HIST_FIELD_FN_STRING;
2043                         hist_field->size = field->size;
2044                 } else if (field->filter_type == FILTER_DYN_STRING) {
2045                         hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2046                 } else if (field->filter_type == FILTER_RDYN_STRING)
2047                         hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2048                 else
2049                         hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2050         } else {
2051                 hist_field->size = field->size;
2052                 hist_field->is_signed = field->is_signed;
2053                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2054                 if (!hist_field->type)
2055                         goto free;
2056
2057                 hist_field->fn_num = select_value_fn(field->size,
2058                                                      field->is_signed);
2059                 if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2060                         destroy_hist_field(hist_field, 0);
2061                         return NULL;
2062                 }
2063         }
2064  out:
2065         hist_field->field = field;
2066         hist_field->flags = flags;
2067
2068         if (var_name) {
2069                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2070                 if (!hist_field->var.name)
2071                         goto free;
2072         }
2073
2074         return hist_field;
2075  free:
2076         destroy_hist_field(hist_field, 0);
2077         return NULL;
2078 }
2079
2080 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2081 {
2082         unsigned int i;
2083
2084         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2085                 if (hist_data->fields[i]) {
2086                         destroy_hist_field(hist_data->fields[i], 0);
2087                         hist_data->fields[i] = NULL;
2088                 }
2089         }
2090
2091         for (i = 0; i < hist_data->n_var_refs; i++) {
2092                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2093                 __destroy_hist_field(hist_data->var_refs[i]);
2094                 hist_data->var_refs[i] = NULL;
2095         }
2096 }
2097
2098 static int init_var_ref(struct hist_field *ref_field,
2099                         struct hist_field *var_field,
2100                         char *system, char *event_name)
2101 {
2102         int err = 0;
2103
2104         ref_field->var.idx = var_field->var.idx;
2105         ref_field->var.hist_data = var_field->hist_data;
2106         ref_field->size = var_field->size;
2107         ref_field->is_signed = var_field->is_signed;
2108         ref_field->flags |= var_field->flags &
2109                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2110
2111         if (system) {
2112                 ref_field->system = kstrdup(system, GFP_KERNEL);
2113                 if (!ref_field->system)
2114                         return -ENOMEM;
2115         }
2116
2117         if (event_name) {
2118                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2119                 if (!ref_field->event_name) {
2120                         err = -ENOMEM;
2121                         goto free;
2122                 }
2123         }
2124
2125         if (var_field->var.name) {
2126                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2127                 if (!ref_field->name) {
2128                         err = -ENOMEM;
2129                         goto free;
2130                 }
2131         } else if (var_field->name) {
2132                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2133                 if (!ref_field->name) {
2134                         err = -ENOMEM;
2135                         goto free;
2136                 }
2137         }
2138
2139         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2140         if (!ref_field->type) {
2141                 err = -ENOMEM;
2142                 goto free;
2143         }
2144  out:
2145         return err;
2146  free:
2147         kfree(ref_field->system);
2148         ref_field->system = NULL;
2149         kfree(ref_field->event_name);
2150         ref_field->event_name = NULL;
2151         kfree(ref_field->name);
2152         ref_field->name = NULL;
2153
2154         goto out;
2155 }
2156
2157 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2158                             struct hist_field *var_field)
2159 {
2160         struct hist_field *ref_field;
2161         int i;
2162
2163         for (i = 0; i < hist_data->n_var_refs; i++) {
2164                 ref_field = hist_data->var_refs[i];
2165                 if (ref_field->var.idx == var_field->var.idx &&
2166                     ref_field->var.hist_data == var_field->hist_data)
2167                         return i;
2168         }
2169
2170         return -ENOENT;
2171 }
2172
2173 /**
2174  * create_var_ref - Create a variable reference and attach it to trigger
2175  * @hist_data: The trigger that will be referencing the variable
2176  * @var_field: The VAR field to create a reference to
2177  * @system: The optional system string
2178  * @event_name: The optional event_name string
2179  *
2180  * Given a variable hist_field, create a VAR_REF hist_field that
2181  * represents a reference to it.
2182  *
2183  * This function also adds the reference to the trigger that
2184  * now references the variable.
2185  *
2186  * Return: The VAR_REF field if successful, NULL if not
2187  */
2188 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2189                                          struct hist_field *var_field,
2190                                          char *system, char *event_name)
2191 {
2192         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2193         struct hist_field *ref_field;
2194         int i;
2195
2196         /* Check if the variable already exists */
2197         for (i = 0; i < hist_data->n_var_refs; i++) {
2198                 ref_field = hist_data->var_refs[i];
2199                 if (ref_field->var.idx == var_field->var.idx &&
2200                     ref_field->var.hist_data == var_field->hist_data) {
2201                         get_hist_field(ref_field);
2202                         return ref_field;
2203                 }
2204         }
2205         /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2206         if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2207                 return NULL;
2208         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2209         if (ref_field) {
2210                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2211                         destroy_hist_field(ref_field, 0);
2212                         return NULL;
2213                 }
2214
2215                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2216                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2217         }
2218
2219         return ref_field;
2220 }
2221
2222 static bool is_var_ref(char *var_name)
2223 {
2224         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2225                 return false;
2226
2227         return true;
2228 }
2229
2230 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2231                                  char *var_name)
2232 {
2233         char *name, *field;
2234         unsigned int i;
2235
2236         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2237                 name = hist_data->attrs->var_defs.name[i];
2238
2239                 if (strcmp(var_name, name) == 0) {
2240                         field = hist_data->attrs->var_defs.expr[i];
2241                         if (contains_operator(field, NULL) || is_var_ref(field))
2242                                 continue;
2243                         return field;
2244                 }
2245         }
2246
2247         return NULL;
2248 }
2249
2250 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2251                                  char *system, char *event_name,
2252                                  char *var_name)
2253 {
2254         struct trace_event_call *call;
2255
2256         if (system && event_name) {
2257                 call = hist_data->event_file->event_call;
2258
2259                 if (strcmp(system, call->class->system) != 0)
2260                         return NULL;
2261
2262                 if (strcmp(event_name, trace_event_name(call)) != 0)
2263                         return NULL;
2264         }
2265
2266         if (!!system != !!event_name)
2267                 return NULL;
2268
2269         if (!is_var_ref(var_name))
2270                 return NULL;
2271
2272         var_name++;
2273
2274         return field_name_from_var(hist_data, var_name);
2275 }
2276
2277 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2278                                         char *system, char *event_name,
2279                                         char *var_name)
2280 {
2281         struct hist_field *var_field = NULL, *ref_field = NULL;
2282         struct trace_array *tr = hist_data->event_file->tr;
2283
2284         if (!is_var_ref(var_name))
2285                 return NULL;
2286
2287         var_name++;
2288
2289         var_field = find_event_var(hist_data, system, event_name, var_name);
2290         if (var_field)
2291                 ref_field = create_var_ref(hist_data, var_field,
2292                                            system, event_name);
2293
2294         if (!ref_field)
2295                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2296
2297         return ref_field;
2298 }
2299
2300 static struct ftrace_event_field *
2301 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2302             char *field_str, unsigned long *flags, unsigned long *buckets)
2303 {
2304         struct ftrace_event_field *field = NULL;
2305         char *field_name, *modifier, *str;
2306         struct trace_array *tr = file->tr;
2307
2308         modifier = str = kstrdup(field_str, GFP_KERNEL);
2309         if (!modifier)
2310                 return ERR_PTR(-ENOMEM);
2311
2312         field_name = strsep(&modifier, ".");
2313         if (modifier) {
2314                 if (strcmp(modifier, "hex") == 0)
2315                         *flags |= HIST_FIELD_FL_HEX;
2316                 else if (strcmp(modifier, "sym") == 0)
2317                         *flags |= HIST_FIELD_FL_SYM;
2318                 /*
2319                  * 'sym-offset' occurrences in the trigger string are modified
2320                  * to 'symXoffset' to simplify arithmetic expression parsing.
2321                  */
2322                 else if (strcmp(modifier, "symXoffset") == 0)
2323                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2324                 else if ((strcmp(modifier, "execname") == 0) &&
2325                          (strcmp(field_name, "common_pid") == 0))
2326                         *flags |= HIST_FIELD_FL_EXECNAME;
2327                 else if (strcmp(modifier, "syscall") == 0)
2328                         *flags |= HIST_FIELD_FL_SYSCALL;
2329                 else if (strcmp(modifier, "stacktrace") == 0)
2330                         *flags |= HIST_FIELD_FL_STACKTRACE;
2331                 else if (strcmp(modifier, "log2") == 0)
2332                         *flags |= HIST_FIELD_FL_LOG2;
2333                 else if (strcmp(modifier, "usecs") == 0)
2334                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2335                 else if (strncmp(modifier, "bucket", 6) == 0) {
2336                         int ret;
2337
2338                         modifier += 6;
2339
2340                         if (*modifier == 's')
2341                                 modifier++;
2342                         if (*modifier != '=')
2343                                 goto error;
2344                         modifier++;
2345                         ret = kstrtoul(modifier, 0, buckets);
2346                         if (ret || !(*buckets))
2347                                 goto error;
2348                         *flags |= HIST_FIELD_FL_BUCKET;
2349                 } else if (strncmp(modifier, "percent", 7) == 0) {
2350                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2351                                 goto error;
2352                         *flags |= HIST_FIELD_FL_PERCENT;
2353                 } else if (strncmp(modifier, "graph", 5) == 0) {
2354                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2355                                 goto error;
2356                         *flags |= HIST_FIELD_FL_GRAPH;
2357                 } else {
2358  error:
2359                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2360                         field = ERR_PTR(-EINVAL);
2361                         goto out;
2362                 }
2363         }
2364
2365         if (strcmp(field_name, "common_timestamp") == 0) {
2366                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2367                 hist_data->enable_timestamps = true;
2368                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2369                         hist_data->attrs->ts_in_usecs = true;
2370         } else if (strcmp(field_name, "common_stacktrace") == 0) {
2371                 *flags |= HIST_FIELD_FL_STACKTRACE;
2372         } else if (strcmp(field_name, "common_cpu") == 0)
2373                 *flags |= HIST_FIELD_FL_CPU;
2374         else if (strcmp(field_name, "hitcount") == 0)
2375                 *flags |= HIST_FIELD_FL_HITCOUNT;
2376         else {
2377                 field = trace_find_event_field(file->event_call, field_name);
2378                 if (!field || !field->size) {
2379                         /*
2380                          * For backward compatibility, if field_name
2381                          * was "cpu" or "stacktrace", then we treat this
2382                          * the same as common_cpu and common_stacktrace
2383                          * respectively. This also works for "CPU", and
2384                          * "STACKTRACE".
2385                          */
2386                         if (field && field->filter_type == FILTER_CPU) {
2387                                 *flags |= HIST_FIELD_FL_CPU;
2388                         } else if (field && field->filter_type == FILTER_STACKTRACE) {
2389                                 *flags |= HIST_FIELD_FL_STACKTRACE;
2390                         } else {
2391                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2392                                          errpos(field_name));
2393                                 field = ERR_PTR(-EINVAL);
2394                                 goto out;
2395                         }
2396                 }
2397         }
2398  out:
2399         kfree(str);
2400
2401         return field;
2402 }
2403
2404 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2405                                        struct hist_field *var_ref,
2406                                        char *var_name)
2407 {
2408         struct hist_field *alias = NULL;
2409         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2410
2411         alias = create_hist_field(hist_data, NULL, flags, var_name);
2412         if (!alias)
2413                 return NULL;
2414
2415         alias->fn_num = var_ref->fn_num;
2416         alias->operands[0] = var_ref;
2417
2418         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2419                 destroy_hist_field(alias, 0);
2420                 return NULL;
2421         }
2422
2423         alias->var_ref_idx = var_ref->var_ref_idx;
2424
2425         return alias;
2426 }
2427
2428 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2429                                       char *str, char *var_name,
2430                                       unsigned long *flags)
2431 {
2432         struct trace_array *tr = hist_data->event_file->tr;
2433         struct hist_field *field = NULL;
2434         u64 constant;
2435
2436         if (kstrtoull(str, 0, &constant)) {
2437                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2438                 return NULL;
2439         }
2440
2441         *flags |= HIST_FIELD_FL_CONST;
2442         field = create_hist_field(hist_data, NULL, *flags, var_name);
2443         if (!field)
2444                 return NULL;
2445
2446         field->constant = constant;
2447
2448         return field;
2449 }
2450
2451 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2452                                      struct trace_event_file *file, char *str,
2453                                      unsigned long *flags, char *var_name)
2454 {
2455         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2456         struct ftrace_event_field *field = NULL;
2457         struct hist_field *hist_field = NULL;
2458         unsigned long buckets = 0;
2459         int ret = 0;
2460
2461         if (isdigit(str[0])) {
2462                 hist_field = parse_const(hist_data, str, var_name, flags);
2463                 if (!hist_field) {
2464                         ret = -EINVAL;
2465                         goto out;
2466                 }
2467                 return hist_field;
2468         }
2469
2470         s = strchr(str, '.');
2471         if (s) {
2472                 s = strchr(++s, '.');
2473                 if (s) {
2474                         ref_system = strsep(&str, ".");
2475                         if (!str) {
2476                                 ret = -EINVAL;
2477                                 goto out;
2478                         }
2479                         ref_event = strsep(&str, ".");
2480                         if (!str) {
2481                                 ret = -EINVAL;
2482                                 goto out;
2483                         }
2484                         ref_var = str;
2485                 }
2486         }
2487
2488         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2489         if (!s) {
2490                 hist_field = parse_var_ref(hist_data, ref_system,
2491                                            ref_event, ref_var);
2492                 if (hist_field) {
2493                         if (var_name) {
2494                                 hist_field = create_alias(hist_data, hist_field, var_name);
2495                                 if (!hist_field) {
2496                                         ret = -ENOMEM;
2497                                         goto out;
2498                                 }
2499                         }
2500                         return hist_field;
2501                 }
2502         } else
2503                 str = s;
2504
2505         field = parse_field(hist_data, file, str, flags, &buckets);
2506         if (IS_ERR(field)) {
2507                 ret = PTR_ERR(field);
2508                 goto out;
2509         }
2510
2511         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2512         if (!hist_field) {
2513                 ret = -ENOMEM;
2514                 goto out;
2515         }
2516         hist_field->buckets = buckets;
2517
2518         return hist_field;
2519  out:
2520         return ERR_PTR(ret);
2521 }
2522
2523 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2524                                      struct trace_event_file *file,
2525                                      char *str, unsigned long flags,
2526                                      char *var_name, unsigned int *n_subexprs);
2527
2528 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2529                                       struct trace_event_file *file,
2530                                       char *str, unsigned long flags,
2531                                       char *var_name, unsigned int *n_subexprs)
2532 {
2533         struct hist_field *operand1, *expr = NULL;
2534         unsigned long operand_flags;
2535         int ret = 0;
2536         char *s;
2537
2538         /* Unary minus operator, increment n_subexprs */
2539         ++*n_subexprs;
2540
2541         /* we support only -(xxx) i.e. explicit parens required */
2542
2543         if (*n_subexprs > 3) {
2544                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2545                 ret = -EINVAL;
2546                 goto free;
2547         }
2548
2549         str++; /* skip leading '-' */
2550
2551         s = strchr(str, '(');
2552         if (s)
2553                 str++;
2554         else {
2555                 ret = -EINVAL;
2556                 goto free;
2557         }
2558
2559         s = strrchr(str, ')');
2560         if (s) {
2561                  /* unary minus not supported in sub-expressions */
2562                 if (*(s+1) != '\0') {
2563                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2564                                  errpos(str));
2565                         ret = -EINVAL;
2566                         goto free;
2567                 }
2568                 *s = '\0';
2569         }
2570         else {
2571                 ret = -EINVAL; /* no closing ')' */
2572                 goto free;
2573         }
2574
2575         flags |= HIST_FIELD_FL_EXPR;
2576         expr = create_hist_field(hist_data, NULL, flags, var_name);
2577         if (!expr) {
2578                 ret = -ENOMEM;
2579                 goto free;
2580         }
2581
2582         operand_flags = 0;
2583         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2584         if (IS_ERR(operand1)) {
2585                 ret = PTR_ERR(operand1);
2586                 goto free;
2587         }
2588         if (operand1->flags & HIST_FIELD_FL_STRING) {
2589                 /* String type can not be the operand of unary operator. */
2590                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2591                 destroy_hist_field(operand1, 0);
2592                 ret = -EINVAL;
2593                 goto free;
2594         }
2595
2596         expr->flags |= operand1->flags &
2597                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2598         expr->fn_num = HIST_FIELD_FN_UMINUS;
2599         expr->operands[0] = operand1;
2600         expr->size = operand1->size;
2601         expr->is_signed = operand1->is_signed;
2602         expr->operator = FIELD_OP_UNARY_MINUS;
2603         expr->name = expr_str(expr, 0);
2604         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2605         if (!expr->type) {
2606                 ret = -ENOMEM;
2607                 goto free;
2608         }
2609
2610         return expr;
2611  free:
2612         destroy_hist_field(expr, 0);
2613         return ERR_PTR(ret);
2614 }
2615
2616 /*
2617  * If the operands are var refs, return pointers the
2618  * variable(s) referenced in var1 and var2, else NULL.
2619  */
2620 static int check_expr_operands(struct trace_array *tr,
2621                                struct hist_field *operand1,
2622                                struct hist_field *operand2,
2623                                struct hist_field **var1,
2624                                struct hist_field **var2)
2625 {
2626         unsigned long operand1_flags = operand1->flags;
2627         unsigned long operand2_flags = operand2->flags;
2628
2629         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2630             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2631                 struct hist_field *var;
2632
2633                 var = find_var_field(operand1->var.hist_data, operand1->name);
2634                 if (!var)
2635                         return -EINVAL;
2636                 operand1_flags = var->flags;
2637                 *var1 = var;
2638         }
2639
2640         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2641             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2642                 struct hist_field *var;
2643
2644                 var = find_var_field(operand2->var.hist_data, operand2->name);
2645                 if (!var)
2646                         return -EINVAL;
2647                 operand2_flags = var->flags;
2648                 *var2 = var;
2649         }
2650
2651         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2652             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2653                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2654                 return -EINVAL;
2655         }
2656
2657         return 0;
2658 }
2659
2660 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2661                                      struct trace_event_file *file,
2662                                      char *str, unsigned long flags,
2663                                      char *var_name, unsigned int *n_subexprs)
2664 {
2665         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2666         struct hist_field *var1 = NULL, *var2 = NULL;
2667         unsigned long operand_flags, operand2_flags;
2668         int field_op, ret = -EINVAL;
2669         char *sep, *operand1_str;
2670         enum hist_field_fn op_fn;
2671         bool combine_consts;
2672
2673         if (*n_subexprs > 3) {
2674                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2675                 return ERR_PTR(-EINVAL);
2676         }
2677
2678         field_op = contains_operator(str, &sep);
2679
2680         if (field_op == FIELD_OP_NONE)
2681                 return parse_atom(hist_data, file, str, &flags, var_name);
2682
2683         if (field_op == FIELD_OP_UNARY_MINUS)
2684                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2685
2686         /* Binary operator found, increment n_subexprs */
2687         ++*n_subexprs;
2688
2689         /* Split the expression string at the root operator */
2690         if (!sep)
2691                 return ERR_PTR(-EINVAL);
2692
2693         *sep = '\0';
2694         operand1_str = str;
2695         str = sep+1;
2696
2697         /* Binary operator requires both operands */
2698         if (*operand1_str == '\0' || *str == '\0')
2699                 return ERR_PTR(-EINVAL);
2700
2701         operand_flags = 0;
2702
2703         /* LHS of string is an expression e.g. a+b in a+b+c */
2704         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2705         if (IS_ERR(operand1))
2706                 return ERR_CAST(operand1);
2707
2708         if (operand1->flags & HIST_FIELD_FL_STRING) {
2709                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2710                 ret = -EINVAL;
2711                 goto free_op1;
2712         }
2713
2714         /* RHS of string is another expression e.g. c in a+b+c */
2715         operand_flags = 0;
2716         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2717         if (IS_ERR(operand2)) {
2718                 ret = PTR_ERR(operand2);
2719                 goto free_op1;
2720         }
2721         if (operand2->flags & HIST_FIELD_FL_STRING) {
2722                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2723                 ret = -EINVAL;
2724                 goto free_operands;
2725         }
2726
2727         switch (field_op) {
2728         case FIELD_OP_MINUS:
2729                 op_fn = HIST_FIELD_FN_MINUS;
2730                 break;
2731         case FIELD_OP_PLUS:
2732                 op_fn = HIST_FIELD_FN_PLUS;
2733                 break;
2734         case FIELD_OP_DIV:
2735                 op_fn = HIST_FIELD_FN_DIV;
2736                 break;
2737         case FIELD_OP_MULT:
2738                 op_fn = HIST_FIELD_FN_MULT;
2739                 break;
2740         default:
2741                 ret = -EINVAL;
2742                 goto free_operands;
2743         }
2744
2745         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2746         if (ret)
2747                 goto free_operands;
2748
2749         operand_flags = var1 ? var1->flags : operand1->flags;
2750         operand2_flags = var2 ? var2->flags : operand2->flags;
2751
2752         /*
2753          * If both operands are constant, the expression can be
2754          * collapsed to a single constant.
2755          */
2756         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2757
2758         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2759
2760         flags |= operand1->flags &
2761                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2762
2763         expr = create_hist_field(hist_data, NULL, flags, var_name);
2764         if (!expr) {
2765                 ret = -ENOMEM;
2766                 goto free_operands;
2767         }
2768
2769         operand1->read_once = true;
2770         operand2->read_once = true;
2771
2772         /* The operands are now owned and free'd by 'expr' */
2773         expr->operands[0] = operand1;
2774         expr->operands[1] = operand2;
2775
2776         if (field_op == FIELD_OP_DIV &&
2777                         operand2_flags & HIST_FIELD_FL_CONST) {
2778                 u64 divisor = var2 ? var2->constant : operand2->constant;
2779
2780                 if (!divisor) {
2781                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2782                         ret = -EDOM;
2783                         goto free_expr;
2784                 }
2785
2786                 /*
2787                  * Copy the divisor here so we don't have to look it up
2788                  * later if this is a var ref
2789                  */
2790                 operand2->constant = divisor;
2791                 op_fn = hist_field_get_div_fn(operand2);
2792         }
2793
2794         expr->fn_num = op_fn;
2795
2796         if (combine_consts) {
2797                 if (var1)
2798                         expr->operands[0] = var1;
2799                 if (var2)
2800                         expr->operands[1] = var2;
2801
2802                 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2803                 expr->fn_num = HIST_FIELD_FN_CONST;
2804
2805                 expr->operands[0] = NULL;
2806                 expr->operands[1] = NULL;
2807
2808                 /*
2809                  * var refs won't be destroyed immediately
2810                  * See: destroy_hist_field()
2811                  */
2812                 destroy_hist_field(operand2, 0);
2813                 destroy_hist_field(operand1, 0);
2814
2815                 expr->name = expr_str(expr, 0);
2816         } else {
2817                 /* The operand sizes should be the same, so just pick one */
2818                 expr->size = operand1->size;
2819                 expr->is_signed = operand1->is_signed;
2820
2821                 expr->operator = field_op;
2822                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2823                 if (!expr->type) {
2824                         ret = -ENOMEM;
2825                         goto free_expr;
2826                 }
2827
2828                 expr->name = expr_str(expr, 0);
2829         }
2830
2831         return expr;
2832
2833 free_operands:
2834         destroy_hist_field(operand2, 0);
2835 free_op1:
2836         destroy_hist_field(operand1, 0);
2837         return ERR_PTR(ret);
2838
2839 free_expr:
2840         destroy_hist_field(expr, 0);
2841         return ERR_PTR(ret);
2842 }
2843
2844 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2845                                  struct trace_event_file *file)
2846 {
2847         struct event_trigger_data *test;
2848
2849         lockdep_assert_held(&event_mutex);
2850
2851         list_for_each_entry(test, &file->triggers, list) {
2852                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2853                         if (test->private_data == hist_data)
2854                                 return test->filter_str;
2855                 }
2856         }
2857
2858         return NULL;
2859 }
2860
2861 static struct event_command trigger_hist_cmd;
2862 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2863                                     struct trace_event_file *file,
2864                                     char *glob, char *cmd,
2865                                     char *param_and_filter);
2866
2867 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2868                             struct hist_trigger_data *hist_data,
2869                             unsigned int n_keys)
2870 {
2871         struct hist_field *target_hist_field, *hist_field;
2872         unsigned int n, i, j;
2873
2874         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2875                 return false;
2876
2877         i = hist_data->n_vals;
2878         j = target_hist_data->n_vals;
2879
2880         for (n = 0; n < n_keys; n++) {
2881                 hist_field = hist_data->fields[i + n];
2882                 target_hist_field = target_hist_data->fields[j + n];
2883
2884                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2885                         return false;
2886                 if (hist_field->size != target_hist_field->size)
2887                         return false;
2888                 if (hist_field->is_signed != target_hist_field->is_signed)
2889                         return false;
2890         }
2891
2892         return true;
2893 }
2894
2895 static struct hist_trigger_data *
2896 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2897                      struct trace_event_file *file)
2898 {
2899         struct hist_trigger_data *hist_data;
2900         struct event_trigger_data *test;
2901         unsigned int n_keys;
2902
2903         lockdep_assert_held(&event_mutex);
2904
2905         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2906
2907         list_for_each_entry(test, &file->triggers, list) {
2908                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2909                         hist_data = test->private_data;
2910
2911                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2912                                 return hist_data;
2913                 }
2914         }
2915
2916         return NULL;
2917 }
2918
2919 static struct trace_event_file *event_file(struct trace_array *tr,
2920                                            char *system, char *event_name)
2921 {
2922         struct trace_event_file *file;
2923
2924         file = __find_event_file(tr, system, event_name);
2925         if (!file)
2926                 return ERR_PTR(-EINVAL);
2927
2928         return file;
2929 }
2930
2931 static struct hist_field *
2932 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2933                          char *system, char *event_name, char *field_name)
2934 {
2935         struct hist_field *event_var;
2936         char *synthetic_name;
2937
2938         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2939         if (!synthetic_name)
2940                 return ERR_PTR(-ENOMEM);
2941
2942         strcpy(synthetic_name, "synthetic_");
2943         strcat(synthetic_name, field_name);
2944
2945         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2946
2947         kfree(synthetic_name);
2948
2949         return event_var;
2950 }
2951
2952 /**
2953  * create_field_var_hist - Automatically create a histogram and var for a field
2954  * @target_hist_data: The target hist trigger
2955  * @subsys_name: Optional subsystem name
2956  * @event_name: Optional event name
2957  * @field_name: The name of the field (and the resulting variable)
2958  *
2959  * Hist trigger actions fetch data from variables, not directly from
2960  * events.  However, for convenience, users are allowed to directly
2961  * specify an event field in an action, which will be automatically
2962  * converted into a variable on their behalf.
2963  *
2964  * If a user specifies a field on an event that isn't the event the
2965  * histogram currently being defined (the target event histogram), the
2966  * only way that can be accomplished is if a new hist trigger is
2967  * created and the field variable defined on that.
2968  *
2969  * This function creates a new histogram compatible with the target
2970  * event (meaning a histogram with the same key as the target
2971  * histogram), and creates a variable for the specified field, but
2972  * with 'synthetic_' prepended to the variable name in order to avoid
2973  * collision with normal field variables.
2974  *
2975  * Return: The variable created for the field.
2976  */
2977 static struct hist_field *
2978 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2979                       char *subsys_name, char *event_name, char *field_name)
2980 {
2981         struct trace_array *tr = target_hist_data->event_file->tr;
2982         struct hist_trigger_data *hist_data;
2983         unsigned int i, n, first = true;
2984         struct field_var_hist *var_hist;
2985         struct trace_event_file *file;
2986         struct hist_field *key_field;
2987         struct hist_field *event_var;
2988         char *saved_filter;
2989         char *cmd;
2990         int ret;
2991
2992         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2993                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2994                 return ERR_PTR(-EINVAL);
2995         }
2996
2997         file = event_file(tr, subsys_name, event_name);
2998
2999         if (IS_ERR(file)) {
3000                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3001                 ret = PTR_ERR(file);
3002                 return ERR_PTR(ret);
3003         }
3004
3005         /*
3006          * Look for a histogram compatible with target.  We'll use the
3007          * found histogram specification to create a new matching
3008          * histogram with our variable on it.  target_hist_data is not
3009          * yet a registered histogram so we can't use that.
3010          */
3011         hist_data = find_compatible_hist(target_hist_data, file);
3012         if (!hist_data) {
3013                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3014                 return ERR_PTR(-EINVAL);
3015         }
3016
3017         /* See if a synthetic field variable has already been created */
3018         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3019                                              event_name, field_name);
3020         if (!IS_ERR_OR_NULL(event_var))
3021                 return event_var;
3022
3023         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3024         if (!var_hist)
3025                 return ERR_PTR(-ENOMEM);
3026
3027         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3028         if (!cmd) {
3029                 kfree(var_hist);
3030                 return ERR_PTR(-ENOMEM);
3031         }
3032
3033         /* Use the same keys as the compatible histogram */
3034         strcat(cmd, "keys=");
3035
3036         for_each_hist_key_field(i, hist_data) {
3037                 key_field = hist_data->fields[i];
3038                 if (!first)
3039                         strcat(cmd, ",");
3040                 strcat(cmd, key_field->field->name);
3041                 first = false;
3042         }
3043
3044         /* Create the synthetic field variable specification */
3045         strcat(cmd, ":synthetic_");
3046         strcat(cmd, field_name);
3047         strcat(cmd, "=");
3048         strcat(cmd, field_name);
3049
3050         /* Use the same filter as the compatible histogram */
3051         saved_filter = find_trigger_filter(hist_data, file);
3052         if (saved_filter) {
3053                 strcat(cmd, " if ");
3054                 strcat(cmd, saved_filter);
3055         }
3056
3057         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3058         if (!var_hist->cmd) {
3059                 kfree(cmd);
3060                 kfree(var_hist);
3061                 return ERR_PTR(-ENOMEM);
3062         }
3063
3064         /* Save the compatible histogram information */
3065         var_hist->hist_data = hist_data;
3066
3067         /* Create the new histogram with our variable */
3068         ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3069                                        "", "hist", cmd);
3070         if (ret) {
3071                 kfree(cmd);
3072                 kfree(var_hist->cmd);
3073                 kfree(var_hist);
3074                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3075                 return ERR_PTR(ret);
3076         }
3077
3078         kfree(cmd);
3079
3080         /* If we can't find the variable, something went wrong */
3081         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3082                                              event_name, field_name);
3083         if (IS_ERR_OR_NULL(event_var)) {
3084                 kfree(var_hist->cmd);
3085                 kfree(var_hist);
3086                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3087                 return ERR_PTR(-EINVAL);
3088         }
3089
3090         n = target_hist_data->n_field_var_hists;
3091         target_hist_data->field_var_hists[n] = var_hist;
3092         target_hist_data->n_field_var_hists++;
3093
3094         return event_var;
3095 }
3096
3097 static struct hist_field *
3098 find_target_event_var(struct hist_trigger_data *hist_data,
3099                       char *subsys_name, char *event_name, char *var_name)
3100 {
3101         struct trace_event_file *file = hist_data->event_file;
3102         struct hist_field *hist_field = NULL;
3103
3104         if (subsys_name) {
3105                 struct trace_event_call *call;
3106
3107                 if (!event_name)
3108                         return NULL;
3109
3110                 call = file->event_call;
3111
3112                 if (strcmp(subsys_name, call->class->system) != 0)
3113                         return NULL;
3114
3115                 if (strcmp(event_name, trace_event_name(call)) != 0)
3116                         return NULL;
3117         }
3118
3119         hist_field = find_var_field(hist_data, var_name);
3120
3121         return hist_field;
3122 }
3123
3124 static inline void __update_field_vars(struct tracing_map_elt *elt,
3125                                        struct trace_buffer *buffer,
3126                                        struct ring_buffer_event *rbe,
3127                                        void *rec,
3128                                        struct field_var **field_vars,
3129                                        unsigned int n_field_vars,
3130                                        unsigned int field_var_str_start)
3131 {
3132         struct hist_elt_data *elt_data = elt->private_data;
3133         unsigned int i, j, var_idx;
3134         u64 var_val;
3135
3136         /* Make sure stacktrace can fit in the string variable length */
3137         BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) >= STR_VAR_LEN_MAX);
3138
3139         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3140                 struct field_var *field_var = field_vars[i];
3141                 struct hist_field *var = field_var->var;
3142                 struct hist_field *val = field_var->val;
3143
3144                 var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3145                 var_idx = var->var.idx;
3146
3147                 if (val->flags & (HIST_FIELD_FL_STRING |
3148                                   HIST_FIELD_FL_STACKTRACE)) {
3149                         char *str = elt_data->field_var_str[j++];
3150                         char *val_str = (char *)(uintptr_t)var_val;
3151                         unsigned int size;
3152
3153                         if (val->flags & HIST_FIELD_FL_STRING) {
3154                                 size = min(val->size, STR_VAR_LEN_MAX);
3155                                 strscpy(str, val_str, size);
3156                         } else {
3157                                 char *stack_start = str + sizeof(unsigned long);
3158                                 int e;
3159
3160                                 e = stack_trace_save((void *)stack_start,
3161                                                      HIST_STACKTRACE_DEPTH,
3162                                                      HIST_STACKTRACE_SKIP);
3163                                 if (e < HIST_STACKTRACE_DEPTH - 1)
3164                                         ((unsigned long *)stack_start)[e] = 0;
3165                                 *((unsigned long *)str) = e;
3166                         }
3167                         var_val = (u64)(uintptr_t)str;
3168                 }
3169                 tracing_map_set_var(elt, var_idx, var_val);
3170         }
3171 }
3172
3173 static void update_field_vars(struct hist_trigger_data *hist_data,
3174                               struct tracing_map_elt *elt,
3175                               struct trace_buffer *buffer,
3176                               struct ring_buffer_event *rbe,
3177                               void *rec)
3178 {
3179         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3180                             hist_data->n_field_vars, 0);
3181 }
3182
3183 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3184                                  struct tracing_map_elt *elt,
3185                                  struct trace_buffer *buffer,  void *rec,
3186                                  struct ring_buffer_event *rbe, void *key,
3187                                  struct action_data *data, u64 *var_ref_vals)
3188 {
3189         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3190                             hist_data->n_save_vars, hist_data->n_field_var_str);
3191 }
3192
3193 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3194                                      struct trace_event_file *file,
3195                                      char *name, int size, const char *type)
3196 {
3197         struct hist_field *var;
3198         int idx;
3199
3200         if (find_var(hist_data, file, name) && !hist_data->remove) {
3201                 var = ERR_PTR(-EINVAL);
3202                 goto out;
3203         }
3204
3205         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3206         if (!var) {
3207                 var = ERR_PTR(-ENOMEM);
3208                 goto out;
3209         }
3210
3211         idx = tracing_map_add_var(hist_data->map);
3212         if (idx < 0) {
3213                 kfree(var);
3214                 var = ERR_PTR(-EINVAL);
3215                 goto out;
3216         }
3217
3218         var->ref = 1;
3219         var->flags = HIST_FIELD_FL_VAR;
3220         var->var.idx = idx;
3221         var->var.hist_data = var->hist_data = hist_data;
3222         var->size = size;
3223         var->var.name = kstrdup(name, GFP_KERNEL);
3224         var->type = kstrdup_const(type, GFP_KERNEL);
3225         if (!var->var.name || !var->type) {
3226                 kfree_const(var->type);
3227                 kfree(var->var.name);
3228                 kfree(var);
3229                 var = ERR_PTR(-ENOMEM);
3230         }
3231  out:
3232         return var;
3233 }
3234
3235 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3236                                           struct trace_event_file *file,
3237                                           char *field_name)
3238 {
3239         struct hist_field *val = NULL, *var = NULL;
3240         unsigned long flags = HIST_FIELD_FL_VAR;
3241         struct trace_array *tr = file->tr;
3242         struct field_var *field_var;
3243         int ret = 0;
3244
3245         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3246                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3247                 ret = -EINVAL;
3248                 goto err;
3249         }
3250
3251         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3252         if (IS_ERR(val)) {
3253                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3254                 ret = PTR_ERR(val);
3255                 goto err;
3256         }
3257
3258         var = create_var(hist_data, file, field_name, val->size, val->type);
3259         if (IS_ERR(var)) {
3260                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3261                 kfree(val);
3262                 ret = PTR_ERR(var);
3263                 goto err;
3264         }
3265
3266         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3267         if (!field_var) {
3268                 kfree(val);
3269                 kfree(var);
3270                 ret =  -ENOMEM;
3271                 goto err;
3272         }
3273
3274         field_var->var = var;
3275         field_var->val = val;
3276  out:
3277         return field_var;
3278  err:
3279         field_var = ERR_PTR(ret);
3280         goto out;
3281 }
3282
3283 /**
3284  * create_target_field_var - Automatically create a variable for a field
3285  * @target_hist_data: The target hist trigger
3286  * @subsys_name: Optional subsystem name
3287  * @event_name: Optional event name
3288  * @var_name: The name of the field (and the resulting variable)
3289  *
3290  * Hist trigger actions fetch data from variables, not directly from
3291  * events.  However, for convenience, users are allowed to directly
3292  * specify an event field in an action, which will be automatically
3293  * converted into a variable on their behalf.
3294  *
3295  * This function creates a field variable with the name var_name on
3296  * the hist trigger currently being defined on the target event.  If
3297  * subsys_name and event_name are specified, this function simply
3298  * verifies that they do in fact match the target event subsystem and
3299  * event name.
3300  *
3301  * Return: The variable created for the field.
3302  */
3303 static struct field_var *
3304 create_target_field_var(struct hist_trigger_data *target_hist_data,
3305                         char *subsys_name, char *event_name, char *var_name)
3306 {
3307         struct trace_event_file *file = target_hist_data->event_file;
3308
3309         if (subsys_name) {
3310                 struct trace_event_call *call;
3311
3312                 if (!event_name)
3313                         return NULL;
3314
3315                 call = file->event_call;
3316
3317                 if (strcmp(subsys_name, call->class->system) != 0)
3318                         return NULL;
3319
3320                 if (strcmp(event_name, trace_event_name(call)) != 0)
3321                         return NULL;
3322         }
3323
3324         return create_field_var(target_hist_data, file, var_name);
3325 }
3326
3327 static bool check_track_val_max(u64 track_val, u64 var_val)
3328 {
3329         if (var_val <= track_val)
3330                 return false;
3331
3332         return true;
3333 }
3334
3335 static bool check_track_val_changed(u64 track_val, u64 var_val)
3336 {
3337         if (var_val == track_val)
3338                 return false;
3339
3340         return true;
3341 }
3342
3343 static u64 get_track_val(struct hist_trigger_data *hist_data,
3344                          struct tracing_map_elt *elt,
3345                          struct action_data *data)
3346 {
3347         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3348         u64 track_val;
3349
3350         track_val = tracing_map_read_var(elt, track_var_idx);
3351
3352         return track_val;
3353 }
3354
3355 static void save_track_val(struct hist_trigger_data *hist_data,
3356                            struct tracing_map_elt *elt,
3357                            struct action_data *data, u64 var_val)
3358 {
3359         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3360
3361         tracing_map_set_var(elt, track_var_idx, var_val);
3362 }
3363
3364 static void save_track_data(struct hist_trigger_data *hist_data,
3365                             struct tracing_map_elt *elt,
3366                             struct trace_buffer *buffer, void *rec,
3367                             struct ring_buffer_event *rbe, void *key,
3368                             struct action_data *data, u64 *var_ref_vals)
3369 {
3370         if (data->track_data.save_data)
3371                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3372                                            key, data, var_ref_vals);
3373 }
3374
3375 static bool check_track_val(struct tracing_map_elt *elt,
3376                             struct action_data *data,
3377                             u64 var_val)
3378 {
3379         struct hist_trigger_data *hist_data;
3380         u64 track_val;
3381
3382         hist_data = data->track_data.track_var->hist_data;
3383         track_val = get_track_val(hist_data, elt, data);
3384
3385         return data->track_data.check_val(track_val, var_val);
3386 }
3387
3388 #ifdef CONFIG_TRACER_SNAPSHOT
3389 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3390 {
3391         /* called with tr->max_lock held */
3392         struct track_data *track_data = tr->cond_snapshot->cond_data;
3393         struct hist_elt_data *elt_data, *track_elt_data;
3394         struct snapshot_context *context = cond_data;
3395         struct action_data *action;
3396         u64 track_val;
3397
3398         if (!track_data)
3399                 return false;
3400
3401         action = track_data->action_data;
3402
3403         track_val = get_track_val(track_data->hist_data, context->elt,
3404                                   track_data->action_data);
3405
3406         if (!action->track_data.check_val(track_data->track_val, track_val))
3407                 return false;
3408
3409         track_data->track_val = track_val;
3410         memcpy(track_data->key, context->key, track_data->key_len);
3411
3412         elt_data = context->elt->private_data;
3413         track_elt_data = track_data->elt.private_data;
3414         if (elt_data->comm)
3415                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3416
3417         track_data->updated = true;
3418
3419         return true;
3420 }
3421
3422 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3423                                      struct tracing_map_elt *elt,
3424                                      struct trace_buffer *buffer, void *rec,
3425                                      struct ring_buffer_event *rbe, void *key,
3426                                      struct action_data *data,
3427                                      u64 *var_ref_vals)
3428 {
3429         struct trace_event_file *file = hist_data->event_file;
3430         struct snapshot_context context;
3431
3432         context.elt = elt;
3433         context.key = key;
3434
3435         tracing_snapshot_cond(file->tr, &context);
3436 }
3437
3438 static void hist_trigger_print_key(struct seq_file *m,
3439                                    struct hist_trigger_data *hist_data,
3440                                    void *key,
3441                                    struct tracing_map_elt *elt);
3442
3443 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3444 {
3445         unsigned int i;
3446
3447         if (!hist_data->n_actions)
3448                 return NULL;
3449
3450         for (i = 0; i < hist_data->n_actions; i++) {
3451                 struct action_data *data = hist_data->actions[i];
3452
3453                 if (data->action == ACTION_SNAPSHOT)
3454                         return data;
3455         }
3456
3457         return NULL;
3458 }
3459
3460 static void track_data_snapshot_print(struct seq_file *m,
3461                                       struct hist_trigger_data *hist_data)
3462 {
3463         struct trace_event_file *file = hist_data->event_file;
3464         struct track_data *track_data;
3465         struct action_data *action;
3466
3467         track_data = tracing_cond_snapshot_data(file->tr);
3468         if (!track_data)
3469                 return;
3470
3471         if (!track_data->updated)
3472                 return;
3473
3474         action = snapshot_action(hist_data);
3475         if (!action)
3476                 return;
3477
3478         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3479         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3480                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3481                    action->track_data.var_str, track_data->track_val);
3482
3483         seq_puts(m, "\ttriggered by event with key: ");
3484         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3485         seq_putc(m, '\n');
3486 }
3487 #else
3488 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3489 {
3490         return false;
3491 }
3492 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3493                                      struct tracing_map_elt *elt,
3494                                      struct trace_buffer *buffer, void *rec,
3495                                      struct ring_buffer_event *rbe, void *key,
3496                                      struct action_data *data,
3497                                      u64 *var_ref_vals) {}
3498 static void track_data_snapshot_print(struct seq_file *m,
3499                                       struct hist_trigger_data *hist_data) {}
3500 #endif /* CONFIG_TRACER_SNAPSHOT */
3501
3502 static void track_data_print(struct seq_file *m,
3503                              struct hist_trigger_data *hist_data,
3504                              struct tracing_map_elt *elt,
3505                              struct action_data *data)
3506 {
3507         u64 track_val = get_track_val(hist_data, elt, data);
3508         unsigned int i, save_var_idx;
3509
3510         if (data->handler == HANDLER_ONMAX)
3511                 seq_printf(m, "\n\tmax: %10llu", track_val);
3512         else if (data->handler == HANDLER_ONCHANGE)
3513                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3514
3515         if (data->action == ACTION_SNAPSHOT)
3516                 return;
3517
3518         for (i = 0; i < hist_data->n_save_vars; i++) {
3519                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3520                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3521                 u64 val;
3522
3523                 save_var_idx = save_var->var.idx;
3524
3525                 val = tracing_map_read_var(elt, save_var_idx);
3526
3527                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3528                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3529                                    (char *)(uintptr_t)(val));
3530                 } else
3531                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3532         }
3533 }
3534
3535 static void ontrack_action(struct hist_trigger_data *hist_data,
3536                            struct tracing_map_elt *elt,
3537                            struct trace_buffer *buffer, void *rec,
3538                            struct ring_buffer_event *rbe, void *key,
3539                            struct action_data *data, u64 *var_ref_vals)
3540 {
3541         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3542
3543         if (check_track_val(elt, data, var_val)) {
3544                 save_track_val(hist_data, elt, data, var_val);
3545                 save_track_data(hist_data, elt, buffer, rec, rbe,
3546                                 key, data, var_ref_vals);
3547         }
3548 }
3549
3550 static void action_data_destroy(struct action_data *data)
3551 {
3552         unsigned int i;
3553
3554         lockdep_assert_held(&event_mutex);
3555
3556         kfree(data->action_name);
3557
3558         for (i = 0; i < data->n_params; i++)
3559                 kfree(data->params[i]);
3560
3561         if (data->synth_event)
3562                 data->synth_event->ref--;
3563
3564         kfree(data->synth_event_name);
3565
3566         kfree(data);
3567 }
3568
3569 static void track_data_destroy(struct hist_trigger_data *hist_data,
3570                                struct action_data *data)
3571 {
3572         struct trace_event_file *file = hist_data->event_file;
3573
3574         destroy_hist_field(data->track_data.track_var, 0);
3575
3576         if (data->action == ACTION_SNAPSHOT) {
3577                 struct track_data *track_data;
3578
3579                 track_data = tracing_cond_snapshot_data(file->tr);
3580                 if (track_data && track_data->hist_data == hist_data) {
3581                         tracing_snapshot_cond_disable(file->tr);
3582                         track_data_free(track_data);
3583                 }
3584         }
3585
3586         kfree(data->track_data.var_str);
3587
3588         action_data_destroy(data);
3589 }
3590
3591 static int action_create(struct hist_trigger_data *hist_data,
3592                          struct action_data *data);
3593
3594 static int track_data_create(struct hist_trigger_data *hist_data,
3595                              struct action_data *data)
3596 {
3597         struct hist_field *var_field, *ref_field, *track_var = NULL;
3598         struct trace_event_file *file = hist_data->event_file;
3599         struct trace_array *tr = file->tr;
3600         char *track_data_var_str;
3601         int ret = 0;
3602
3603         track_data_var_str = data->track_data.var_str;
3604         if (track_data_var_str[0] != '$') {
3605                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3606                 return -EINVAL;
3607         }
3608         track_data_var_str++;
3609
3610         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3611         if (!var_field) {
3612                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3613                 return -EINVAL;
3614         }
3615
3616         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3617         if (!ref_field)
3618                 return -ENOMEM;
3619
3620         data->track_data.var_ref = ref_field;
3621
3622         if (data->handler == HANDLER_ONMAX)
3623                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3624         if (IS_ERR(track_var)) {
3625                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3626                 ret = PTR_ERR(track_var);
3627                 goto out;
3628         }
3629
3630         if (data->handler == HANDLER_ONCHANGE)
3631                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3632         if (IS_ERR(track_var)) {
3633                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3634                 ret = PTR_ERR(track_var);
3635                 goto out;
3636         }
3637         data->track_data.track_var = track_var;
3638
3639         ret = action_create(hist_data, data);
3640  out:
3641         return ret;
3642 }
3643
3644 static int parse_action_params(struct trace_array *tr, char *params,
3645                                struct action_data *data)
3646 {
3647         char *param, *saved_param;
3648         bool first_param = true;
3649         int ret = 0;
3650
3651         while (params) {
3652                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3653                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3654                         ret = -EINVAL;
3655                         goto out;
3656                 }
3657
3658                 param = strsep(&params, ",");
3659                 if (!param) {
3660                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3661                         ret = -EINVAL;
3662                         goto out;
3663                 }
3664
3665                 param = strstrip(param);
3666                 if (strlen(param) < 2) {
3667                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3668                         ret = -EINVAL;
3669                         goto out;
3670                 }
3671
3672                 saved_param = kstrdup(param, GFP_KERNEL);
3673                 if (!saved_param) {
3674                         ret = -ENOMEM;
3675                         goto out;
3676                 }
3677
3678                 if (first_param && data->use_trace_keyword) {
3679                         data->synth_event_name = saved_param;
3680                         first_param = false;
3681                         continue;
3682                 }
3683                 first_param = false;
3684
3685                 data->params[data->n_params++] = saved_param;
3686         }
3687  out:
3688         return ret;
3689 }
3690
3691 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3692                         enum handler_id handler)
3693 {
3694         char *action_name;
3695         int ret = 0;
3696
3697         strsep(&str, ".");
3698         if (!str) {
3699                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3700                 ret = -EINVAL;
3701                 goto out;
3702         }
3703
3704         action_name = strsep(&str, "(");
3705         if (!action_name || !str) {
3706                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3707                 ret = -EINVAL;
3708                 goto out;
3709         }
3710
3711         if (str_has_prefix(action_name, "save")) {
3712                 char *params = strsep(&str, ")");
3713
3714                 if (!params) {
3715                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3716                         ret = -EINVAL;
3717                         goto out;
3718                 }
3719
3720                 ret = parse_action_params(tr, params, data);
3721                 if (ret)
3722                         goto out;
3723
3724                 if (handler == HANDLER_ONMAX)
3725                         data->track_data.check_val = check_track_val_max;
3726                 else if (handler == HANDLER_ONCHANGE)
3727                         data->track_data.check_val = check_track_val_changed;
3728                 else {
3729                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3730                         ret = -EINVAL;
3731                         goto out;
3732                 }
3733
3734                 data->track_data.save_data = save_track_data_vars;
3735                 data->fn = ontrack_action;
3736                 data->action = ACTION_SAVE;
3737         } else if (str_has_prefix(action_name, "snapshot")) {
3738                 char *params = strsep(&str, ")");
3739
3740                 if (!str) {
3741                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3742                         ret = -EINVAL;
3743                         goto out;
3744                 }
3745
3746                 if (handler == HANDLER_ONMAX)
3747                         data->track_data.check_val = check_track_val_max;
3748                 else if (handler == HANDLER_ONCHANGE)
3749                         data->track_data.check_val = check_track_val_changed;
3750                 else {
3751                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3752                         ret = -EINVAL;
3753                         goto out;
3754                 }
3755
3756                 data->track_data.save_data = save_track_data_snapshot;
3757                 data->fn = ontrack_action;
3758                 data->action = ACTION_SNAPSHOT;
3759         } else {
3760                 char *params = strsep(&str, ")");
3761
3762                 if (str_has_prefix(action_name, "trace"))
3763                         data->use_trace_keyword = true;
3764
3765                 if (params) {
3766                         ret = parse_action_params(tr, params, data);
3767                         if (ret)
3768                                 goto out;
3769                 }
3770
3771                 if (handler == HANDLER_ONMAX)
3772                         data->track_data.check_val = check_track_val_max;
3773                 else if (handler == HANDLER_ONCHANGE)
3774                         data->track_data.check_val = check_track_val_changed;
3775
3776                 if (handler != HANDLER_ONMATCH) {
3777                         data->track_data.save_data = action_trace;
3778                         data->fn = ontrack_action;
3779                 } else
3780                         data->fn = action_trace;
3781
3782                 data->action = ACTION_TRACE;
3783         }
3784
3785         data->action_name = kstrdup(action_name, GFP_KERNEL);
3786         if (!data->action_name) {
3787                 ret = -ENOMEM;
3788                 goto out;
3789         }
3790
3791         data->handler = handler;
3792  out:
3793         return ret;
3794 }
3795
3796 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3797                                             char *str, enum handler_id handler)
3798 {
3799         struct action_data *data;
3800         int ret = -EINVAL;
3801         char *var_str;
3802
3803         data = kzalloc(sizeof(*data), GFP_KERNEL);
3804         if (!data)
3805                 return ERR_PTR(-ENOMEM);
3806
3807         var_str = strsep(&str, ")");
3808         if (!var_str || !str) {
3809                 ret = -EINVAL;
3810                 goto free;
3811         }
3812
3813         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3814         if (!data->track_data.var_str) {
3815                 ret = -ENOMEM;
3816                 goto free;
3817         }
3818
3819         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3820         if (ret)
3821                 goto free;
3822  out:
3823         return data;
3824  free:
3825         track_data_destroy(hist_data, data);
3826         data = ERR_PTR(ret);
3827         goto out;
3828 }
3829
3830 static void onmatch_destroy(struct action_data *data)
3831 {
3832         kfree(data->match_data.event);
3833         kfree(data->match_data.event_system);
3834
3835         action_data_destroy(data);
3836 }
3837
3838 static void destroy_field_var(struct field_var *field_var)
3839 {
3840         if (!field_var)
3841                 return;
3842
3843         destroy_hist_field(field_var->var, 0);
3844         destroy_hist_field(field_var->val, 0);
3845
3846         kfree(field_var);
3847 }
3848
3849 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3850 {
3851         unsigned int i;
3852
3853         for (i = 0; i < hist_data->n_field_vars; i++)
3854                 destroy_field_var(hist_data->field_vars[i]);
3855
3856         for (i = 0; i < hist_data->n_save_vars; i++)
3857                 destroy_field_var(hist_data->save_vars[i]);
3858 }
3859
3860 static void save_field_var(struct hist_trigger_data *hist_data,
3861                            struct field_var *field_var)
3862 {
3863         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3864
3865         /* Stack traces are saved in the string storage too */
3866         if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
3867                 hist_data->n_field_var_str++;
3868 }
3869
3870
3871 static int check_synth_field(struct synth_event *event,
3872                              struct hist_field *hist_field,
3873                              unsigned int field_pos)
3874 {
3875         struct synth_field *field;
3876
3877         if (field_pos >= event->n_fields)
3878                 return -EINVAL;
3879
3880         field = event->fields[field_pos];
3881
3882         /*
3883          * A dynamic string synth field can accept static or
3884          * dynamic. A static string synth field can only accept a
3885          * same-sized static string, which is checked for later.
3886          */
3887         if (strstr(hist_field->type, "char[") && field->is_string
3888             && field->is_dynamic)
3889                 return 0;
3890
3891         if (strstr(hist_field->type, "long[") && field->is_stack)
3892                 return 0;
3893
3894         if (strcmp(field->type, hist_field->type) != 0) {
3895                 if (field->size != hist_field->size ||
3896                     (!field->is_string && field->is_signed != hist_field->is_signed))
3897                         return -EINVAL;
3898         }
3899
3900         return 0;
3901 }
3902
3903 static struct hist_field *
3904 trace_action_find_var(struct hist_trigger_data *hist_data,
3905                       struct action_data *data,
3906                       char *system, char *event, char *var)
3907 {
3908         struct trace_array *tr = hist_data->event_file->tr;
3909         struct hist_field *hist_field;
3910
3911         var++; /* skip '$' */
3912
3913         hist_field = find_target_event_var(hist_data, system, event, var);
3914         if (!hist_field) {
3915                 if (!system && data->handler == HANDLER_ONMATCH) {
3916                         system = data->match_data.event_system;
3917                         event = data->match_data.event;
3918                 }
3919
3920                 hist_field = find_event_var(hist_data, system, event, var);
3921         }
3922
3923         if (!hist_field)
3924                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3925
3926         return hist_field;
3927 }
3928
3929 static struct hist_field *
3930 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3931                               struct action_data *data, char *system,
3932                               char *event, char *var)
3933 {
3934         struct hist_field *hist_field = NULL;
3935         struct field_var *field_var;
3936
3937         /*
3938          * First try to create a field var on the target event (the
3939          * currently being defined).  This will create a variable for
3940          * unqualified fields on the target event, or if qualified,
3941          * target fields that have qualified names matching the target.
3942          */
3943         field_var = create_target_field_var(hist_data, system, event, var);
3944
3945         if (field_var && !IS_ERR(field_var)) {
3946                 save_field_var(hist_data, field_var);
3947                 hist_field = field_var->var;
3948         } else {
3949                 field_var = NULL;
3950                 /*
3951                  * If no explicit system.event is specified, default to
3952                  * looking for fields on the onmatch(system.event.xxx)
3953                  * event.
3954                  */
3955                 if (!system && data->handler == HANDLER_ONMATCH) {
3956                         system = data->match_data.event_system;
3957                         event = data->match_data.event;
3958                 }
3959
3960                 if (!event)
3961                         goto free;
3962                 /*
3963                  * At this point, we're looking at a field on another
3964                  * event.  Because we can't modify a hist trigger on
3965                  * another event to add a variable for a field, we need
3966                  * to create a new trigger on that event and create the
3967                  * variable at the same time.
3968                  */
3969                 hist_field = create_field_var_hist(hist_data, system, event, var);
3970                 if (IS_ERR(hist_field))
3971                         goto free;
3972         }
3973  out:
3974         return hist_field;
3975  free:
3976         destroy_field_var(field_var);
3977         hist_field = NULL;
3978         goto out;
3979 }
3980
3981 static int trace_action_create(struct hist_trigger_data *hist_data,
3982                                struct action_data *data)
3983 {
3984         struct trace_array *tr = hist_data->event_file->tr;
3985         char *event_name, *param, *system = NULL;
3986         struct hist_field *hist_field, *var_ref;
3987         unsigned int i;
3988         unsigned int field_pos = 0;
3989         struct synth_event *event;
3990         char *synth_event_name;
3991         int var_ref_idx, ret = 0;
3992
3993         lockdep_assert_held(&event_mutex);
3994
3995         /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
3996         if (data->n_params > SYNTH_FIELDS_MAX)
3997                 return -EINVAL;
3998
3999         if (data->use_trace_keyword)
4000                 synth_event_name = data->synth_event_name;
4001         else
4002                 synth_event_name = data->action_name;
4003
4004         event = find_synth_event(synth_event_name);
4005         if (!event) {
4006                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4007                 return -EINVAL;
4008         }
4009
4010         event->ref++;
4011
4012         for (i = 0; i < data->n_params; i++) {
4013                 char *p;
4014
4015                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4016                 if (!param) {
4017                         ret = -ENOMEM;
4018                         goto err;
4019                 }
4020
4021                 system = strsep(&param, ".");
4022                 if (!param) {
4023                         param = (char *)system;
4024                         system = event_name = NULL;
4025                 } else {
4026                         event_name = strsep(&param, ".");
4027                         if (!param) {
4028                                 kfree(p);
4029                                 ret = -EINVAL;
4030                                 goto err;
4031                         }
4032                 }
4033
4034                 if (param[0] == '$')
4035                         hist_field = trace_action_find_var(hist_data, data,
4036                                                            system, event_name,
4037                                                            param);
4038                 else
4039                         hist_field = trace_action_create_field_var(hist_data,
4040                                                                    data,
4041                                                                    system,
4042                                                                    event_name,
4043                                                                    param);
4044
4045                 if (!hist_field) {
4046                         kfree(p);
4047                         ret = -EINVAL;
4048                         goto err;
4049                 }
4050
4051                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4052                         var_ref = create_var_ref(hist_data, hist_field,
4053                                                  system, event_name);
4054                         if (!var_ref) {
4055                                 kfree(p);
4056                                 ret = -ENOMEM;
4057                                 goto err;
4058                         }
4059
4060                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4061                         if (WARN_ON(var_ref_idx < 0)) {
4062                                 kfree(p);
4063                                 ret = var_ref_idx;
4064                                 goto err;
4065                         }
4066
4067                         data->var_ref_idx[i] = var_ref_idx;
4068
4069                         field_pos++;
4070                         kfree(p);
4071                         continue;
4072                 }
4073
4074                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4075                 kfree(p);
4076                 ret = -EINVAL;
4077                 goto err;
4078         }
4079
4080         if (field_pos != event->n_fields) {
4081                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4082                 ret = -EINVAL;
4083                 goto err;
4084         }
4085
4086         data->synth_event = event;
4087  out:
4088         return ret;
4089  err:
4090         event->ref--;
4091
4092         goto out;
4093 }
4094
4095 static int action_create(struct hist_trigger_data *hist_data,
4096                          struct action_data *data)
4097 {
4098         struct trace_event_file *file = hist_data->event_file;
4099         struct trace_array *tr = file->tr;
4100         struct track_data *track_data;
4101         struct field_var *field_var;
4102         unsigned int i;
4103         char *param;
4104         int ret = 0;
4105
4106         if (data->action == ACTION_TRACE)
4107                 return trace_action_create(hist_data, data);
4108
4109         if (data->action == ACTION_SNAPSHOT) {
4110                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4111                 if (IS_ERR(track_data)) {
4112                         ret = PTR_ERR(track_data);
4113                         goto out;
4114                 }
4115
4116                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4117                                                    cond_snapshot_update);
4118                 if (ret)
4119                         track_data_free(track_data);
4120
4121                 goto out;
4122         }
4123
4124         if (data->action == ACTION_SAVE) {
4125                 if (hist_data->n_save_vars) {
4126                         ret = -EEXIST;
4127                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4128                         goto out;
4129                 }
4130
4131                 for (i = 0; i < data->n_params; i++) {
4132                         param = kstrdup(data->params[i], GFP_KERNEL);
4133                         if (!param) {
4134                                 ret = -ENOMEM;
4135                                 goto out;
4136                         }
4137
4138                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4139                         if (IS_ERR(field_var)) {
4140                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4141                                          errpos(param));
4142                                 ret = PTR_ERR(field_var);
4143                                 kfree(param);
4144                                 goto out;
4145                         }
4146
4147                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4148                         if (field_var->val->flags &
4149                             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4150                                 hist_data->n_save_var_str++;
4151                         kfree(param);
4152                 }
4153         }
4154  out:
4155         return ret;
4156 }
4157
4158 static int onmatch_create(struct hist_trigger_data *hist_data,
4159                           struct action_data *data)
4160 {
4161         return action_create(hist_data, data);
4162 }
4163
4164 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4165 {
4166         char *match_event, *match_event_system;
4167         struct action_data *data;
4168         int ret = -EINVAL;
4169
4170         data = kzalloc(sizeof(*data), GFP_KERNEL);
4171         if (!data)
4172                 return ERR_PTR(-ENOMEM);
4173
4174         match_event = strsep(&str, ")");
4175         if (!match_event || !str) {
4176                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4177                 goto free;
4178         }
4179
4180         match_event_system = strsep(&match_event, ".");
4181         if (!match_event) {
4182                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4183                 goto free;
4184         }
4185
4186         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4187                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4188                 goto free;
4189         }
4190
4191         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4192         if (!data->match_data.event) {
4193                 ret = -ENOMEM;
4194                 goto free;
4195         }
4196
4197         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4198         if (!data->match_data.event_system) {
4199                 ret = -ENOMEM;
4200                 goto free;
4201         }
4202
4203         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4204         if (ret)
4205                 goto free;
4206  out:
4207         return data;
4208  free:
4209         onmatch_destroy(data);
4210         data = ERR_PTR(ret);
4211         goto out;
4212 }
4213
4214 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4215 {
4216         hist_data->fields[HITCOUNT_IDX] =
4217                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4218         if (!hist_data->fields[HITCOUNT_IDX])
4219                 return -ENOMEM;
4220
4221         hist_data->n_vals++;
4222         hist_data->n_fields++;
4223
4224         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4225                 return -EINVAL;
4226
4227         return 0;
4228 }
4229
4230 static int __create_val_field(struct hist_trigger_data *hist_data,
4231                               unsigned int val_idx,
4232                               struct trace_event_file *file,
4233                               char *var_name, char *field_str,
4234                               unsigned long flags)
4235 {
4236         struct hist_field *hist_field;
4237         int ret = 0, n_subexprs = 0;
4238
4239         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4240         if (IS_ERR(hist_field)) {
4241                 ret = PTR_ERR(hist_field);
4242                 goto out;
4243         }
4244
4245         /* values and variables should not have some modifiers */
4246         if (hist_field->flags & HIST_FIELD_FL_VAR) {
4247                 /* Variable */
4248                 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4249                                          HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2))
4250                         goto err;
4251         } else {
4252                 /* Value */
4253                 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4254                                          HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 |
4255                                          HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET |
4256                                          HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE))
4257                         goto err;
4258         }
4259
4260         hist_data->fields[val_idx] = hist_field;
4261
4262         ++hist_data->n_vals;
4263         ++hist_data->n_fields;
4264
4265         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4266                 ret = -EINVAL;
4267  out:
4268         return ret;
4269  err:
4270         hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str));
4271         return -EINVAL;
4272 }
4273
4274 static int create_val_field(struct hist_trigger_data *hist_data,
4275                             unsigned int val_idx,
4276                             struct trace_event_file *file,
4277                             char *field_str)
4278 {
4279         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4280                 return -EINVAL;
4281
4282         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4283 }
4284
4285 static const char no_comm[] = "(no comm)";
4286
4287 static u64 hist_field_execname(struct hist_field *hist_field,
4288                                struct tracing_map_elt *elt,
4289                                struct trace_buffer *buffer,
4290                                struct ring_buffer_event *rbe,
4291                                void *event)
4292 {
4293         struct hist_elt_data *elt_data;
4294
4295         if (WARN_ON_ONCE(!elt))
4296                 return (u64)(unsigned long)no_comm;
4297
4298         elt_data = elt->private_data;
4299
4300         if (WARN_ON_ONCE(!elt_data->comm))
4301                 return (u64)(unsigned long)no_comm;
4302
4303         return (u64)(unsigned long)(elt_data->comm);
4304 }
4305
4306 static u64 hist_field_stack(struct hist_field *hist_field,
4307                             struct tracing_map_elt *elt,
4308                             struct trace_buffer *buffer,
4309                             struct ring_buffer_event *rbe,
4310                             void *event)
4311 {
4312         u32 str_item = *(u32 *)(event + hist_field->field->offset);
4313         int str_loc = str_item & 0xffff;
4314         char *addr = (char *)(event + str_loc);
4315
4316         return (u64)(unsigned long)addr;
4317 }
4318
4319 static u64 hist_fn_call(struct hist_field *hist_field,
4320                         struct tracing_map_elt *elt,
4321                         struct trace_buffer *buffer,
4322                         struct ring_buffer_event *rbe,
4323                         void *event)
4324 {
4325         switch (hist_field->fn_num) {
4326         case HIST_FIELD_FN_VAR_REF:
4327                 return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4328         case HIST_FIELD_FN_COUNTER:
4329                 return hist_field_counter(hist_field, elt, buffer, rbe, event);
4330         case HIST_FIELD_FN_CONST:
4331                 return hist_field_const(hist_field, elt, buffer, rbe, event);
4332         case HIST_FIELD_FN_LOG2:
4333                 return hist_field_log2(hist_field, elt, buffer, rbe, event);
4334         case HIST_FIELD_FN_BUCKET:
4335                 return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4336         case HIST_FIELD_FN_TIMESTAMP:
4337                 return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4338         case HIST_FIELD_FN_CPU:
4339                 return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4340         case HIST_FIELD_FN_STRING:
4341                 return hist_field_string(hist_field, elt, buffer, rbe, event);
4342         case HIST_FIELD_FN_DYNSTRING:
4343                 return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4344         case HIST_FIELD_FN_RELDYNSTRING:
4345                 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4346         case HIST_FIELD_FN_PSTRING:
4347                 return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4348         case HIST_FIELD_FN_S64:
4349                 return hist_field_s64(hist_field, elt, buffer, rbe, event);
4350         case HIST_FIELD_FN_U64:
4351                 return hist_field_u64(hist_field, elt, buffer, rbe, event);
4352         case HIST_FIELD_FN_S32:
4353                 return hist_field_s32(hist_field, elt, buffer, rbe, event);
4354         case HIST_FIELD_FN_U32:
4355                 return hist_field_u32(hist_field, elt, buffer, rbe, event);
4356         case HIST_FIELD_FN_S16:
4357                 return hist_field_s16(hist_field, elt, buffer, rbe, event);
4358         case HIST_FIELD_FN_U16:
4359                 return hist_field_u16(hist_field, elt, buffer, rbe, event);
4360         case HIST_FIELD_FN_S8:
4361                 return hist_field_s8(hist_field, elt, buffer, rbe, event);
4362         case HIST_FIELD_FN_U8:
4363                 return hist_field_u8(hist_field, elt, buffer, rbe, event);
4364         case HIST_FIELD_FN_UMINUS:
4365                 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4366         case HIST_FIELD_FN_MINUS:
4367                 return hist_field_minus(hist_field, elt, buffer, rbe, event);
4368         case HIST_FIELD_FN_PLUS:
4369                 return hist_field_plus(hist_field, elt, buffer, rbe, event);
4370         case HIST_FIELD_FN_DIV:
4371                 return hist_field_div(hist_field, elt, buffer, rbe, event);
4372         case HIST_FIELD_FN_MULT:
4373                 return hist_field_mult(hist_field, elt, buffer, rbe, event);
4374         case HIST_FIELD_FN_DIV_POWER2:
4375                 return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4376         case HIST_FIELD_FN_DIV_NOT_POWER2:
4377                 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4378         case HIST_FIELD_FN_DIV_MULT_SHIFT:
4379                 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4380         case HIST_FIELD_FN_EXECNAME:
4381                 return hist_field_execname(hist_field, elt, buffer, rbe, event);
4382         case HIST_FIELD_FN_STACK:
4383                 return hist_field_stack(hist_field, elt, buffer, rbe, event);
4384         default:
4385                 return 0;
4386         }
4387 }
4388
4389 /* Convert a var that points to common_pid.execname to a string */
4390 static void update_var_execname(struct hist_field *hist_field)
4391 {
4392         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4393                 HIST_FIELD_FL_EXECNAME;
4394         hist_field->size = MAX_FILTER_STR_VAL;
4395         hist_field->is_signed = 0;
4396
4397         kfree_const(hist_field->type);
4398         hist_field->type = "char[]";
4399
4400         hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4401 }
4402
4403 static int create_var_field(struct hist_trigger_data *hist_data,
4404                             unsigned int val_idx,
4405                             struct trace_event_file *file,
4406                             char *var_name, char *expr_str)
4407 {
4408         struct trace_array *tr = hist_data->event_file->tr;
4409         unsigned long flags = 0;
4410         int ret;
4411
4412         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4413                 return -EINVAL;
4414
4415         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4416                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4417                 return -EINVAL;
4418         }
4419
4420         flags |= HIST_FIELD_FL_VAR;
4421         hist_data->n_vars++;
4422         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4423                 return -EINVAL;
4424
4425         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4426
4427         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4428                 update_var_execname(hist_data->fields[val_idx]);
4429
4430         if (!ret && hist_data->fields[val_idx]->flags &
4431             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4432                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4433
4434         return ret;
4435 }
4436
4437 static int create_val_fields(struct hist_trigger_data *hist_data,
4438                              struct trace_event_file *file)
4439 {
4440         unsigned int i, j = 1, n_hitcount = 0;
4441         char *fields_str, *field_str;
4442         int ret;
4443
4444         ret = create_hitcount_val(hist_data);
4445         if (ret)
4446                 goto out;
4447
4448         fields_str = hist_data->attrs->vals_str;
4449         if (!fields_str)
4450                 goto out;
4451
4452         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4453                      j < TRACING_MAP_VALS_MAX; i++) {
4454                 field_str = strsep(&fields_str, ",");
4455                 if (!field_str)
4456                         break;
4457
4458                 if (strcmp(field_str, "hitcount") == 0) {
4459                         if (!n_hitcount++)
4460                                 continue;
4461                 }
4462
4463                 ret = create_val_field(hist_data, j++, file, field_str);
4464                 if (ret)
4465                         goto out;
4466         }
4467
4468         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4469                 ret = -EINVAL;
4470  out:
4471         /* There is only raw hitcount but nohitcount suppresses it. */
4472         if (j == 1 && hist_data->attrs->no_hitcount) {
4473                 hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0);
4474                 ret = -ENOENT;
4475         }
4476
4477         return ret;
4478 }
4479
4480 static int create_key_field(struct hist_trigger_data *hist_data,
4481                             unsigned int key_idx,
4482                             unsigned int key_offset,
4483                             struct trace_event_file *file,
4484                             char *field_str)
4485 {
4486         struct trace_array *tr = hist_data->event_file->tr;
4487         struct hist_field *hist_field = NULL;
4488         unsigned long flags = 0;
4489         unsigned int key_size;
4490         int ret = 0, n_subexprs = 0;
4491
4492         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4493                 return -EINVAL;
4494
4495         flags |= HIST_FIELD_FL_KEY;
4496
4497         if (strcmp(field_str, "stacktrace") == 0) {
4498                 flags |= HIST_FIELD_FL_STACKTRACE;
4499                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4500                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4501         } else {
4502                 hist_field = parse_expr(hist_data, file, field_str, flags,
4503                                         NULL, &n_subexprs);
4504                 if (IS_ERR(hist_field)) {
4505                         ret = PTR_ERR(hist_field);
4506                         goto out;
4507                 }
4508
4509                 if (field_has_hist_vars(hist_field, 0)) {
4510                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4511                         destroy_hist_field(hist_field, 0);
4512                         ret = -EINVAL;
4513                         goto out;
4514                 }
4515
4516                 key_size = hist_field->size;
4517         }
4518
4519         hist_data->fields[key_idx] = hist_field;
4520
4521         key_size = ALIGN(key_size, sizeof(u64));
4522         hist_data->fields[key_idx]->size = key_size;
4523         hist_data->fields[key_idx]->offset = key_offset;
4524
4525         hist_data->key_size += key_size;
4526
4527         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4528                 ret = -EINVAL;
4529                 goto out;
4530         }
4531
4532         hist_data->n_keys++;
4533         hist_data->n_fields++;
4534
4535         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4536                 return -EINVAL;
4537
4538         ret = key_size;
4539  out:
4540         return ret;
4541 }
4542
4543 static int create_key_fields(struct hist_trigger_data *hist_data,
4544                              struct trace_event_file *file)
4545 {
4546         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4547         char *fields_str, *field_str;
4548         int ret = -EINVAL;
4549
4550         fields_str = hist_data->attrs->keys_str;
4551         if (!fields_str)
4552                 goto out;
4553
4554         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4555                 field_str = strsep(&fields_str, ",");
4556                 if (!field_str)
4557                         break;
4558                 ret = create_key_field(hist_data, i, key_offset,
4559                                        file, field_str);
4560                 if (ret < 0)
4561                         goto out;
4562                 key_offset += ret;
4563         }
4564         if (fields_str) {
4565                 ret = -EINVAL;
4566                 goto out;
4567         }
4568         ret = 0;
4569  out:
4570         return ret;
4571 }
4572
4573 static int create_var_fields(struct hist_trigger_data *hist_data,
4574                              struct trace_event_file *file)
4575 {
4576         unsigned int i, j = hist_data->n_vals;
4577         int ret = 0;
4578
4579         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4580
4581         for (i = 0; i < n_vars; i++) {
4582                 char *var_name = hist_data->attrs->var_defs.name[i];
4583                 char *expr = hist_data->attrs->var_defs.expr[i];
4584
4585                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4586                 if (ret)
4587                         goto out;
4588         }
4589  out:
4590         return ret;
4591 }
4592
4593 static void free_var_defs(struct hist_trigger_data *hist_data)
4594 {
4595         unsigned int i;
4596
4597         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4598                 kfree(hist_data->attrs->var_defs.name[i]);
4599                 kfree(hist_data->attrs->var_defs.expr[i]);
4600         }
4601
4602         hist_data->attrs->var_defs.n_vars = 0;
4603 }
4604
4605 static int parse_var_defs(struct hist_trigger_data *hist_data)
4606 {
4607         struct trace_array *tr = hist_data->event_file->tr;
4608         char *s, *str, *var_name, *field_str;
4609         unsigned int i, j, n_vars = 0;
4610         int ret = 0;
4611
4612         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4613                 str = hist_data->attrs->assignment_str[i];
4614                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4615                         field_str = strsep(&str, ",");
4616                         if (!field_str)
4617                                 break;
4618
4619                         var_name = strsep(&field_str, "=");
4620                         if (!var_name || !field_str) {
4621                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4622                                          errpos(var_name));
4623                                 ret = -EINVAL;
4624                                 goto free;
4625                         }
4626
4627                         if (n_vars == TRACING_MAP_VARS_MAX) {
4628                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4629                                 ret = -EINVAL;
4630                                 goto free;
4631                         }
4632
4633                         s = kstrdup(var_name, GFP_KERNEL);
4634                         if (!s) {
4635                                 ret = -ENOMEM;
4636                                 goto free;
4637                         }
4638                         hist_data->attrs->var_defs.name[n_vars] = s;
4639
4640                         s = kstrdup(field_str, GFP_KERNEL);
4641                         if (!s) {
4642                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4643                                 hist_data->attrs->var_defs.name[n_vars] = NULL;
4644                                 ret = -ENOMEM;
4645                                 goto free;
4646                         }
4647                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4648
4649                         hist_data->attrs->var_defs.n_vars = n_vars;
4650                 }
4651         }
4652
4653         return ret;
4654  free:
4655         free_var_defs(hist_data);
4656
4657         return ret;
4658 }
4659
4660 static int create_hist_fields(struct hist_trigger_data *hist_data,
4661                               struct trace_event_file *file)
4662 {
4663         int ret;
4664
4665         ret = parse_var_defs(hist_data);
4666         if (ret)
4667                 return ret;
4668
4669         ret = create_val_fields(hist_data, file);
4670         if (ret)
4671                 goto out;
4672
4673         ret = create_var_fields(hist_data, file);
4674         if (ret)
4675                 goto out;
4676
4677         ret = create_key_fields(hist_data, file);
4678
4679  out:
4680         free_var_defs(hist_data);
4681
4682         return ret;
4683 }
4684
4685 static int is_descending(struct trace_array *tr, const char *str)
4686 {
4687         if (!str)
4688                 return 0;
4689
4690         if (strcmp(str, "descending") == 0)
4691                 return 1;
4692
4693         if (strcmp(str, "ascending") == 0)
4694                 return 0;
4695
4696         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4697
4698         return -EINVAL;
4699 }
4700
4701 static int create_sort_keys(struct hist_trigger_data *hist_data)
4702 {
4703         struct trace_array *tr = hist_data->event_file->tr;
4704         char *fields_str = hist_data->attrs->sort_key_str;
4705         struct tracing_map_sort_key *sort_key;
4706         int descending, ret = 0;
4707         unsigned int i, j, k;
4708
4709         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4710
4711         if (!fields_str)
4712                 goto out;
4713
4714         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4715                 struct hist_field *hist_field;
4716                 char *field_str, *field_name;
4717                 const char *test_name;
4718
4719                 sort_key = &hist_data->sort_keys[i];
4720
4721                 field_str = strsep(&fields_str, ",");
4722                 if (!field_str)
4723                         break;
4724
4725                 if (!*field_str) {
4726                         ret = -EINVAL;
4727                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4728                         break;
4729                 }
4730
4731                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4732                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4733                         ret = -EINVAL;
4734                         break;
4735                 }
4736
4737                 field_name = strsep(&field_str, ".");
4738                 if (!field_name || !*field_name) {
4739                         ret = -EINVAL;
4740                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4741                         break;
4742                 }
4743
4744                 if (strcmp(field_name, "hitcount") == 0) {
4745                         descending = is_descending(tr, field_str);
4746                         if (descending < 0) {
4747                                 ret = descending;
4748                                 break;
4749                         }
4750                         sort_key->descending = descending;
4751                         continue;
4752                 }
4753
4754                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4755                         unsigned int idx;
4756
4757                         hist_field = hist_data->fields[j];
4758                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4759                                 continue;
4760
4761                         idx = k++;
4762
4763                         test_name = hist_field_name(hist_field, 0);
4764
4765                         if (strcmp(field_name, test_name) == 0) {
4766                                 sort_key->field_idx = idx;
4767                                 descending = is_descending(tr, field_str);
4768                                 if (descending < 0) {
4769                                         ret = descending;
4770                                         goto out;
4771                                 }
4772                                 sort_key->descending = descending;
4773                                 break;
4774                         }
4775                 }
4776                 if (j == hist_data->n_fields) {
4777                         ret = -EINVAL;
4778                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4779                         break;
4780                 }
4781         }
4782
4783         hist_data->n_sort_keys = i;
4784  out:
4785         return ret;
4786 }
4787
4788 static void destroy_actions(struct hist_trigger_data *hist_data)
4789 {
4790         unsigned int i;
4791
4792         for (i = 0; i < hist_data->n_actions; i++) {
4793                 struct action_data *data = hist_data->actions[i];
4794
4795                 if (data->handler == HANDLER_ONMATCH)
4796                         onmatch_destroy(data);
4797                 else if (data->handler == HANDLER_ONMAX ||
4798                          data->handler == HANDLER_ONCHANGE)
4799                         track_data_destroy(hist_data, data);
4800                 else
4801                         kfree(data);
4802         }
4803 }
4804
4805 static int parse_actions(struct hist_trigger_data *hist_data)
4806 {
4807         struct trace_array *tr = hist_data->event_file->tr;
4808         struct action_data *data;
4809         unsigned int i;
4810         int ret = 0;
4811         char *str;
4812         int len;
4813
4814         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4815                 str = hist_data->attrs->action_str[i];
4816
4817                 if ((len = str_has_prefix(str, "onmatch("))) {
4818                         char *action_str = str + len;
4819
4820                         data = onmatch_parse(tr, action_str);
4821                         if (IS_ERR(data)) {
4822                                 ret = PTR_ERR(data);
4823                                 break;
4824                         }
4825                 } else if ((len = str_has_prefix(str, "onmax("))) {
4826                         char *action_str = str + len;
4827
4828                         data = track_data_parse(hist_data, action_str,
4829                                                 HANDLER_ONMAX);
4830                         if (IS_ERR(data)) {
4831                                 ret = PTR_ERR(data);
4832                                 break;
4833                         }
4834                 } else if ((len = str_has_prefix(str, "onchange("))) {
4835                         char *action_str = str + len;
4836
4837                         data = track_data_parse(hist_data, action_str,
4838                                                 HANDLER_ONCHANGE);
4839                         if (IS_ERR(data)) {
4840                                 ret = PTR_ERR(data);
4841                                 break;
4842                         }
4843                 } else {
4844                         ret = -EINVAL;
4845                         break;
4846                 }
4847
4848                 hist_data->actions[hist_data->n_actions++] = data;
4849         }
4850
4851         return ret;
4852 }
4853
4854 static int create_actions(struct hist_trigger_data *hist_data)
4855 {
4856         struct action_data *data;
4857         unsigned int i;
4858         int ret = 0;
4859
4860         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4861                 data = hist_data->actions[i];
4862
4863                 if (data->handler == HANDLER_ONMATCH) {
4864                         ret = onmatch_create(hist_data, data);
4865                         if (ret)
4866                                 break;
4867                 } else if (data->handler == HANDLER_ONMAX ||
4868                            data->handler == HANDLER_ONCHANGE) {
4869                         ret = track_data_create(hist_data, data);
4870                         if (ret)
4871                                 break;
4872                 } else {
4873                         ret = -EINVAL;
4874                         break;
4875                 }
4876         }
4877
4878         return ret;
4879 }
4880
4881 static void print_actions(struct seq_file *m,
4882                           struct hist_trigger_data *hist_data,
4883                           struct tracing_map_elt *elt)
4884 {
4885         unsigned int i;
4886
4887         for (i = 0; i < hist_data->n_actions; i++) {
4888                 struct action_data *data = hist_data->actions[i];
4889
4890                 if (data->action == ACTION_SNAPSHOT)
4891                         continue;
4892
4893                 if (data->handler == HANDLER_ONMAX ||
4894                     data->handler == HANDLER_ONCHANGE)
4895                         track_data_print(m, hist_data, elt, data);
4896         }
4897 }
4898
4899 static void print_action_spec(struct seq_file *m,
4900                               struct hist_trigger_data *hist_data,
4901                               struct action_data *data)
4902 {
4903         unsigned int i;
4904
4905         if (data->action == ACTION_SAVE) {
4906                 for (i = 0; i < hist_data->n_save_vars; i++) {
4907                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4908                         if (i < hist_data->n_save_vars - 1)
4909                                 seq_puts(m, ",");
4910                 }
4911         } else if (data->action == ACTION_TRACE) {
4912                 if (data->use_trace_keyword)
4913                         seq_printf(m, "%s", data->synth_event_name);
4914                 for (i = 0; i < data->n_params; i++) {
4915                         if (i || data->use_trace_keyword)
4916                                 seq_puts(m, ",");
4917                         seq_printf(m, "%s", data->params[i]);
4918                 }
4919         }
4920 }
4921
4922 static void print_track_data_spec(struct seq_file *m,
4923                                   struct hist_trigger_data *hist_data,
4924                                   struct action_data *data)
4925 {
4926         if (data->handler == HANDLER_ONMAX)
4927                 seq_puts(m, ":onmax(");
4928         else if (data->handler == HANDLER_ONCHANGE)
4929                 seq_puts(m, ":onchange(");
4930         seq_printf(m, "%s", data->track_data.var_str);
4931         seq_printf(m, ").%s(", data->action_name);
4932
4933         print_action_spec(m, hist_data, data);
4934
4935         seq_puts(m, ")");
4936 }
4937
4938 static void print_onmatch_spec(struct seq_file *m,
4939                                struct hist_trigger_data *hist_data,
4940                                struct action_data *data)
4941 {
4942         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4943                    data->match_data.event);
4944
4945         seq_printf(m, "%s(", data->action_name);
4946
4947         print_action_spec(m, hist_data, data);
4948
4949         seq_puts(m, ")");
4950 }
4951
4952 static bool actions_match(struct hist_trigger_data *hist_data,
4953                           struct hist_trigger_data *hist_data_test)
4954 {
4955         unsigned int i, j;
4956
4957         if (hist_data->n_actions != hist_data_test->n_actions)
4958                 return false;
4959
4960         for (i = 0; i < hist_data->n_actions; i++) {
4961                 struct action_data *data = hist_data->actions[i];
4962                 struct action_data *data_test = hist_data_test->actions[i];
4963                 char *action_name, *action_name_test;
4964
4965                 if (data->handler != data_test->handler)
4966                         return false;
4967                 if (data->action != data_test->action)
4968                         return false;
4969
4970                 if (data->n_params != data_test->n_params)
4971                         return false;
4972
4973                 for (j = 0; j < data->n_params; j++) {
4974                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4975                                 return false;
4976                 }
4977
4978                 if (data->use_trace_keyword)
4979                         action_name = data->synth_event_name;
4980                 else
4981                         action_name = data->action_name;
4982
4983                 if (data_test->use_trace_keyword)
4984                         action_name_test = data_test->synth_event_name;
4985                 else
4986                         action_name_test = data_test->action_name;
4987
4988                 if (strcmp(action_name, action_name_test) != 0)
4989                         return false;
4990
4991                 if (data->handler == HANDLER_ONMATCH) {
4992                         if (strcmp(data->match_data.event_system,
4993                                    data_test->match_data.event_system) != 0)
4994                                 return false;
4995                         if (strcmp(data->match_data.event,
4996                                    data_test->match_data.event) != 0)
4997                                 return false;
4998                 } else if (data->handler == HANDLER_ONMAX ||
4999                            data->handler == HANDLER_ONCHANGE) {
5000                         if (strcmp(data->track_data.var_str,
5001                                    data_test->track_data.var_str) != 0)
5002                                 return false;
5003                 }
5004         }
5005
5006         return true;
5007 }
5008
5009
5010 static void print_actions_spec(struct seq_file *m,
5011                                struct hist_trigger_data *hist_data)
5012 {
5013         unsigned int i;
5014
5015         for (i = 0; i < hist_data->n_actions; i++) {
5016                 struct action_data *data = hist_data->actions[i];
5017
5018                 if (data->handler == HANDLER_ONMATCH)
5019                         print_onmatch_spec(m, hist_data, data);
5020                 else if (data->handler == HANDLER_ONMAX ||
5021                          data->handler == HANDLER_ONCHANGE)
5022                         print_track_data_spec(m, hist_data, data);
5023         }
5024 }
5025
5026 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5027 {
5028         unsigned int i;
5029
5030         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5031                 kfree(hist_data->field_var_hists[i]->cmd);
5032                 kfree(hist_data->field_var_hists[i]);
5033         }
5034 }
5035
5036 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5037 {
5038         if (!hist_data)
5039                 return;
5040
5041         destroy_hist_trigger_attrs(hist_data->attrs);
5042         destroy_hist_fields(hist_data);
5043         tracing_map_destroy(hist_data->map);
5044
5045         destroy_actions(hist_data);
5046         destroy_field_vars(hist_data);
5047         destroy_field_var_hists(hist_data);
5048
5049         kfree(hist_data);
5050 }
5051
5052 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5053 {
5054         struct tracing_map *map = hist_data->map;
5055         struct ftrace_event_field *field;
5056         struct hist_field *hist_field;
5057         int i, idx = 0;
5058
5059         for_each_hist_field(i, hist_data) {
5060                 hist_field = hist_data->fields[i];
5061                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5062                         tracing_map_cmp_fn_t cmp_fn;
5063
5064                         field = hist_field->field;
5065
5066                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5067                                 cmp_fn = tracing_map_cmp_none;
5068                         else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
5069                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5070                                                              hist_field->is_signed);
5071                         else if (is_string_field(field))
5072                                 cmp_fn = tracing_map_cmp_string;
5073                         else
5074                                 cmp_fn = tracing_map_cmp_num(field->size,
5075                                                              field->is_signed);
5076                         idx = tracing_map_add_key_field(map,
5077                                                         hist_field->offset,
5078                                                         cmp_fn);
5079                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5080                         idx = tracing_map_add_sum_field(map);
5081
5082                 if (idx < 0)
5083                         return idx;
5084
5085                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5086                         idx = tracing_map_add_var(map);
5087                         if (idx < 0)
5088                                 return idx;
5089                         hist_field->var.idx = idx;
5090                         hist_field->var.hist_data = hist_data;
5091                 }
5092         }
5093
5094         return 0;
5095 }
5096
5097 static struct hist_trigger_data *
5098 create_hist_data(unsigned int map_bits,
5099                  struct hist_trigger_attrs *attrs,
5100                  struct trace_event_file *file,
5101                  bool remove)
5102 {
5103         const struct tracing_map_ops *map_ops = NULL;
5104         struct hist_trigger_data *hist_data;
5105         int ret = 0;
5106
5107         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5108         if (!hist_data)
5109                 return ERR_PTR(-ENOMEM);
5110
5111         hist_data->attrs = attrs;
5112         hist_data->remove = remove;
5113         hist_data->event_file = file;
5114
5115         ret = parse_actions(hist_data);
5116         if (ret)
5117                 goto free;
5118
5119         ret = create_hist_fields(hist_data, file);
5120         if (ret)
5121                 goto free;
5122
5123         ret = create_sort_keys(hist_data);
5124         if (ret)
5125                 goto free;
5126
5127         map_ops = &hist_trigger_elt_data_ops;
5128
5129         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5130                                             map_ops, hist_data);
5131         if (IS_ERR(hist_data->map)) {
5132                 ret = PTR_ERR(hist_data->map);
5133                 hist_data->map = NULL;
5134                 goto free;
5135         }
5136
5137         ret = create_tracing_map_fields(hist_data);
5138         if (ret)
5139                 goto free;
5140  out:
5141         return hist_data;
5142  free:
5143         hist_data->attrs = NULL;
5144
5145         destroy_hist_data(hist_data);
5146
5147         hist_data = ERR_PTR(ret);
5148
5149         goto out;
5150 }
5151
5152 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5153                                     struct tracing_map_elt *elt,
5154                                     struct trace_buffer *buffer, void *rec,
5155                                     struct ring_buffer_event *rbe,
5156                                     u64 *var_ref_vals)
5157 {
5158         struct hist_elt_data *elt_data;
5159         struct hist_field *hist_field;
5160         unsigned int i, var_idx;
5161         u64 hist_val;
5162
5163         elt_data = elt->private_data;
5164         elt_data->var_ref_vals = var_ref_vals;
5165
5166         for_each_hist_val_field(i, hist_data) {
5167                 hist_field = hist_data->fields[i];
5168                 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5169                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5170                         var_idx = hist_field->var.idx;
5171
5172                         if (hist_field->flags &
5173                             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) {
5174                                 unsigned int str_start, var_str_idx, idx;
5175                                 char *str, *val_str;
5176                                 unsigned int size;
5177
5178                                 str_start = hist_data->n_field_var_str +
5179                                         hist_data->n_save_var_str;
5180                                 var_str_idx = hist_field->var_str_idx;
5181                                 idx = str_start + var_str_idx;
5182
5183                                 str = elt_data->field_var_str[idx];
5184                                 val_str = (char *)(uintptr_t)hist_val;
5185
5186                                 if (hist_field->flags & HIST_FIELD_FL_STRING) {
5187                                         size = min(hist_field->size, STR_VAR_LEN_MAX);
5188                                         strscpy(str, val_str, size);
5189                                 } else {
5190                                         char *stack_start = str + sizeof(unsigned long);
5191                                         int e;
5192
5193                                         e = stack_trace_save((void *)stack_start,
5194                                                              HIST_STACKTRACE_DEPTH,
5195                                                              HIST_STACKTRACE_SKIP);
5196                                         if (e < HIST_STACKTRACE_DEPTH - 1)
5197                                                 ((unsigned long *)stack_start)[e] = 0;
5198                                         *((unsigned long *)str) = e;
5199                                 }
5200                                 hist_val = (u64)(uintptr_t)str;
5201                         }
5202                         tracing_map_set_var(elt, var_idx, hist_val);
5203                         continue;
5204                 }
5205                 tracing_map_update_sum(elt, i, hist_val);
5206         }
5207
5208         for_each_hist_key_field(i, hist_data) {
5209                 hist_field = hist_data->fields[i];
5210                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5211                         hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5212                         var_idx = hist_field->var.idx;
5213                         tracing_map_set_var(elt, var_idx, hist_val);
5214                 }
5215         }
5216
5217         update_field_vars(hist_data, elt, buffer, rbe, rec);
5218 }
5219
5220 static inline void add_to_key(char *compound_key, void *key,
5221                               struct hist_field *key_field, void *rec)
5222 {
5223         size_t size = key_field->size;
5224
5225         if (key_field->flags & HIST_FIELD_FL_STRING) {
5226                 struct ftrace_event_field *field;
5227
5228                 field = key_field->field;
5229                 if (field->filter_type == FILTER_DYN_STRING ||
5230                     field->filter_type == FILTER_RDYN_STRING)
5231                         size = *(u32 *)(rec + field->offset) >> 16;
5232                 else if (field->filter_type == FILTER_STATIC_STRING)
5233                         size = field->size;
5234
5235                 /* ensure NULL-termination */
5236                 if (size > key_field->size - 1)
5237                         size = key_field->size - 1;
5238
5239                 strncpy(compound_key + key_field->offset, (char *)key, size);
5240         } else
5241                 memcpy(compound_key + key_field->offset, key, size);
5242 }
5243
5244 static void
5245 hist_trigger_actions(struct hist_trigger_data *hist_data,
5246                      struct tracing_map_elt *elt,
5247                      struct trace_buffer *buffer, void *rec,
5248                      struct ring_buffer_event *rbe, void *key,
5249                      u64 *var_ref_vals)
5250 {
5251         struct action_data *data;
5252         unsigned int i;
5253
5254         for (i = 0; i < hist_data->n_actions; i++) {
5255                 data = hist_data->actions[i];
5256                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5257         }
5258 }
5259
5260 static void event_hist_trigger(struct event_trigger_data *data,
5261                                struct trace_buffer *buffer, void *rec,
5262                                struct ring_buffer_event *rbe)
5263 {
5264         struct hist_trigger_data *hist_data = data->private_data;
5265         bool use_compound_key = (hist_data->n_keys > 1);
5266         unsigned long entries[HIST_STACKTRACE_DEPTH];
5267         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5268         char compound_key[HIST_KEY_SIZE_MAX];
5269         struct tracing_map_elt *elt = NULL;
5270         struct hist_field *key_field;
5271         u64 field_contents;
5272         void *key = NULL;
5273         unsigned int i;
5274
5275         if (unlikely(!rbe))
5276                 return;
5277
5278         memset(compound_key, 0, hist_data->key_size);
5279
5280         for_each_hist_key_field(i, hist_data) {
5281                 key_field = hist_data->fields[i];
5282
5283                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5284                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5285                         if (key_field->field) {
5286                                 unsigned long *stack, n_entries;
5287
5288                                 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5289                                 stack = (unsigned long *)(long)field_contents;
5290                                 n_entries = *stack;
5291                                 memcpy(entries, ++stack, n_entries * sizeof(unsigned long));
5292                         } else {
5293                                 stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5294                                                  HIST_STACKTRACE_SKIP);
5295                         }
5296                         key = entries;
5297                 } else {
5298                         field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5299                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5300                                 key = (void *)(unsigned long)field_contents;
5301                                 use_compound_key = true;
5302                         } else
5303                                 key = (void *)&field_contents;
5304                 }
5305
5306                 if (use_compound_key)
5307                         add_to_key(compound_key, key, key_field, rec);
5308         }
5309
5310         if (use_compound_key)
5311                 key = compound_key;
5312
5313         if (hist_data->n_var_refs &&
5314             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5315                 return;
5316
5317         elt = tracing_map_insert(hist_data->map, key);
5318         if (!elt)
5319                 return;
5320
5321         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5322
5323         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5324                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5325 }
5326
5327 static void hist_trigger_stacktrace_print(struct seq_file *m,
5328                                           unsigned long *stacktrace_entries,
5329                                           unsigned int max_entries)
5330 {
5331         unsigned int spaces = 8;
5332         unsigned int i;
5333
5334         for (i = 0; i < max_entries; i++) {
5335                 if (!stacktrace_entries[i])
5336                         return;
5337
5338                 seq_printf(m, "%*c", 1 + spaces, ' ');
5339                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5340         }
5341 }
5342
5343 static void hist_trigger_print_key(struct seq_file *m,
5344                                    struct hist_trigger_data *hist_data,
5345                                    void *key,
5346                                    struct tracing_map_elt *elt)
5347 {
5348         struct hist_field *key_field;
5349         bool multiline = false;
5350         const char *field_name;
5351         unsigned int i;
5352         u64 uval;
5353
5354         seq_puts(m, "{ ");
5355
5356         for_each_hist_key_field(i, hist_data) {
5357                 key_field = hist_data->fields[i];
5358
5359                 if (i > hist_data->n_vals)
5360                         seq_puts(m, ", ");
5361
5362                 field_name = hist_field_name(key_field, 0);
5363
5364                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5365                         uval = *(u64 *)(key + key_field->offset);
5366                         seq_printf(m, "%s: %llx", field_name, uval);
5367                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5368                         uval = *(u64 *)(key + key_field->offset);
5369                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5370                                    uval, (void *)(uintptr_t)uval);
5371                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5372                         uval = *(u64 *)(key + key_field->offset);
5373                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5374                                    uval, (void *)(uintptr_t)uval);
5375                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5376                         struct hist_elt_data *elt_data = elt->private_data;
5377                         char *comm;
5378
5379                         if (WARN_ON_ONCE(!elt_data))
5380                                 return;
5381
5382                         comm = elt_data->comm;
5383
5384                         uval = *(u64 *)(key + key_field->offset);
5385                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5386                                    comm, uval);
5387                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5388                         const char *syscall_name;
5389
5390                         uval = *(u64 *)(key + key_field->offset);
5391                         syscall_name = get_syscall_name(uval);
5392                         if (!syscall_name)
5393                                 syscall_name = "unknown_syscall";
5394
5395                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5396                                    syscall_name, uval);
5397                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5398                         if (key_field->field)
5399                                 seq_printf(m, "%s.stacktrace", key_field->field->name);
5400                         else
5401                                 seq_puts(m, "common_stacktrace:\n");
5402                         hist_trigger_stacktrace_print(m,
5403                                                       key + key_field->offset,
5404                                                       HIST_STACKTRACE_DEPTH);
5405                         multiline = true;
5406                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5407                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5408                                    *(u64 *)(key + key_field->offset));
5409                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5410                         unsigned long buckets = key_field->buckets;
5411                         uval = *(u64 *)(key + key_field->offset);
5412                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5413                                    uval, uval + buckets -1);
5414                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5415                         seq_printf(m, "%s: %-50s", field_name,
5416                                    (char *)(key + key_field->offset));
5417                 } else {
5418                         uval = *(u64 *)(key + key_field->offset);
5419                         seq_printf(m, "%s: %10llu", field_name, uval);
5420                 }
5421         }
5422
5423         if (!multiline)
5424                 seq_puts(m, " ");
5425
5426         seq_puts(m, "}");
5427 }
5428
5429 /* Get the 100 times of the percentage of @val in @total */
5430 static inline unsigned int __get_percentage(u64 val, u64 total)
5431 {
5432         if (!total)
5433                 goto div0;
5434
5435         if (val < (U64_MAX / 10000))
5436                 return (unsigned int)div64_ul(val * 10000, total);
5437
5438         total = div64_u64(total, 10000);
5439         if (!total)
5440                 goto div0;
5441
5442         return (unsigned int)div64_ul(val, total);
5443 div0:
5444         return val ? UINT_MAX : 0;
5445 }
5446
5447 #define BAR_CHAR '#'
5448
5449 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5450 {
5451         unsigned int len = __get_percentage(val, max);
5452         int i;
5453
5454         if (len == UINT_MAX) {
5455                 snprintf(buf, size, "[ERROR]");
5456                 return buf;
5457         }
5458
5459         len = len * size / 10000;
5460         for (i = 0; i < len && i < size; i++)
5461                 buf[i] = BAR_CHAR;
5462         while (i < size)
5463                 buf[i++] = ' ';
5464         buf[size] = '\0';
5465
5466         return buf;
5467 }
5468
5469 struct hist_val_stat {
5470         u64 max;
5471         u64 total;
5472 };
5473
5474 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5475                                    const char *field_name, unsigned long flags,
5476                                    struct hist_val_stat *stats,
5477                                    struct tracing_map_elt *elt)
5478 {
5479         u64 val = tracing_map_read_sum(elt, idx);
5480         unsigned int pc;
5481         char bar[21];
5482
5483         if (flags & HIST_FIELD_FL_PERCENT) {
5484                 pc = __get_percentage(val, stats[idx].total);
5485                 if (pc == UINT_MAX)
5486                         seq_printf(m, " %s (%%):[ERROR]", field_name);
5487                 else
5488                         seq_printf(m, " %s (%%): %3u.%02u", field_name,
5489                                         pc / 100, pc % 100);
5490         } else if (flags & HIST_FIELD_FL_GRAPH) {
5491                 seq_printf(m, " %s: %20s", field_name,
5492                            __fill_bar_str(bar, 20, val, stats[idx].max));
5493         } else if (flags & HIST_FIELD_FL_HEX) {
5494                 seq_printf(m, " %s: %10llx", field_name, val);
5495         } else {
5496                 seq_printf(m, " %s: %10llu", field_name, val);
5497         }
5498 }
5499
5500 static void hist_trigger_entry_print(struct seq_file *m,
5501                                      struct hist_trigger_data *hist_data,
5502                                      struct hist_val_stat *stats,
5503                                      void *key,
5504                                      struct tracing_map_elt *elt)
5505 {
5506         const char *field_name;
5507         unsigned int i = HITCOUNT_IDX;
5508         unsigned long flags;
5509
5510         hist_trigger_print_key(m, hist_data, key, elt);
5511
5512         /* At first, show the raw hitcount if !nohitcount */
5513         if (!hist_data->attrs->no_hitcount)
5514                 hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5515
5516         for (i = 1; i < hist_data->n_vals; i++) {
5517                 field_name = hist_field_name(hist_data->fields[i], 0);
5518                 flags = hist_data->fields[i]->flags;
5519                 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
5520                         continue;
5521
5522                 seq_puts(m, " ");
5523                 hist_trigger_print_val(m, i, field_name, flags, stats, elt);
5524         }
5525
5526         print_actions(m, hist_data, elt);
5527
5528         seq_puts(m, "\n");
5529 }
5530
5531 static int print_entries(struct seq_file *m,
5532                          struct hist_trigger_data *hist_data)
5533 {
5534         struct tracing_map_sort_entry **sort_entries = NULL;
5535         struct tracing_map *map = hist_data->map;
5536         int i, j, n_entries;
5537         struct hist_val_stat *stats = NULL;
5538         u64 val;
5539
5540         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5541                                              hist_data->n_sort_keys,
5542                                              &sort_entries);
5543         if (n_entries < 0)
5544                 return n_entries;
5545
5546         /* Calculate the max and the total for each field if needed. */
5547         for (j = 0; j < hist_data->n_vals; j++) {
5548                 if (!(hist_data->fields[j]->flags &
5549                         (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5550                         continue;
5551                 if (!stats) {
5552                         stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5553                                        GFP_KERNEL);
5554                         if (!stats) {
5555                                 n_entries = -ENOMEM;
5556                                 goto out;
5557                         }
5558                 }
5559                 for (i = 0; i < n_entries; i++) {
5560                         val = tracing_map_read_sum(sort_entries[i]->elt, j);
5561                         stats[j].total += val;
5562                         if (stats[j].max < val)
5563                                 stats[j].max = val;
5564                 }
5565         }
5566
5567         for (i = 0; i < n_entries; i++)
5568                 hist_trigger_entry_print(m, hist_data, stats,
5569                                          sort_entries[i]->key,
5570                                          sort_entries[i]->elt);
5571
5572         kfree(stats);
5573 out:
5574         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5575
5576         return n_entries;
5577 }
5578
5579 static void hist_trigger_show(struct seq_file *m,
5580                               struct event_trigger_data *data, int n)
5581 {
5582         struct hist_trigger_data *hist_data;
5583         int n_entries;
5584
5585         if (n > 0)
5586                 seq_puts(m, "\n\n");
5587
5588         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5589         data->ops->print(m, data);
5590         seq_puts(m, "#\n\n");
5591
5592         hist_data = data->private_data;
5593         n_entries = print_entries(m, hist_data);
5594         if (n_entries < 0)
5595                 n_entries = 0;
5596
5597         track_data_snapshot_print(m, hist_data);
5598
5599         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5600                    (u64)atomic64_read(&hist_data->map->hits),
5601                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5602 }
5603
5604 static int hist_show(struct seq_file *m, void *v)
5605 {
5606         struct event_trigger_data *data;
5607         struct trace_event_file *event_file;
5608         int n = 0, ret = 0;
5609
5610         mutex_lock(&event_mutex);
5611
5612         event_file = event_file_data(m->private);
5613         if (unlikely(!event_file)) {
5614                 ret = -ENODEV;
5615                 goto out_unlock;
5616         }
5617
5618         list_for_each_entry(data, &event_file->triggers, list) {
5619                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5620                         hist_trigger_show(m, data, n++);
5621         }
5622
5623  out_unlock:
5624         mutex_unlock(&event_mutex);
5625
5626         return ret;
5627 }
5628
5629 static int event_hist_open(struct inode *inode, struct file *file)
5630 {
5631         int ret;
5632
5633         ret = tracing_open_file_tr(inode, file);
5634         if (ret)
5635                 return ret;
5636
5637         /* Clear private_data to avoid warning in single_open() */
5638         file->private_data = NULL;
5639         return single_open(file, hist_show, file);
5640 }
5641
5642 const struct file_operations event_hist_fops = {
5643         .open = event_hist_open,
5644         .read = seq_read,
5645         .llseek = seq_lseek,
5646         .release = tracing_single_release_file_tr,
5647 };
5648
5649 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5650 static void hist_field_debug_show_flags(struct seq_file *m,
5651                                         unsigned long flags)
5652 {
5653         seq_puts(m, "      flags:\n");
5654
5655         if (flags & HIST_FIELD_FL_KEY)
5656                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5657         else if (flags & HIST_FIELD_FL_HITCOUNT)
5658                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5659         else if (flags & HIST_FIELD_FL_VAR)
5660                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5661         else if (flags & HIST_FIELD_FL_VAR_REF)
5662                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5663         else
5664                 seq_puts(m, "        VAL: normal u64 value\n");
5665
5666         if (flags & HIST_FIELD_FL_ALIAS)
5667                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5668         else if (flags & HIST_FIELD_FL_CONST)
5669                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5670 }
5671
5672 static int hist_field_debug_show(struct seq_file *m,
5673                                  struct hist_field *field, unsigned long flags)
5674 {
5675         if ((field->flags & flags) != flags) {
5676                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5677                 return -EINVAL;
5678         }
5679
5680         hist_field_debug_show_flags(m, field->flags);
5681         if (field->field)
5682                 seq_printf(m, "      ftrace_event_field name: %s\n",
5683                            field->field->name);
5684
5685         if (field->flags & HIST_FIELD_FL_VAR) {
5686                 seq_printf(m, "      var.name: %s\n", field->var.name);
5687                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5688                            field->var.idx);
5689         }
5690
5691         if (field->flags & HIST_FIELD_FL_CONST)
5692                 seq_printf(m, "      constant: %llu\n", field->constant);
5693
5694         if (field->flags & HIST_FIELD_FL_ALIAS)
5695                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5696                            field->var_ref_idx);
5697
5698         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5699                 seq_printf(m, "      name: %s\n", field->name);
5700                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5701                            field->var.idx);
5702                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5703                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5704                            field->var_ref_idx);
5705                 if (field->system)
5706                         seq_printf(m, "      system: %s\n", field->system);
5707                 if (field->event_name)
5708                         seq_printf(m, "      event_name: %s\n", field->event_name);
5709         }
5710
5711         seq_printf(m, "      type: %s\n", field->type);
5712         seq_printf(m, "      size: %u\n", field->size);
5713         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5714
5715         return 0;
5716 }
5717
5718 static int field_var_debug_show(struct seq_file *m,
5719                                 struct field_var *field_var, unsigned int i,
5720                                 bool save_vars)
5721 {
5722         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5723         struct hist_field *field;
5724         int ret = 0;
5725
5726         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5727
5728         field = field_var->var;
5729
5730         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5731
5732         hist_field_debug_show_flags(m, field->flags);
5733         seq_printf(m, "      var.name: %s\n", field->var.name);
5734         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5735                    field->var.idx);
5736
5737         field = field_var->val;
5738
5739         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5740         if (field->field)
5741                 seq_printf(m, "      ftrace_event_field name: %s\n",
5742                            field->field->name);
5743         else {
5744                 ret = -EINVAL;
5745                 goto out;
5746         }
5747
5748         seq_printf(m, "      type: %s\n", field->type);
5749         seq_printf(m, "      size: %u\n", field->size);
5750         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5751 out:
5752         return ret;
5753 }
5754
5755 static int hist_action_debug_show(struct seq_file *m,
5756                                   struct action_data *data, int i)
5757 {
5758         int ret = 0;
5759
5760         if (data->handler == HANDLER_ONMAX ||
5761             data->handler == HANDLER_ONCHANGE) {
5762                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5763                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5764                                             HIST_FIELD_FL_VAR_REF);
5765                 if (ret)
5766                         goto out;
5767
5768                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5769                 ret = hist_field_debug_show(m, data->track_data.track_var,
5770                                             HIST_FIELD_FL_VAR);
5771                 if (ret)
5772                         goto out;
5773         }
5774
5775         if (data->handler == HANDLER_ONMATCH) {
5776                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5777                            i, data->match_data.event_system);
5778                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5779                            i, data->match_data.event);
5780         }
5781 out:
5782         return ret;
5783 }
5784
5785 static int hist_actions_debug_show(struct seq_file *m,
5786                                    struct hist_trigger_data *hist_data)
5787 {
5788         int i, ret = 0;
5789
5790         if (hist_data->n_actions)
5791                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5792
5793         for (i = 0; i < hist_data->n_actions; i++) {
5794                 struct action_data *action = hist_data->actions[i];
5795
5796                 ret = hist_action_debug_show(m, action, i);
5797                 if (ret)
5798                         goto out;
5799         }
5800
5801         if (hist_data->n_save_vars)
5802                 seq_puts(m, "\n  save action variables (save() params):\n");
5803
5804         for (i = 0; i < hist_data->n_save_vars; i++) {
5805                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5806                 if (ret)
5807                         goto out;
5808         }
5809 out:
5810         return ret;
5811 }
5812
5813 static void hist_trigger_debug_show(struct seq_file *m,
5814                                     struct event_trigger_data *data, int n)
5815 {
5816         struct hist_trigger_data *hist_data;
5817         int i, ret;
5818
5819         if (n > 0)
5820                 seq_puts(m, "\n\n");
5821
5822         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5823         data->ops->print(m, data);
5824         seq_puts(m, "#\n\n");
5825
5826         hist_data = data->private_data;
5827
5828         seq_printf(m, "hist_data: %p\n\n", hist_data);
5829         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5830         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5831         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5832
5833         seq_puts(m, "\n  val fields:\n\n");
5834
5835         seq_puts(m, "    hist_data->fields[0]:\n");
5836         ret = hist_field_debug_show(m, hist_data->fields[0],
5837                                     HIST_FIELD_FL_HITCOUNT);
5838         if (ret)
5839                 return;
5840
5841         for (i = 1; i < hist_data->n_vals; i++) {
5842                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5843                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5844                 if (ret)
5845                         return;
5846         }
5847
5848         seq_puts(m, "\n  key fields:\n");
5849
5850         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5851                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5852                 ret = hist_field_debug_show(m, hist_data->fields[i],
5853                                             HIST_FIELD_FL_KEY);
5854                 if (ret)
5855                         return;
5856         }
5857
5858         if (hist_data->n_var_refs)
5859                 seq_puts(m, "\n  variable reference fields:\n");
5860
5861         for (i = 0; i < hist_data->n_var_refs; i++) {
5862                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5863                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5864                                             HIST_FIELD_FL_VAR_REF);
5865                 if (ret)
5866                         return;
5867         }
5868
5869         if (hist_data->n_field_vars)
5870                 seq_puts(m, "\n  field variables:\n");
5871
5872         for (i = 0; i < hist_data->n_field_vars; i++) {
5873                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5874                 if (ret)
5875                         return;
5876         }
5877
5878         ret = hist_actions_debug_show(m, hist_data);
5879         if (ret)
5880                 return;
5881 }
5882
5883 static int hist_debug_show(struct seq_file *m, void *v)
5884 {
5885         struct event_trigger_data *data;
5886         struct trace_event_file *event_file;
5887         int n = 0, ret = 0;
5888
5889         mutex_lock(&event_mutex);
5890
5891         event_file = event_file_data(m->private);
5892         if (unlikely(!event_file)) {
5893                 ret = -ENODEV;
5894                 goto out_unlock;
5895         }
5896
5897         list_for_each_entry(data, &event_file->triggers, list) {
5898                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5899                         hist_trigger_debug_show(m, data, n++);
5900         }
5901
5902  out_unlock:
5903         mutex_unlock(&event_mutex);
5904
5905         return ret;
5906 }
5907
5908 static int event_hist_debug_open(struct inode *inode, struct file *file)
5909 {
5910         int ret;
5911
5912         ret = tracing_open_file_tr(inode, file);
5913         if (ret)
5914                 return ret;
5915
5916         /* Clear private_data to avoid warning in single_open() */
5917         file->private_data = NULL;
5918         return single_open(file, hist_debug_show, file);
5919 }
5920
5921 const struct file_operations event_hist_debug_fops = {
5922         .open = event_hist_debug_open,
5923         .read = seq_read,
5924         .llseek = seq_lseek,
5925         .release = tracing_single_release_file_tr,
5926 };
5927 #endif
5928
5929 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5930 {
5931         const char *field_name = hist_field_name(hist_field, 0);
5932
5933         if (hist_field->var.name)
5934                 seq_printf(m, "%s=", hist_field->var.name);
5935
5936         if (hist_field->flags & HIST_FIELD_FL_CPU)
5937                 seq_puts(m, "common_cpu");
5938         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5939                 seq_printf(m, "%llu", hist_field->constant);
5940         else if (field_name) {
5941                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5942                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5943                         seq_putc(m, '$');
5944                 seq_printf(m, "%s", field_name);
5945         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5946                 seq_puts(m, "common_timestamp");
5947
5948         if (hist_field->flags) {
5949                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5950                     !(hist_field->flags & HIST_FIELD_FL_EXPR) &&
5951                     !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) {
5952                         const char *flags = get_hist_field_flags(hist_field);
5953
5954                         if (flags)
5955                                 seq_printf(m, ".%s", flags);
5956                 }
5957         }
5958         if (hist_field->buckets)
5959                 seq_printf(m, "=%ld", hist_field->buckets);
5960 }
5961
5962 static int event_hist_trigger_print(struct seq_file *m,
5963                                     struct event_trigger_data *data)
5964 {
5965         struct hist_trigger_data *hist_data = data->private_data;
5966         struct hist_field *field;
5967         bool have_var = false;
5968         bool show_val = false;
5969         unsigned int i;
5970
5971         seq_puts(m, HIST_PREFIX);
5972
5973         if (data->name)
5974                 seq_printf(m, "%s:", data->name);
5975
5976         seq_puts(m, "keys=");
5977
5978         for_each_hist_key_field(i, hist_data) {
5979                 field = hist_data->fields[i];
5980
5981                 if (i > hist_data->n_vals)
5982                         seq_puts(m, ",");
5983
5984                 if (field->flags & HIST_FIELD_FL_STACKTRACE) {
5985                         if (field->field)
5986                                 seq_printf(m, "%s.stacktrace", field->field->name);
5987                         else
5988                                 seq_puts(m, "common_stacktrace");
5989                 } else
5990                         hist_field_print(m, field);
5991         }
5992
5993         seq_puts(m, ":vals=");
5994
5995         for_each_hist_val_field(i, hist_data) {
5996                 field = hist_data->fields[i];
5997                 if (field->flags & HIST_FIELD_FL_VAR) {
5998                         have_var = true;
5999                         continue;
6000                 }
6001
6002                 if (i == HITCOUNT_IDX) {
6003                         if (hist_data->attrs->no_hitcount)
6004                                 continue;
6005                         seq_puts(m, "hitcount");
6006                 } else {
6007                         if (show_val)
6008                                 seq_puts(m, ",");
6009                         hist_field_print(m, field);
6010                 }
6011                 show_val = true;
6012         }
6013
6014         if (have_var) {
6015                 unsigned int n = 0;
6016
6017                 seq_puts(m, ":");
6018
6019                 for_each_hist_val_field(i, hist_data) {
6020                         field = hist_data->fields[i];
6021
6022                         if (field->flags & HIST_FIELD_FL_VAR) {
6023                                 if (n++)
6024                                         seq_puts(m, ",");
6025                                 hist_field_print(m, field);
6026                         }
6027                 }
6028         }
6029
6030         seq_puts(m, ":sort=");
6031
6032         for (i = 0; i < hist_data->n_sort_keys; i++) {
6033                 struct tracing_map_sort_key *sort_key;
6034                 unsigned int idx, first_key_idx;
6035
6036                 /* skip VAR vals */
6037                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
6038
6039                 sort_key = &hist_data->sort_keys[i];
6040                 idx = sort_key->field_idx;
6041
6042                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
6043                         return -EINVAL;
6044
6045                 if (i > 0)
6046                         seq_puts(m, ",");
6047
6048                 if (idx == HITCOUNT_IDX)
6049                         seq_puts(m, "hitcount");
6050                 else {
6051                         if (idx >= first_key_idx)
6052                                 idx += hist_data->n_vars;
6053                         hist_field_print(m, hist_data->fields[idx]);
6054                 }
6055
6056                 if (sort_key->descending)
6057                         seq_puts(m, ".descending");
6058         }
6059         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
6060         if (hist_data->enable_timestamps)
6061                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
6062         if (hist_data->attrs->no_hitcount)
6063                 seq_puts(m, ":nohitcount");
6064
6065         print_actions_spec(m, hist_data);
6066
6067         if (data->filter_str)
6068                 seq_printf(m, " if %s", data->filter_str);
6069
6070         if (data->paused)
6071                 seq_puts(m, " [paused]");
6072         else
6073                 seq_puts(m, " [active]");
6074
6075         seq_putc(m, '\n');
6076
6077         return 0;
6078 }
6079
6080 static int event_hist_trigger_init(struct event_trigger_data *data)
6081 {
6082         struct hist_trigger_data *hist_data = data->private_data;
6083
6084         if (!data->ref && hist_data->attrs->name)
6085                 save_named_trigger(hist_data->attrs->name, data);
6086
6087         data->ref++;
6088
6089         return 0;
6090 }
6091
6092 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6093 {
6094         struct trace_event_file *file;
6095         unsigned int i;
6096         char *cmd;
6097         int ret;
6098
6099         for (i = 0; i < hist_data->n_field_var_hists; i++) {
6100                 file = hist_data->field_var_hists[i]->hist_data->event_file;
6101                 cmd = hist_data->field_var_hists[i]->cmd;
6102                 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
6103                                                "!hist", "hist", cmd);
6104                 WARN_ON_ONCE(ret < 0);
6105         }
6106 }
6107
6108 static void event_hist_trigger_free(struct event_trigger_data *data)
6109 {
6110         struct hist_trigger_data *hist_data = data->private_data;
6111
6112         if (WARN_ON_ONCE(data->ref <= 0))
6113                 return;
6114
6115         data->ref--;
6116         if (!data->ref) {
6117                 if (data->name)
6118                         del_named_trigger(data);
6119
6120                 trigger_data_free(data);
6121
6122                 remove_hist_vars(hist_data);
6123
6124                 unregister_field_var_hists(hist_data);
6125
6126                 destroy_hist_data(hist_data);
6127         }
6128 }
6129
6130 static struct event_trigger_ops event_hist_trigger_ops = {
6131         .trigger                = event_hist_trigger,
6132         .print                  = event_hist_trigger_print,
6133         .init                   = event_hist_trigger_init,
6134         .free                   = event_hist_trigger_free,
6135 };
6136
6137 static int event_hist_trigger_named_init(struct event_trigger_data *data)
6138 {
6139         data->ref++;
6140
6141         save_named_trigger(data->named_data->name, data);
6142
6143         event_hist_trigger_init(data->named_data);
6144
6145         return 0;
6146 }
6147
6148 static void event_hist_trigger_named_free(struct event_trigger_data *data)
6149 {
6150         if (WARN_ON_ONCE(data->ref <= 0))
6151                 return;
6152
6153         event_hist_trigger_free(data->named_data);
6154
6155         data->ref--;
6156         if (!data->ref) {
6157                 del_named_trigger(data);
6158                 trigger_data_free(data);
6159         }
6160 }
6161
6162 static struct event_trigger_ops event_hist_trigger_named_ops = {
6163         .trigger                = event_hist_trigger,
6164         .print                  = event_hist_trigger_print,
6165         .init                   = event_hist_trigger_named_init,
6166         .free                   = event_hist_trigger_named_free,
6167 };
6168
6169 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6170                                                             char *param)
6171 {
6172         return &event_hist_trigger_ops;
6173 }
6174
6175 static void hist_clear(struct event_trigger_data *data)
6176 {
6177         struct hist_trigger_data *hist_data = data->private_data;
6178
6179         if (data->name)
6180                 pause_named_trigger(data);
6181
6182         tracepoint_synchronize_unregister();
6183
6184         tracing_map_clear(hist_data->map);
6185
6186         if (data->name)
6187                 unpause_named_trigger(data);
6188 }
6189
6190 static bool compatible_field(struct ftrace_event_field *field,
6191                              struct ftrace_event_field *test_field)
6192 {
6193         if (field == test_field)
6194                 return true;
6195         if (field == NULL || test_field == NULL)
6196                 return false;
6197         if (strcmp(field->name, test_field->name) != 0)
6198                 return false;
6199         if (strcmp(field->type, test_field->type) != 0)
6200                 return false;
6201         if (field->size != test_field->size)
6202                 return false;
6203         if (field->is_signed != test_field->is_signed)
6204                 return false;
6205
6206         return true;
6207 }
6208
6209 static bool hist_trigger_match(struct event_trigger_data *data,
6210                                struct event_trigger_data *data_test,
6211                                struct event_trigger_data *named_data,
6212                                bool ignore_filter)
6213 {
6214         struct tracing_map_sort_key *sort_key, *sort_key_test;
6215         struct hist_trigger_data *hist_data, *hist_data_test;
6216         struct hist_field *key_field, *key_field_test;
6217         unsigned int i;
6218
6219         if (named_data && (named_data != data_test) &&
6220             (named_data != data_test->named_data))
6221                 return false;
6222
6223         if (!named_data && is_named_trigger(data_test))
6224                 return false;
6225
6226         hist_data = data->private_data;
6227         hist_data_test = data_test->private_data;
6228
6229         if (hist_data->n_vals != hist_data_test->n_vals ||
6230             hist_data->n_fields != hist_data_test->n_fields ||
6231             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6232                 return false;
6233
6234         if (!ignore_filter) {
6235                 if ((data->filter_str && !data_test->filter_str) ||
6236                    (!data->filter_str && data_test->filter_str))
6237                         return false;
6238         }
6239
6240         for_each_hist_field(i, hist_data) {
6241                 key_field = hist_data->fields[i];
6242                 key_field_test = hist_data_test->fields[i];
6243
6244                 if (key_field->flags != key_field_test->flags)
6245                         return false;
6246                 if (!compatible_field(key_field->field, key_field_test->field))
6247                         return false;
6248                 if (key_field->offset != key_field_test->offset)
6249                         return false;
6250                 if (key_field->size != key_field_test->size)
6251                         return false;
6252                 if (key_field->is_signed != key_field_test->is_signed)
6253                         return false;
6254                 if (!!key_field->var.name != !!key_field_test->var.name)
6255                         return false;
6256                 if (key_field->var.name &&
6257                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
6258                         return false;
6259         }
6260
6261         for (i = 0; i < hist_data->n_sort_keys; i++) {
6262                 sort_key = &hist_data->sort_keys[i];
6263                 sort_key_test = &hist_data_test->sort_keys[i];
6264
6265                 if (sort_key->field_idx != sort_key_test->field_idx ||
6266                     sort_key->descending != sort_key_test->descending)
6267                         return false;
6268         }
6269
6270         if (!ignore_filter && data->filter_str &&
6271             (strcmp(data->filter_str, data_test->filter_str) != 0))
6272                 return false;
6273
6274         if (!actions_match(hist_data, hist_data_test))
6275                 return false;
6276
6277         return true;
6278 }
6279
6280 static bool existing_hist_update_only(char *glob,
6281                                       struct event_trigger_data *data,
6282                                       struct trace_event_file *file)
6283 {
6284         struct hist_trigger_data *hist_data = data->private_data;
6285         struct event_trigger_data *test, *named_data = NULL;
6286         bool updated = false;
6287
6288         if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6289             !hist_data->attrs->clear)
6290                 goto out;
6291
6292         if (hist_data->attrs->name) {
6293                 named_data = find_named_trigger(hist_data->attrs->name);
6294                 if (named_data) {
6295                         if (!hist_trigger_match(data, named_data, named_data,
6296                                                 true))
6297                                 goto out;
6298                 }
6299         }
6300
6301         if (hist_data->attrs->name && !named_data)
6302                 goto out;
6303
6304         list_for_each_entry(test, &file->triggers, list) {
6305                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6306                         if (!hist_trigger_match(data, test, named_data, false))
6307                                 continue;
6308                         if (hist_data->attrs->pause)
6309                                 test->paused = true;
6310                         else if (hist_data->attrs->cont)
6311                                 test->paused = false;
6312                         else if (hist_data->attrs->clear)
6313                                 hist_clear(test);
6314                         updated = true;
6315                         goto out;
6316                 }
6317         }
6318  out:
6319         return updated;
6320 }
6321
6322 static int hist_register_trigger(char *glob,
6323                                  struct event_trigger_data *data,
6324                                  struct trace_event_file *file)
6325 {
6326         struct hist_trigger_data *hist_data = data->private_data;
6327         struct event_trigger_data *test, *named_data = NULL;
6328         struct trace_array *tr = file->tr;
6329         int ret = 0;
6330
6331         if (hist_data->attrs->name) {
6332                 named_data = find_named_trigger(hist_data->attrs->name);
6333                 if (named_data) {
6334                         if (!hist_trigger_match(data, named_data, named_data,
6335                                                 true)) {
6336                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6337                                 ret = -EINVAL;
6338                                 goto out;
6339                         }
6340                 }
6341         }
6342
6343         if (hist_data->attrs->name && !named_data)
6344                 goto new;
6345
6346         lockdep_assert_held(&event_mutex);
6347
6348         list_for_each_entry(test, &file->triggers, list) {
6349                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6350                         if (hist_trigger_match(data, test, named_data, false)) {
6351                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6352                                 ret = -EEXIST;
6353                                 goto out;
6354                         }
6355                 }
6356         }
6357  new:
6358         if (hist_data->attrs->cont || hist_data->attrs->clear) {
6359                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6360                 ret = -ENOENT;
6361                 goto out;
6362         }
6363
6364         if (hist_data->attrs->pause)
6365                 data->paused = true;
6366
6367         if (named_data) {
6368                 data->private_data = named_data->private_data;
6369                 set_named_trigger_data(data, named_data);
6370                 data->ops = &event_hist_trigger_named_ops;
6371         }
6372
6373         if (data->ops->init) {
6374                 ret = data->ops->init(data);
6375                 if (ret < 0)
6376                         goto out;
6377         }
6378
6379         if (hist_data->enable_timestamps) {
6380                 char *clock = hist_data->attrs->clock;
6381
6382                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6383                 if (ret) {
6384                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6385                         goto out;
6386                 }
6387
6388                 tracing_set_filter_buffering(file->tr, true);
6389         }
6390
6391         if (named_data)
6392                 destroy_hist_data(hist_data);
6393  out:
6394         return ret;
6395 }
6396
6397 static int hist_trigger_enable(struct event_trigger_data *data,
6398                                struct trace_event_file *file)
6399 {
6400         int ret = 0;
6401
6402         list_add_tail_rcu(&data->list, &file->triggers);
6403
6404         update_cond_flag(file);
6405
6406         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6407                 list_del_rcu(&data->list);
6408                 update_cond_flag(file);
6409                 ret--;
6410         }
6411
6412         return ret;
6413 }
6414
6415 static bool have_hist_trigger_match(struct event_trigger_data *data,
6416                                     struct trace_event_file *file)
6417 {
6418         struct hist_trigger_data *hist_data = data->private_data;
6419         struct event_trigger_data *test, *named_data = NULL;
6420         bool match = false;
6421
6422         lockdep_assert_held(&event_mutex);
6423
6424         if (hist_data->attrs->name)
6425                 named_data = find_named_trigger(hist_data->attrs->name);
6426
6427         list_for_each_entry(test, &file->triggers, list) {
6428                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6429                         if (hist_trigger_match(data, test, named_data, false)) {
6430                                 match = true;
6431                                 break;
6432                         }
6433                 }
6434         }
6435
6436         return match;
6437 }
6438
6439 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6440                                     struct trace_event_file *file)
6441 {
6442         struct hist_trigger_data *hist_data = data->private_data;
6443         struct event_trigger_data *test, *named_data = NULL;
6444
6445         lockdep_assert_held(&event_mutex);
6446
6447         if (hist_data->attrs->name)
6448                 named_data = find_named_trigger(hist_data->attrs->name);
6449
6450         list_for_each_entry(test, &file->triggers, list) {
6451                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6452                         if (!hist_trigger_match(data, test, named_data, false))
6453                                 continue;
6454                         hist_data = test->private_data;
6455                         if (check_var_refs(hist_data))
6456                                 return true;
6457                         break;
6458                 }
6459         }
6460
6461         return false;
6462 }
6463
6464 static void hist_unregister_trigger(char *glob,
6465                                     struct event_trigger_data *data,
6466                                     struct trace_event_file *file)
6467 {
6468         struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6469         struct hist_trigger_data *hist_data = data->private_data;
6470
6471         lockdep_assert_held(&event_mutex);
6472
6473         if (hist_data->attrs->name)
6474                 named_data = find_named_trigger(hist_data->attrs->name);
6475
6476         list_for_each_entry(iter, &file->triggers, list) {
6477                 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6478                         if (!hist_trigger_match(data, iter, named_data, false))
6479                                 continue;
6480                         test = iter;
6481                         list_del_rcu(&test->list);
6482                         trace_event_trigger_enable_disable(file, 0);
6483                         update_cond_flag(file);
6484                         break;
6485                 }
6486         }
6487
6488         if (test && test->ops->free)
6489                 test->ops->free(test);
6490
6491         if (hist_data->enable_timestamps) {
6492                 if (!hist_data->remove || test)
6493                         tracing_set_filter_buffering(file->tr, false);
6494         }
6495 }
6496
6497 static bool hist_file_check_refs(struct trace_event_file *file)
6498 {
6499         struct hist_trigger_data *hist_data;
6500         struct event_trigger_data *test;
6501
6502         lockdep_assert_held(&event_mutex);
6503
6504         list_for_each_entry(test, &file->triggers, list) {
6505                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6506                         hist_data = test->private_data;
6507                         if (check_var_refs(hist_data))
6508                                 return true;
6509                 }
6510         }
6511
6512         return false;
6513 }
6514
6515 static void hist_unreg_all(struct trace_event_file *file)
6516 {
6517         struct event_trigger_data *test, *n;
6518         struct hist_trigger_data *hist_data;
6519         struct synth_event *se;
6520         const char *se_name;
6521
6522         lockdep_assert_held(&event_mutex);
6523
6524         if (hist_file_check_refs(file))
6525                 return;
6526
6527         list_for_each_entry_safe(test, n, &file->triggers, list) {
6528                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6529                         hist_data = test->private_data;
6530                         list_del_rcu(&test->list);
6531                         trace_event_trigger_enable_disable(file, 0);
6532
6533                         se_name = trace_event_name(file->event_call);
6534                         se = find_synth_event(se_name);
6535                         if (se)
6536                                 se->ref--;
6537
6538                         update_cond_flag(file);
6539                         if (hist_data->enable_timestamps)
6540                                 tracing_set_filter_buffering(file->tr, false);
6541                         if (test->ops->free)
6542                                 test->ops->free(test);
6543                 }
6544         }
6545 }
6546
6547 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6548                                     struct trace_event_file *file,
6549                                     char *glob, char *cmd,
6550                                     char *param_and_filter)
6551 {
6552         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6553         struct event_trigger_data *trigger_data;
6554         struct hist_trigger_attrs *attrs;
6555         struct hist_trigger_data *hist_data;
6556         char *param, *filter, *p, *start;
6557         struct synth_event *se;
6558         const char *se_name;
6559         bool remove;
6560         int ret = 0;
6561
6562         lockdep_assert_held(&event_mutex);
6563
6564         if (WARN_ON(!glob))
6565                 return -EINVAL;
6566
6567         if (glob[0]) {
6568                 hist_err_clear();
6569                 last_cmd_set(file, param_and_filter);
6570         }
6571
6572         remove = event_trigger_check_remove(glob);
6573
6574         if (event_trigger_empty_param(param_and_filter))
6575                 return -EINVAL;
6576
6577         /*
6578          * separate the trigger from the filter (k:v [if filter])
6579          * allowing for whitespace in the trigger
6580          */
6581         p = param = param_and_filter;
6582         do {
6583                 p = strstr(p, "if");
6584                 if (!p)
6585                         break;
6586                 if (p == param_and_filter)
6587                         return -EINVAL;
6588                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6589                         p++;
6590                         continue;
6591                 }
6592                 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6593                         return -EINVAL;
6594                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6595                         p++;
6596                         continue;
6597                 }
6598                 break;
6599         } while (1);
6600
6601         if (!p)
6602                 filter = NULL;
6603         else {
6604                 *(p - 1) = '\0';
6605                 filter = strstrip(p);
6606                 param = strstrip(param);
6607         }
6608
6609         /*
6610          * To simplify arithmetic expression parsing, replace occurrences of
6611          * '.sym-offset' modifier with '.symXoffset'
6612          */
6613         start = strstr(param, ".sym-offset");
6614         while (start) {
6615                 *(start + 4) = 'X';
6616                 start = strstr(start + 11, ".sym-offset");
6617         }
6618
6619         attrs = parse_hist_trigger_attrs(file->tr, param);
6620         if (IS_ERR(attrs))
6621                 return PTR_ERR(attrs);
6622
6623         if (attrs->map_bits)
6624                 hist_trigger_bits = attrs->map_bits;
6625
6626         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6627         if (IS_ERR(hist_data)) {
6628                 destroy_hist_trigger_attrs(attrs);
6629                 return PTR_ERR(hist_data);
6630         }
6631
6632         trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data);
6633         if (!trigger_data) {
6634                 ret = -ENOMEM;
6635                 goto out_free;
6636         }
6637
6638         ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6639         if (ret < 0)
6640                 goto out_free;
6641
6642         if (remove) {
6643                 if (!have_hist_trigger_match(trigger_data, file))
6644                         goto out_free;
6645
6646                 if (hist_trigger_check_refs(trigger_data, file)) {
6647                         ret = -EBUSY;
6648                         goto out_free;
6649                 }
6650
6651                 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6652                 se_name = trace_event_name(file->event_call);
6653                 se = find_synth_event(se_name);
6654                 if (se)
6655                         se->ref--;
6656                 ret = 0;
6657                 goto out_free;
6658         }
6659
6660         if (existing_hist_update_only(glob, trigger_data, file))
6661                 goto out_free;
6662
6663         ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6664         if (ret < 0)
6665                 goto out_free;
6666
6667         if (get_named_trigger_data(trigger_data))
6668                 goto enable;
6669
6670         ret = create_actions(hist_data);
6671         if (ret)
6672                 goto out_unreg;
6673
6674         if (has_hist_vars(hist_data) || hist_data->n_var_refs) {
6675                 ret = save_hist_vars(hist_data);
6676                 if (ret)
6677                         goto out_unreg;
6678         }
6679
6680         ret = tracing_map_init(hist_data->map);
6681         if (ret)
6682                 goto out_unreg;
6683 enable:
6684         ret = hist_trigger_enable(trigger_data, file);
6685         if (ret)
6686                 goto out_unreg;
6687
6688         se_name = trace_event_name(file->event_call);
6689         se = find_synth_event(se_name);
6690         if (se)
6691                 se->ref++;
6692  out:
6693         if (ret == 0 && glob[0])
6694                 hist_err_clear();
6695
6696         return ret;
6697  out_unreg:
6698         event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6699  out_free:
6700         event_trigger_reset_filter(cmd_ops, trigger_data);
6701
6702         remove_hist_vars(hist_data);
6703
6704         kfree(trigger_data);
6705
6706         destroy_hist_data(hist_data);
6707         goto out;
6708 }
6709
6710 static struct event_command trigger_hist_cmd = {
6711         .name                   = "hist",
6712         .trigger_type           = ETT_EVENT_HIST,
6713         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6714         .parse                  = event_hist_trigger_parse,
6715         .reg                    = hist_register_trigger,
6716         .unreg                  = hist_unregister_trigger,
6717         .unreg_all              = hist_unreg_all,
6718         .get_trigger_ops        = event_hist_get_trigger_ops,
6719         .set_filter             = set_trigger_filter,
6720 };
6721
6722 __init int register_trigger_hist_cmd(void)
6723 {
6724         int ret;
6725
6726         ret = register_event_command(&trigger_hist_cmd);
6727         WARN_ON(ret < 0);
6728
6729         return ret;
6730 }
6731
6732 static void
6733 hist_enable_trigger(struct event_trigger_data *data,
6734                     struct trace_buffer *buffer,  void *rec,
6735                     struct ring_buffer_event *event)
6736 {
6737         struct enable_trigger_data *enable_data = data->private_data;
6738         struct event_trigger_data *test;
6739
6740         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6741                                 lockdep_is_held(&event_mutex)) {
6742                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6743                         if (enable_data->enable)
6744                                 test->paused = false;
6745                         else
6746                                 test->paused = true;
6747                 }
6748         }
6749 }
6750
6751 static void
6752 hist_enable_count_trigger(struct event_trigger_data *data,
6753                           struct trace_buffer *buffer,  void *rec,
6754                           struct ring_buffer_event *event)
6755 {
6756         if (!data->count)
6757                 return;
6758
6759         if (data->count != -1)
6760                 (data->count)--;
6761
6762         hist_enable_trigger(data, buffer, rec, event);
6763 }
6764
6765 static struct event_trigger_ops hist_enable_trigger_ops = {
6766         .trigger                = hist_enable_trigger,
6767         .print                  = event_enable_trigger_print,
6768         .init                   = event_trigger_init,
6769         .free                   = event_enable_trigger_free,
6770 };
6771
6772 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6773         .trigger                = hist_enable_count_trigger,
6774         .print                  = event_enable_trigger_print,
6775         .init                   = event_trigger_init,
6776         .free                   = event_enable_trigger_free,
6777 };
6778
6779 static struct event_trigger_ops hist_disable_trigger_ops = {
6780         .trigger                = hist_enable_trigger,
6781         .print                  = event_enable_trigger_print,
6782         .init                   = event_trigger_init,
6783         .free                   = event_enable_trigger_free,
6784 };
6785
6786 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6787         .trigger                = hist_enable_count_trigger,
6788         .print                  = event_enable_trigger_print,
6789         .init                   = event_trigger_init,
6790         .free                   = event_enable_trigger_free,
6791 };
6792
6793 static struct event_trigger_ops *
6794 hist_enable_get_trigger_ops(char *cmd, char *param)
6795 {
6796         struct event_trigger_ops *ops;
6797         bool enable;
6798
6799         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6800
6801         if (enable)
6802                 ops = param ? &hist_enable_count_trigger_ops :
6803                         &hist_enable_trigger_ops;
6804         else
6805                 ops = param ? &hist_disable_count_trigger_ops :
6806                         &hist_disable_trigger_ops;
6807
6808         return ops;
6809 }
6810
6811 static void hist_enable_unreg_all(struct trace_event_file *file)
6812 {
6813         struct event_trigger_data *test, *n;
6814
6815         list_for_each_entry_safe(test, n, &file->triggers, list) {
6816                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6817                         list_del_rcu(&test->list);
6818                         update_cond_flag(file);
6819                         trace_event_trigger_enable_disable(file, 0);
6820                         if (test->ops->free)
6821                                 test->ops->free(test);
6822                 }
6823         }
6824 }
6825
6826 static struct event_command trigger_hist_enable_cmd = {
6827         .name                   = ENABLE_HIST_STR,
6828         .trigger_type           = ETT_HIST_ENABLE,
6829         .parse                  = event_enable_trigger_parse,
6830         .reg                    = event_enable_register_trigger,
6831         .unreg                  = event_enable_unregister_trigger,
6832         .unreg_all              = hist_enable_unreg_all,
6833         .get_trigger_ops        = hist_enable_get_trigger_ops,
6834         .set_filter             = set_trigger_filter,
6835 };
6836
6837 static struct event_command trigger_hist_disable_cmd = {
6838         .name                   = DISABLE_HIST_STR,
6839         .trigger_type           = ETT_HIST_ENABLE,
6840         .parse                  = event_enable_trigger_parse,
6841         .reg                    = event_enable_register_trigger,
6842         .unreg                  = event_enable_unregister_trigger,
6843         .unreg_all              = hist_enable_unreg_all,
6844         .get_trigger_ops        = hist_enable_get_trigger_ops,
6845         .set_filter             = set_trigger_filter,
6846 };
6847
6848 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6849 {
6850         unregister_event_command(&trigger_hist_enable_cmd);
6851         unregister_event_command(&trigger_hist_disable_cmd);
6852 }
6853
6854 __init int register_trigger_hist_enable_disable_cmds(void)
6855 {
6856         int ret;
6857
6858         ret = register_event_command(&trigger_hist_enable_cmd);
6859         if (WARN_ON(ret < 0))
6860                 return ret;
6861         ret = register_event_command(&trigger_hist_disable_cmd);
6862         if (WARN_ON(ret < 0))
6863                 unregister_trigger_hist_enable_disable_cmds();
6864
6865         return ret;
6866 }