kernel/trace/trace_events_hist.c

   1 /*
   2  * trace_events_hist - trace event hist triggers
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
  15  */
  16
  17 #include <linux/module.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/mutex.h>
  20 #include <linux/slab.h>
  21 #include <linux/stacktrace.h>
  22 #include <linux/rculist.h>
  23
  24 #include "tracing_map.h"
  25 #include "trace.h"
  26
  27 struct hist_field;
  28
  29 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  30                                 struct tracing_map_elt *elt,
  31                                 struct ring_buffer_event *rbe,
  32                                 void *event);
  33
  34 #define HIST_FIELD_OPERANDS_MAX 2
  35 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  36
  37 enum field_op_id {
  38         FIELD_OP_NONE,
  39         FIELD_OP_PLUS,
  40         FIELD_OP_MINUS,
  41         FIELD_OP_UNARY_MINUS,
  42 };
  43
  44 struct hist_var {
  45         char                            *name;
  46         struct hist_trigger_data        *hist_data;
  47         unsigned int                    idx;
  48 };
  49
  50 struct hist_field {
  51         struct ftrace_event_field       *field;
  52         unsigned long                   flags;
  53         hist_field_fn_t                 fn;
  54         unsigned int                    size;
  55         unsigned int                    offset;
  56         unsigned int                    is_signed;
  57         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
  58         struct hist_trigger_data        *hist_data;
  59         struct hist_var                 var;
  60         enum field_op_id                operator;
  61         char                            *name;
  62 };
  63
  64 static u64 hist_field_none(struct hist_field *field,
  65                            struct tracing_map_elt *elt,
  66                            struct ring_buffer_event *rbe,
  67                            void *event)
  68 {
  69         return 0;
  70 }
  71
  72 static u64 hist_field_counter(struct hist_field *field,
  73                               struct tracing_map_elt *elt,
  74                               struct ring_buffer_event *rbe,
  75                               void *event)
  76 {
  77         return 1;
  78 }
  79
  80 static u64 hist_field_string(struct hist_field *hist_field,
  81                              struct tracing_map_elt *elt,
  82                              struct ring_buffer_event *rbe,
  83                              void *event)
  84 {
  85         char *addr = (char *)(event + hist_field->field->offset);
  86
  87         return (u64)(unsigned long)addr;
  88 }
  89
  90 static u64 hist_field_dynstring(struct hist_field *hist_field,
  91                                 struct tracing_map_elt *elt,
  92                                 struct ring_buffer_event *rbe,
  93                                 void *event)
  94 {
  95         u32 str_item = *(u32 *)(event + hist_field->field->offset);
  96         int str_loc = str_item & 0xffff;
  97         char *addr = (char *)(event + str_loc);
  98
  99         return (u64)(unsigned long)addr;
 100 }
 101
 102 static u64 hist_field_pstring(struct hist_field *hist_field,
 103                               struct tracing_map_elt *elt,
 104                               struct ring_buffer_event *rbe,
 105                               void *event)
 106 {
 107         char **addr = (char **)(event + hist_field->field->offset);
 108
 109         return (u64)(unsigned long)*addr;
 110 }
 111
 112 static u64 hist_field_log2(struct hist_field *hist_field,
 113                            struct tracing_map_elt *elt,
 114                            struct ring_buffer_event *rbe,
 115                            void *event)
 116 {
 117         struct hist_field *operand = hist_field->operands[0];
 118
 119         u64 val = operand->fn(operand, elt, rbe, event);
 120
 121         return (u64) ilog2(roundup_pow_of_two(val));
 122 }
 123
 124 static u64 hist_field_plus(struct hist_field *hist_field,
 125                            struct tracing_map_elt *elt,
 126                            struct ring_buffer_event *rbe,
 127                            void *event)
 128 {
 129         struct hist_field *operand1 = hist_field->operands[0];
 130         struct hist_field *operand2 = hist_field->operands[1];
 131
 132         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 133         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 134
 135         return val1 + val2;
 136 }
 137
 138 static u64 hist_field_minus(struct hist_field *hist_field,
 139                             struct tracing_map_elt *elt,
 140                             struct ring_buffer_event *rbe,
 141                             void *event)
 142 {
 143         struct hist_field *operand1 = hist_field->operands[0];
 144         struct hist_field *operand2 = hist_field->operands[1];
 145
 146         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 147         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 148
 149         return val1 - val2;
 150 }
 151
 152 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 153                                   struct tracing_map_elt *elt,
 154                                   struct ring_buffer_event *rbe,
 155                                   void *event)
 156 {
 157         struct hist_field *operand = hist_field->operands[0];
 158
 159         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 160         u64 val = (u64)-sval;
 161
 162         return val;
 163 }
 164
 165 #define DEFINE_HIST_FIELD_FN(type)                                      \
 166         static u64 hist_field_##type(struct hist_field *hist_field,     \
 167                                      struct tracing_map_elt *elt,       \
 168                                      struct ring_buffer_event *rbe,     \
 169                                      void *event)                       \
 170 {                                                                       \
 171         type *addr = (type *)(event + hist_field->field->offset);       \
 172                                                                         \
 173         return (u64)(unsigned long)*addr;                               \
 174 }
 175
 176 DEFINE_HIST_FIELD_FN(s64);
 177 DEFINE_HIST_FIELD_FN(u64);
 178 DEFINE_HIST_FIELD_FN(s32);
 179 DEFINE_HIST_FIELD_FN(u32);
 180 DEFINE_HIST_FIELD_FN(s16);
 181 DEFINE_HIST_FIELD_FN(u16);
 182 DEFINE_HIST_FIELD_FN(s8);
 183 DEFINE_HIST_FIELD_FN(u8);
 184
 185 #define for_each_hist_field(i, hist_data)       \
 186         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 187
 188 #define for_each_hist_val_field(i, hist_data)   \
 189         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 190
 191 #define for_each_hist_key_field(i, hist_data)   \
 192         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 193
 194 #define HIST_STACKTRACE_DEPTH   16
 195 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 196 #define HIST_STACKTRACE_SKIP    5
 197
 198 #define HITCOUNT_IDX            0
 199 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 200
 201 enum hist_field_flags {
 202         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 203         HIST_FIELD_FL_KEY               = 1 << 1,
 204         HIST_FIELD_FL_STRING            = 1 << 2,
 205         HIST_FIELD_FL_HEX               = 1 << 3,
 206         HIST_FIELD_FL_SYM               = 1 << 4,
 207         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 208         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 209         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 210         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 211         HIST_FIELD_FL_LOG2              = 1 << 9,
 212         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 213         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 214         HIST_FIELD_FL_VAR               = 1 << 12,
 215         HIST_FIELD_FL_EXPR              = 1 << 13,
 216 };
 217
 218 struct var_defs {
 219         unsigned int    n_vars;
 220         char            *name[TRACING_MAP_VARS_MAX];
 221         char            *expr[TRACING_MAP_VARS_MAX];
 222 };
 223
 224 struct hist_trigger_attrs {
 225         char            *keys_str;
 226         char            *vals_str;
 227         char            *sort_key_str;
 228         char            *name;
 229         bool            pause;
 230         bool            cont;
 231         bool            clear;
 232         bool            ts_in_usecs;
 233         unsigned int    map_bits;
 234
 235         char            *assignment_str[TRACING_MAP_VARS_MAX];
 236         unsigned int    n_assignments;
 237
 238         struct var_defs var_defs;
 239 };
 240
 241 struct hist_trigger_data {
 242         struct hist_field               *fields[HIST_FIELDS_MAX];
 243         unsigned int                    n_vals;
 244         unsigned int                    n_keys;
 245         unsigned int                    n_fields;
 246         unsigned int                    n_vars;
 247         unsigned int                    key_size;
 248         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 249         unsigned int                    n_sort_keys;
 250         struct trace_event_file         *event_file;
 251         struct hist_trigger_attrs       *attrs;
 252         struct tracing_map              *map;
 253         bool                            enable_timestamps;
 254         bool                            remove;
 255 };
 256
 257 static u64 hist_field_timestamp(struct hist_field *hist_field,
 258                                 struct tracing_map_elt *elt,
 259                                 struct ring_buffer_event *rbe,
 260                                 void *event)
 261 {
 262         struct hist_trigger_data *hist_data = hist_field->hist_data;
 263         struct trace_array *tr = hist_data->event_file->tr;
 264
 265         u64 ts = ring_buffer_event_time_stamp(rbe);
 266
 267         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 268                 ts = ns2usecs(ts);
 269
 270         return ts;
 271 }
 272
 273 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 274                                          const char *var_name)
 275 {
 276         struct hist_field *hist_field, *found = NULL;
 277         int i;
 278
 279         for_each_hist_field(i, hist_data) {
 280                 hist_field = hist_data->fields[i];
 281                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 282                     strcmp(hist_field->var.name, var_name) == 0) {
 283                         found = hist_field;
 284                         break;
 285                 }
 286         }
 287
 288         return found;
 289 }
 290
 291 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 292                                    struct trace_event_file *file,
 293                                    const char *var_name)
 294 {
 295         struct hist_trigger_data *test_data;
 296         struct event_trigger_data *test;
 297         struct hist_field *hist_field;
 298
 299         hist_field = find_var_field(hist_data, var_name);
 300         if (hist_field)
 301                 return hist_field;
 302
 303         list_for_each_entry_rcu(test, &file->triggers, list) {
 304                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 305                         test_data = test->private_data;
 306                         hist_field = find_var_field(test_data, var_name);
 307                         if (hist_field)
 308                                 return hist_field;
 309                 }
 310         }
 311
 312         return NULL;
 313 }
 314
 315 struct hist_elt_data {
 316         char *comm;
 317 };
 318
 319 static const char *hist_field_name(struct hist_field *field,
 320                                    unsigned int level)
 321 {
 322         const char *field_name = "";
 323
 324         if (level > 1)
 325                 return field_name;
 326
 327         if (field->field)
 328                 field_name = field->field->name;
 329         else if (field->flags & HIST_FIELD_FL_LOG2)
 330                 field_name = hist_field_name(field->operands[0], ++level);
 331         else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
 332                 field_name = "common_timestamp";
 333         else if (field->flags & HIST_FIELD_FL_EXPR)
 334                 field_name = field->name;
 335
 336         if (field_name == NULL)
 337                 field_name = "";
 338
 339         return field_name;
 340 }
 341
 342 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
 343 {
 344         hist_field_fn_t fn = NULL;
 345
 346         switch (field_size) {
 347         case 8:
 348                 if (field_is_signed)
 349                         fn = hist_field_s64;
 350                 else
 351                         fn = hist_field_u64;
 352                 break;
 353         case 4:
 354                 if (field_is_signed)
 355                         fn = hist_field_s32;
 356                 else
 357                         fn = hist_field_u32;
 358                 break;
 359         case 2:
 360                 if (field_is_signed)
 361                         fn = hist_field_s16;
 362                 else
 363                         fn = hist_field_u16;
 364                 break;
 365         case 1:
 366                 if (field_is_signed)
 367                         fn = hist_field_s8;
 368                 else
 369                         fn = hist_field_u8;
 370                 break;
 371         }
 372
 373         return fn;
 374 }
 375
 376 static int parse_map_size(char *str)
 377 {
 378         unsigned long size, map_bits;
 379         int ret;
 380
 381         strsep(&str, "=");
 382         if (!str) {
 383                 ret = -EINVAL;
 384                 goto out;
 385         }
 386
 387         ret = kstrtoul(str, 0, &size);
 388         if (ret)
 389                 goto out;
 390
 391         map_bits = ilog2(roundup_pow_of_two(size));
 392         if (map_bits < TRACING_MAP_BITS_MIN ||
 393             map_bits > TRACING_MAP_BITS_MAX)
 394                 ret = -EINVAL;
 395         else
 396                 ret = map_bits;
 397  out:
 398         return ret;
 399 }
 400
 401 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
 402 {
 403         unsigned int i;
 404
 405         if (!attrs)
 406                 return;
 407
 408         for (i = 0; i < attrs->n_assignments; i++)
 409                 kfree(attrs->assignment_str[i]);
 410
 411         kfree(attrs->name);
 412         kfree(attrs->sort_key_str);
 413         kfree(attrs->keys_str);
 414         kfree(attrs->vals_str);
 415         kfree(attrs);
 416 }
 417
 418 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
 419 {
 420         int ret = 0;
 421
 422         if ((strncmp(str, "key=", strlen("key=")) == 0) ||
 423             (strncmp(str, "keys=", strlen("keys=")) == 0)) {
 424                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
 425                 if (!attrs->keys_str) {
 426                         ret = -ENOMEM;
 427                         goto out;
 428                 }
 429         } else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
 430                  (strncmp(str, "vals=", strlen("vals=")) == 0) ||
 431                  (strncmp(str, "values=", strlen("values=")) == 0)) {
 432                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
 433                 if (!attrs->vals_str) {
 434                         ret = -ENOMEM;
 435                         goto out;
 436                 }
 437         } else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
 438                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
 439                 if (!attrs->sort_key_str) {
 440                         ret = -ENOMEM;
 441                         goto out;
 442                 }
 443         } else if (strncmp(str, "name=", strlen("name=")) == 0) {
 444                 attrs->name = kstrdup(str, GFP_KERNEL);
 445                 if (!attrs->name) {
 446                         ret = -ENOMEM;
 447                         goto out;
 448                 }
 449         } else if (strncmp(str, "size=", strlen("size=")) == 0) {
 450                 int map_bits = parse_map_size(str);
 451
 452                 if (map_bits < 0) {
 453                         ret = map_bits;
 454                         goto out;
 455                 }
 456                 attrs->map_bits = map_bits;
 457         } else {
 458                 char *assignment;
 459
 460                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
 461                         ret = -EINVAL;
 462                         goto out;
 463                 }
 464
 465                 assignment = kstrdup(str, GFP_KERNEL);
 466                 if (!assignment) {
 467                         ret = -ENOMEM;
 468                         goto out;
 469                 }
 470
 471                 attrs->assignment_str[attrs->n_assignments++] = assignment;
 472         }
 473  out:
 474         return ret;
 475 }
 476
 477 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
 478 {
 479         struct hist_trigger_attrs *attrs;
 480         int ret = 0;
 481
 482         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
 483         if (!attrs)
 484                 return ERR_PTR(-ENOMEM);
 485
 486         while (trigger_str) {
 487                 char *str = strsep(&trigger_str, ":");
 488
 489                 if (strchr(str, '=')) {
 490                         ret = parse_assignment(str, attrs);
 491                         if (ret)
 492                                 goto free;
 493                 } else if (strcmp(str, "pause") == 0)
 494                         attrs->pause = true;
 495                 else if ((strcmp(str, "cont") == 0) ||
 496                          (strcmp(str, "continue") == 0))
 497                         attrs->cont = true;
 498                 else if (strcmp(str, "clear") == 0)
 499                         attrs->clear = true;
 500                 else {
 501                         ret = -EINVAL;
 502                         goto free;
 503                 }
 504         }
 505
 506         if (!attrs->keys_str) {
 507                 ret = -EINVAL;
 508                 goto free;
 509         }
 510
 511         return attrs;
 512  free:
 513         destroy_hist_trigger_attrs(attrs);
 514
 515         return ERR_PTR(ret);
 516 }
 517
 518 static inline void save_comm(char *comm, struct task_struct *task)
 519 {
 520         if (!task->pid) {
 521                 strcpy(comm, "<idle>");
 522                 return;
 523         }
 524
 525         if (WARN_ON_ONCE(task->pid < 0)) {
 526                 strcpy(comm, "<XXX>");
 527                 return;
 528         }
 529
 530         memcpy(comm, task->comm, TASK_COMM_LEN);
 531 }
 532
 533 static void hist_elt_data_free(struct hist_elt_data *elt_data)
 534 {
 535         kfree(elt_data->comm);
 536         kfree(elt_data);
 537 }
 538
 539 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
 540 {
 541         struct hist_elt_data *elt_data = elt->private_data;
 542
 543         hist_elt_data_free(elt_data);
 544 }
 545
 546 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
 547 {
 548         struct hist_trigger_data *hist_data = elt->map->private_data;
 549         unsigned int size = TASK_COMM_LEN;
 550         struct hist_elt_data *elt_data;
 551         struct hist_field *key_field;
 552         unsigned int i;
 553
 554         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 555         if (!elt_data)
 556                 return -ENOMEM;
 557
 558         for_each_hist_key_field(i, hist_data) {
 559                 key_field = hist_data->fields[i];
 560
 561                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
 562                         elt_data->comm = kzalloc(size, GFP_KERNEL);
 563                         if (!elt_data->comm) {
 564                                 kfree(elt_data);
 565                                 return -ENOMEM;
 566                         }
 567                         break;
 568                 }
 569         }
 570
 571         elt->private_data = elt_data;
 572
 573         return 0;
 574 }
 575
 576 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
 577 {
 578         struct hist_elt_data *elt_data = elt->private_data;
 579
 580         if (elt_data->comm)
 581                 save_comm(elt_data->comm, current);
 582 }
 583
 584 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
 585         .elt_alloc      = hist_trigger_elt_data_alloc,
 586         .elt_free       = hist_trigger_elt_data_free,
 587         .elt_init       = hist_trigger_elt_data_init,
 588 };
 589
 590 static const char *get_hist_field_flags(struct hist_field *hist_field)
 591 {
 592         const char *flags_str = NULL;
 593
 594         if (hist_field->flags & HIST_FIELD_FL_HEX)
 595                 flags_str = "hex";
 596         else if (hist_field->flags & HIST_FIELD_FL_SYM)
 597                 flags_str = "sym";
 598         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
 599                 flags_str = "sym-offset";
 600         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
 601                 flags_str = "execname";
 602         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
 603                 flags_str = "syscall";
 604         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
 605                 flags_str = "log2";
 606         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
 607                 flags_str = "usecs";
 608
 609         return flags_str;
 610 }
 611
 612 static void expr_field_str(struct hist_field *field, char *expr)
 613 {
 614         strcat(expr, hist_field_name(field, 0));
 615
 616         if (field->flags) {
 617                 const char *flags_str = get_hist_field_flags(field);
 618
 619                 if (flags_str) {
 620                         strcat(expr, ".");
 621                         strcat(expr, flags_str);
 622                 }
 623         }
 624 }
 625
 626 static char *expr_str(struct hist_field *field, unsigned int level)
 627 {
 628         char *expr;
 629
 630         if (level > 1)
 631                 return NULL;
 632
 633         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
 634         if (!expr)
 635                 return NULL;
 636
 637         if (!field->operands[0]) {
 638                 expr_field_str(field, expr);
 639                 return expr;
 640         }
 641
 642         if (field->operator == FIELD_OP_UNARY_MINUS) {
 643                 char *subexpr;
 644
 645                 strcat(expr, "-(");
 646                 subexpr = expr_str(field->operands[0], ++level);
 647                 if (!subexpr) {
 648                         kfree(expr);
 649                         return NULL;
 650                 }
 651                 strcat(expr, subexpr);
 652                 strcat(expr, ")");
 653
 654                 kfree(subexpr);
 655
 656                 return expr;
 657         }
 658
 659         expr_field_str(field->operands[0], expr);
 660
 661         switch (field->operator) {
 662         case FIELD_OP_MINUS:
 663                 strcat(expr, "-");
 664                 break;
 665         case FIELD_OP_PLUS:
 666                 strcat(expr, "+");
 667                 break;
 668         default:
 669                 kfree(expr);
 670                 return NULL;
 671         }
 672
 673         expr_field_str(field->operands[1], expr);
 674
 675         return expr;
 676 }
 677
 678 static int contains_operator(char *str)
 679 {
 680         enum field_op_id field_op = FIELD_OP_NONE;
 681         char *op;
 682
 683         op = strpbrk(str, "+-");
 684         if (!op)
 685                 return FIELD_OP_NONE;
 686
 687         switch (*op) {
 688         case '-':
 689                 if (*str == '-')
 690                         field_op = FIELD_OP_UNARY_MINUS;
 691                 else
 692                         field_op = FIELD_OP_MINUS;
 693                 break;
 694         case '+':
 695                 field_op = FIELD_OP_PLUS;
 696                 break;
 697         default:
 698                 break;
 699         }
 700
 701         return field_op;
 702 }
 703
 704 static void destroy_hist_field(struct hist_field *hist_field,
 705                                unsigned int level)
 706 {
 707         unsigned int i;
 708
 709         if (level > 3)
 710                 return;
 711
 712         if (!hist_field)
 713                 return;
 714
 715         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
 716                 destroy_hist_field(hist_field->operands[i], level + 1);
 717
 718         kfree(hist_field->var.name);
 719         kfree(hist_field->name);
 720
 721         kfree(hist_field);
 722 }
 723
 724 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
 725                                             struct ftrace_event_field *field,
 726                                             unsigned long flags,
 727                                             char *var_name)
 728 {
 729         struct hist_field *hist_field;
 730
 731         if (field && is_function_field(field))
 732                 return NULL;
 733
 734         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
 735         if (!hist_field)
 736                 return NULL;
 737
 738         hist_field->hist_data = hist_data;
 739
 740         if (flags & HIST_FIELD_FL_EXPR)
 741                 goto out; /* caller will populate */
 742
 743         if (flags & HIST_FIELD_FL_HITCOUNT) {
 744                 hist_field->fn = hist_field_counter;
 745                 goto out;
 746         }
 747
 748         if (flags & HIST_FIELD_FL_STACKTRACE) {
 749                 hist_field->fn = hist_field_none;
 750                 goto out;
 751         }
 752
 753         if (flags & HIST_FIELD_FL_LOG2) {
 754                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
 755                 hist_field->fn = hist_field_log2;
 756                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
 757                 hist_field->size = hist_field->operands[0]->size;
 758                 goto out;
 759         }
 760
 761         if (flags & HIST_FIELD_FL_TIMESTAMP) {
 762                 hist_field->fn = hist_field_timestamp;
 763                 hist_field->size = sizeof(u64);
 764                 goto out;
 765         }
 766
 767         if (WARN_ON_ONCE(!field))
 768                 goto out;
 769
 770         if (is_string_field(field)) {
 771                 flags |= HIST_FIELD_FL_STRING;
 772
 773                 if (field->filter_type == FILTER_STATIC_STRING)
 774                         hist_field->fn = hist_field_string;
 775                 else if (field->filter_type == FILTER_DYN_STRING)
 776                         hist_field->fn = hist_field_dynstring;
 777                 else
 778                         hist_field->fn = hist_field_pstring;
 779         } else {
 780                 hist_field->fn = select_value_fn(field->size,
 781                                                  field->is_signed);
 782                 if (!hist_field->fn) {
 783                         destroy_hist_field(hist_field, 0);
 784                         return NULL;
 785                 }
 786         }
 787  out:
 788         hist_field->field = field;
 789         hist_field->flags = flags;
 790
 791         if (var_name) {
 792                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
 793                 if (!hist_field->var.name)
 794                         goto free;
 795         }
 796
 797         return hist_field;
 798  free:
 799         destroy_hist_field(hist_field, 0);
 800         return NULL;
 801 }
 802
 803 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
 804 {
 805         unsigned int i;
 806
 807         for (i = 0; i < HIST_FIELDS_MAX; i++) {
 808                 if (hist_data->fields[i]) {
 809                         destroy_hist_field(hist_data->fields[i], 0);
 810                         hist_data->fields[i] = NULL;
 811                 }
 812         }
 813 }
 814
 815 static struct ftrace_event_field *
 816 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
 817             char *field_str, unsigned long *flags)
 818 {
 819         struct ftrace_event_field *field = NULL;
 820         char *field_name, *modifier, *str;
 821
 822         modifier = str = kstrdup(field_str, GFP_KERNEL);
 823         if (!modifier)
 824                 return ERR_PTR(-ENOMEM);
 825
 826         field_name = strsep(&modifier, ".");
 827         if (modifier) {
 828                 if (strcmp(modifier, "hex") == 0)
 829                         *flags |= HIST_FIELD_FL_HEX;
 830                 else if (strcmp(modifier, "sym") == 0)
 831                         *flags |= HIST_FIELD_FL_SYM;
 832                 else if (strcmp(modifier, "sym-offset") == 0)
 833                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
 834                 else if ((strcmp(modifier, "execname") == 0) &&
 835                          (strcmp(field_name, "common_pid") == 0))
 836                         *flags |= HIST_FIELD_FL_EXECNAME;
 837                 else if (strcmp(modifier, "syscall") == 0)
 838                         *flags |= HIST_FIELD_FL_SYSCALL;
 839                 else if (strcmp(modifier, "log2") == 0)
 840                         *flags |= HIST_FIELD_FL_LOG2;
 841                 else if (strcmp(modifier, "usecs") == 0)
 842                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
 843                 else {
 844                         field = ERR_PTR(-EINVAL);
 845                         goto out;
 846                 }
 847         }
 848
 849         if (strcmp(field_name, "common_timestamp") == 0) {
 850                 *flags |= HIST_FIELD_FL_TIMESTAMP;
 851                 hist_data->enable_timestamps = true;
 852                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
 853                         hist_data->attrs->ts_in_usecs = true;
 854         } else {
 855                 field = trace_find_event_field(file->event_call, field_name);
 856                 if (!field || !field->size) {
 857                         field = ERR_PTR(-EINVAL);
 858                         goto out;
 859                 }
 860         }
 861  out:
 862         kfree(str);
 863
 864         return field;
 865 }
 866
 867 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
 868                                      struct trace_event_file *file, char *str,
 869                                      unsigned long *flags, char *var_name)
 870 {
 871         struct ftrace_event_field *field = NULL;
 872         struct hist_field *hist_field = NULL;
 873         int ret = 0;
 874
 875         field = parse_field(hist_data, file, str, flags);
 876         if (IS_ERR(field)) {
 877                 ret = PTR_ERR(field);
 878                 goto out;
 879         }
 880
 881         hist_field = create_hist_field(hist_data, field, *flags, var_name);
 882         if (!hist_field) {
 883                 ret = -ENOMEM;
 884                 goto out;
 885         }
 886
 887         return hist_field;
 888  out:
 889         return ERR_PTR(ret);
 890 }
 891
 892 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
 893                                      struct trace_event_file *file,
 894                                      char *str, unsigned long flags,
 895                                      char *var_name, unsigned int level);
 896
 897 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
 898                                       struct trace_event_file *file,
 899                                       char *str, unsigned long flags,
 900                                       char *var_name, unsigned int level)
 901 {
 902         struct hist_field *operand1, *expr = NULL;
 903         unsigned long operand_flags;
 904         int ret = 0;
 905         char *s;
 906
 907         /* we support only -(xxx) i.e. explicit parens required */
 908
 909         if (level > 3) {
 910                 ret = -EINVAL;
 911                 goto free;
 912         }
 913
 914         str++; /* skip leading '-' */
 915
 916         s = strchr(str, '(');
 917         if (s)
 918                 str++;
 919         else {
 920                 ret = -EINVAL;
 921                 goto free;
 922         }
 923
 924         s = strrchr(str, ')');
 925         if (s)
 926                 *s = '\0';
 927         else {
 928                 ret = -EINVAL; /* no closing ')' */
 929                 goto free;
 930         }
 931
 932         flags |= HIST_FIELD_FL_EXPR;
 933         expr = create_hist_field(hist_data, NULL, flags, var_name);
 934         if (!expr) {
 935                 ret = -ENOMEM;
 936                 goto free;
 937         }
 938
 939         operand_flags = 0;
 940         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
 941         if (IS_ERR(operand1)) {
 942                 ret = PTR_ERR(operand1);
 943                 goto free;
 944         }
 945
 946         expr->flags |= operand1->flags &
 947                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
 948         expr->fn = hist_field_unary_minus;
 949         expr->operands[0] = operand1;
 950         expr->operator = FIELD_OP_UNARY_MINUS;
 951         expr->name = expr_str(expr, 0);
 952
 953         return expr;
 954  free:
 955         destroy_hist_field(expr, 0);
 956         return ERR_PTR(ret);
 957 }
 958
 959 static int check_expr_operands(struct hist_field *operand1,
 960                                struct hist_field *operand2)
 961 {
 962         unsigned long operand1_flags = operand1->flags;
 963         unsigned long operand2_flags = operand2->flags;
 964
 965         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
 966             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS))
 967                 return -EINVAL;
 968
 969         return 0;
 970 }
 971
 972 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
 973                                      struct trace_event_file *file,
 974                                      char *str, unsigned long flags,
 975                                      char *var_name, unsigned int level)
 976 {
 977         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
 978         unsigned long operand_flags;
 979         int field_op, ret = -EINVAL;
 980         char *sep, *operand1_str;
 981
 982         if (level > 3)
 983                 return ERR_PTR(-EINVAL);
 984
 985         field_op = contains_operator(str);
 986
 987         if (field_op == FIELD_OP_NONE)
 988                 return parse_atom(hist_data, file, str, &flags, var_name);
 989
 990         if (field_op == FIELD_OP_UNARY_MINUS)
 991                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
 992
 993         switch (field_op) {
 994         case FIELD_OP_MINUS:
 995                 sep = "-";
 996                 break;
 997         case FIELD_OP_PLUS:
 998                 sep = "+";
 999                 break;
1000         default:
1001                 goto free;
1002         }
1003
1004         operand1_str = strsep(&str, sep);
1005         if (!operand1_str || !str)
1006                 goto free;
1007
1008         operand_flags = 0;
1009         operand1 = parse_atom(hist_data, file, operand1_str,
1010                               &operand_flags, NULL);
1011         if (IS_ERR(operand1)) {
1012                 ret = PTR_ERR(operand1);
1013                 operand1 = NULL;
1014                 goto free;
1015         }
1016
1017         /* rest of string could be another expression e.g. b+c in a+b+c */
1018         operand_flags = 0;
1019         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
1020         if (IS_ERR(operand2)) {
1021                 ret = PTR_ERR(operand2);
1022                 operand2 = NULL;
1023                 goto free;
1024         }
1025
1026         ret = check_expr_operands(operand1, operand2);
1027         if (ret)
1028                 goto free;
1029
1030         flags |= HIST_FIELD_FL_EXPR;
1031
1032         flags |= operand1->flags &
1033                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1034
1035         expr = create_hist_field(hist_data, NULL, flags, var_name);
1036         if (!expr) {
1037                 ret = -ENOMEM;
1038                 goto free;
1039         }
1040
1041         expr->operands[0] = operand1;
1042         expr->operands[1] = operand2;
1043         expr->operator = field_op;
1044         expr->name = expr_str(expr, 0);
1045
1046         switch (field_op) {
1047         case FIELD_OP_MINUS:
1048                 expr->fn = hist_field_minus;
1049                 break;
1050         case FIELD_OP_PLUS:
1051                 expr->fn = hist_field_plus;
1052                 break;
1053         default:
1054                 goto free;
1055         }
1056
1057         return expr;
1058  free:
1059         destroy_hist_field(operand1, 0);
1060         destroy_hist_field(operand2, 0);
1061         destroy_hist_field(expr, 0);
1062
1063         return ERR_PTR(ret);
1064 }
1065
1066 static int create_hitcount_val(struct hist_trigger_data *hist_data)
1067 {
1068         hist_data->fields[HITCOUNT_IDX] =
1069                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
1070         if (!hist_data->fields[HITCOUNT_IDX])
1071                 return -ENOMEM;
1072
1073         hist_data->n_vals++;
1074         hist_data->n_fields++;
1075
1076         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
1077                 return -EINVAL;
1078
1079         return 0;
1080 }
1081
1082 static int __create_val_field(struct hist_trigger_data *hist_data,
1083                               unsigned int val_idx,
1084                               struct trace_event_file *file,
1085                               char *var_name, char *field_str,
1086                               unsigned long flags)
1087 {
1088         struct hist_field *hist_field;
1089         int ret = 0;
1090
1091         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
1092         if (IS_ERR(hist_field)) {
1093                 ret = PTR_ERR(hist_field);
1094                 goto out;
1095         }
1096
1097         hist_data->fields[val_idx] = hist_field;
1098
1099         ++hist_data->n_vals;
1100         ++hist_data->n_fields;
1101
1102         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
1103                 ret = -EINVAL;
1104  out:
1105         return ret;
1106 }
1107
1108 static int create_val_field(struct hist_trigger_data *hist_data,
1109                             unsigned int val_idx,
1110                             struct trace_event_file *file,
1111                             char *field_str)
1112 {
1113         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
1114                 return -EINVAL;
1115
1116         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
1117 }
1118
1119 static int create_var_field(struct hist_trigger_data *hist_data,
1120                             unsigned int val_idx,
1121                             struct trace_event_file *file,
1122                             char *var_name, char *expr_str)
1123 {
1124         unsigned long flags = 0;
1125
1126         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
1127                 return -EINVAL;
1128         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
1129                 return -EINVAL;
1130         }
1131
1132         flags |= HIST_FIELD_FL_VAR;
1133         hist_data->n_vars++;
1134         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
1135                 return -EINVAL;
1136
1137         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
1138 }
1139
1140 static int create_val_fields(struct hist_trigger_data *hist_data,
1141                              struct trace_event_file *file)
1142 {
1143         char *fields_str, *field_str;
1144         unsigned int i, j = 1;
1145         int ret;
1146
1147         ret = create_hitcount_val(hist_data);
1148         if (ret)
1149                 goto out;
1150
1151         fields_str = hist_data->attrs->vals_str;
1152         if (!fields_str)
1153                 goto out;
1154
1155         strsep(&fields_str, "=");
1156         if (!fields_str)
1157                 goto out;
1158
1159         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
1160                      j < TRACING_MAP_VALS_MAX; i++) {
1161                 field_str = strsep(&fields_str, ",");
1162                 if (!field_str)
1163                         break;
1164
1165                 if (strcmp(field_str, "hitcount") == 0)
1166                         continue;
1167
1168                 ret = create_val_field(hist_data, j++, file, field_str);
1169                 if (ret)
1170                         goto out;
1171         }
1172
1173         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
1174                 ret = -EINVAL;
1175  out:
1176         return ret;
1177 }
1178
1179 static int create_key_field(struct hist_trigger_data *hist_data,
1180                             unsigned int key_idx,
1181                             unsigned int key_offset,
1182                             struct trace_event_file *file,
1183                             char *field_str)
1184 {
1185         struct hist_field *hist_field = NULL;
1186
1187         unsigned long flags = 0;
1188         unsigned int key_size;
1189         int ret = 0;
1190
1191         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
1192                 return -EINVAL;
1193
1194         flags |= HIST_FIELD_FL_KEY;
1195
1196         if (strcmp(field_str, "stacktrace") == 0) {
1197                 flags |= HIST_FIELD_FL_STACKTRACE;
1198                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
1199                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
1200         } else {
1201                 hist_field = parse_expr(hist_data, file, field_str, flags,
1202                                         NULL, 0);
1203                 if (IS_ERR(hist_field)) {
1204                         ret = PTR_ERR(hist_field);
1205                         goto out;
1206                 }
1207
1208                 key_size = hist_field->size;
1209         }
1210
1211         hist_data->fields[key_idx] = hist_field;
1212
1213         key_size = ALIGN(key_size, sizeof(u64));
1214         hist_data->fields[key_idx]->size = key_size;
1215         hist_data->fields[key_idx]->offset = key_offset;
1216
1217         hist_data->key_size += key_size;
1218
1219         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
1220                 ret = -EINVAL;
1221                 goto out;
1222         }
1223
1224         hist_data->n_keys++;
1225         hist_data->n_fields++;
1226
1227         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
1228                 return -EINVAL;
1229
1230         ret = key_size;
1231  out:
1232         return ret;
1233 }
1234
1235 static int create_key_fields(struct hist_trigger_data *hist_data,
1236                              struct trace_event_file *file)
1237 {
1238         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
1239         char *fields_str, *field_str;
1240         int ret = -EINVAL;
1241
1242         fields_str = hist_data->attrs->keys_str;
1243         if (!fields_str)
1244                 goto out;
1245
1246         strsep(&fields_str, "=");
1247         if (!fields_str)
1248                 goto out;
1249
1250         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
1251                 field_str = strsep(&fields_str, ",");
1252                 if (!field_str)
1253                         break;
1254                 ret = create_key_field(hist_data, i, key_offset,
1255                                        file, field_str);
1256                 if (ret < 0)
1257                         goto out;
1258                 key_offset += ret;
1259         }
1260         if (fields_str) {
1261                 ret = -EINVAL;
1262                 goto out;
1263         }
1264         ret = 0;
1265  out:
1266         return ret;
1267 }
1268
1269 static int create_var_fields(struct hist_trigger_data *hist_data,
1270                              struct trace_event_file *file)
1271 {
1272         unsigned int i, j = hist_data->n_vals;
1273         int ret = 0;
1274
1275         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
1276
1277         for (i = 0; i < n_vars; i++) {
1278                 char *var_name = hist_data->attrs->var_defs.name[i];
1279                 char *expr = hist_data->attrs->var_defs.expr[i];
1280
1281                 ret = create_var_field(hist_data, j++, file, var_name, expr);
1282                 if (ret)
1283                         goto out;
1284         }
1285  out:
1286         return ret;
1287 }
1288
1289 static void free_var_defs(struct hist_trigger_data *hist_data)
1290 {
1291         unsigned int i;
1292
1293         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1294                 kfree(hist_data->attrs->var_defs.name[i]);
1295                 kfree(hist_data->attrs->var_defs.expr[i]);
1296         }
1297
1298         hist_data->attrs->var_defs.n_vars = 0;
1299 }
1300
1301 static int parse_var_defs(struct hist_trigger_data *hist_data)
1302 {
1303         char *s, *str, *var_name, *field_str;
1304         unsigned int i, j, n_vars = 0;
1305         int ret = 0;
1306
1307         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
1308                 str = hist_data->attrs->assignment_str[i];
1309                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
1310                         field_str = strsep(&str, ",");
1311                         if (!field_str)
1312                                 break;
1313
1314                         var_name = strsep(&field_str, "=");
1315                         if (!var_name || !field_str) {
1316                                 ret = -EINVAL;
1317                                 goto free;
1318                         }
1319
1320                         if (n_vars == TRACING_MAP_VARS_MAX) {
1321                                 ret = -EINVAL;
1322                                 goto free;
1323                         }
1324
1325                         s = kstrdup(var_name, GFP_KERNEL);
1326                         if (!s) {
1327                                 ret = -ENOMEM;
1328                                 goto free;
1329                         }
1330                         hist_data->attrs->var_defs.name[n_vars] = s;
1331
1332                         s = kstrdup(field_str, GFP_KERNEL);
1333                         if (!s) {
1334                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
1335                                 ret = -ENOMEM;
1336                                 goto free;
1337                         }
1338                         hist_data->attrs->var_defs.expr[n_vars++] = s;
1339
1340                         hist_data->attrs->var_defs.n_vars = n_vars;
1341                 }
1342         }
1343
1344         return ret;
1345  free:
1346         free_var_defs(hist_data);
1347
1348         return ret;
1349 }
1350
1351 static int create_hist_fields(struct hist_trigger_data *hist_data,
1352                               struct trace_event_file *file)
1353 {
1354         int ret;
1355
1356         ret = parse_var_defs(hist_data);
1357         if (ret)
1358                 goto out;
1359
1360         ret = create_val_fields(hist_data, file);
1361         if (ret)
1362                 goto out;
1363
1364         ret = create_var_fields(hist_data, file);
1365         if (ret)
1366                 goto out;
1367
1368         ret = create_key_fields(hist_data, file);
1369         if (ret)
1370                 goto out;
1371  out:
1372         free_var_defs(hist_data);
1373
1374         return ret;
1375 }
1376
1377 static int is_descending(const char *str)
1378 {
1379         if (!str)
1380                 return 0;
1381
1382         if (strcmp(str, "descending") == 0)
1383                 return 1;
1384
1385         if (strcmp(str, "ascending") == 0)
1386                 return 0;
1387
1388         return -EINVAL;
1389 }
1390
1391 static int create_sort_keys(struct hist_trigger_data *hist_data)
1392 {
1393         char *fields_str = hist_data->attrs->sort_key_str;
1394         struct tracing_map_sort_key *sort_key;
1395         int descending, ret = 0;
1396         unsigned int i, j, k;
1397
1398         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
1399
1400         if (!fields_str)
1401                 goto out;
1402
1403         strsep(&fields_str, "=");
1404         if (!fields_str) {
1405                 ret = -EINVAL;
1406                 goto out;
1407         }
1408
1409         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
1410                 struct hist_field *hist_field;
1411                 char *field_str, *field_name;
1412                 const char *test_name;
1413
1414                 sort_key = &hist_data->sort_keys[i];
1415
1416                 field_str = strsep(&fields_str, ",");
1417                 if (!field_str) {
1418                         if (i == 0)
1419                                 ret = -EINVAL;
1420                         break;
1421                 }
1422
1423                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
1424                         ret = -EINVAL;
1425                         break;
1426                 }
1427
1428                 field_name = strsep(&field_str, ".");
1429                 if (!field_name) {
1430                         ret = -EINVAL;
1431                         break;
1432                 }
1433
1434                 if (strcmp(field_name, "hitcount") == 0) {
1435                         descending = is_descending(field_str);
1436                         if (descending < 0) {
1437                                 ret = descending;
1438                                 break;
1439                         }
1440                         sort_key->descending = descending;
1441                         continue;
1442                 }
1443
1444                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
1445                         unsigned int idx;
1446
1447                         hist_field = hist_data->fields[j];
1448                         if (hist_field->flags & HIST_FIELD_FL_VAR)
1449                                 continue;
1450
1451                         idx = k++;
1452
1453                         test_name = hist_field_name(hist_field, 0);
1454
1455                         if (strcmp(field_name, test_name) == 0) {
1456                                 sort_key->field_idx = idx;
1457                                 descending = is_descending(field_str);
1458                                 if (descending < 0) {
1459                                         ret = descending;
1460                                         goto out;
1461                                 }
1462                                 sort_key->descending = descending;
1463                                 break;
1464                         }
1465                 }
1466                 if (j == hist_data->n_fields) {
1467                         ret = -EINVAL;
1468                         break;
1469                 }
1470         }
1471
1472         hist_data->n_sort_keys = i;
1473  out:
1474         return ret;
1475 }
1476
1477 static void destroy_hist_data(struct hist_trigger_data *hist_data)
1478 {
1479         destroy_hist_trigger_attrs(hist_data->attrs);
1480         destroy_hist_fields(hist_data);
1481         tracing_map_destroy(hist_data->map);
1482         kfree(hist_data);
1483 }
1484
1485 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
1486 {
1487         struct tracing_map *map = hist_data->map;
1488         struct ftrace_event_field *field;
1489         struct hist_field *hist_field;
1490         int i, idx;
1491
1492         for_each_hist_field(i, hist_data) {
1493                 hist_field = hist_data->fields[i];
1494                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
1495                         tracing_map_cmp_fn_t cmp_fn;
1496
1497                         field = hist_field->field;
1498
1499                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1500                                 cmp_fn = tracing_map_cmp_none;
1501                         else if (!field)
1502                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
1503                                                              hist_field->is_signed);
1504                         else if (is_string_field(field))
1505                                 cmp_fn = tracing_map_cmp_string;
1506                         else
1507                                 cmp_fn = tracing_map_cmp_num(field->size,
1508                                                              field->is_signed);
1509                         idx = tracing_map_add_key_field(map,
1510                                                         hist_field->offset,
1511                                                         cmp_fn);
1512                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
1513                         idx = tracing_map_add_sum_field(map);
1514
1515                 if (idx < 0)
1516                         return idx;
1517
1518                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
1519                         idx = tracing_map_add_var(map);
1520                         if (idx < 0)
1521                                 return idx;
1522                         hist_field->var.idx = idx;
1523                         hist_field->var.hist_data = hist_data;
1524                 }
1525         }
1526
1527         return 0;
1528 }
1529
1530 static struct hist_trigger_data *
1531 create_hist_data(unsigned int map_bits,
1532                  struct hist_trigger_attrs *attrs,
1533                  struct trace_event_file *file,
1534                  bool remove)
1535 {
1536         const struct tracing_map_ops *map_ops = NULL;
1537         struct hist_trigger_data *hist_data;
1538         int ret = 0;
1539
1540         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
1541         if (!hist_data)
1542                 return ERR_PTR(-ENOMEM);
1543
1544         hist_data->attrs = attrs;
1545         hist_data->remove = remove;
1546
1547         ret = create_hist_fields(hist_data, file);
1548         if (ret)
1549                 goto free;
1550
1551         ret = create_sort_keys(hist_data);
1552         if (ret)
1553                 goto free;
1554
1555         map_ops = &hist_trigger_elt_data_ops;
1556
1557         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
1558                                             map_ops, hist_data);
1559         if (IS_ERR(hist_data->map)) {
1560                 ret = PTR_ERR(hist_data->map);
1561                 hist_data->map = NULL;
1562                 goto free;
1563         }
1564
1565         ret = create_tracing_map_fields(hist_data);
1566         if (ret)
1567                 goto free;
1568
1569         ret = tracing_map_init(hist_data->map);
1570         if (ret)
1571                 goto free;
1572
1573         hist_data->event_file = file;
1574  out:
1575         return hist_data;
1576  free:
1577         hist_data->attrs = NULL;
1578
1579         destroy_hist_data(hist_data);
1580
1581         hist_data = ERR_PTR(ret);
1582
1583         goto out;
1584 }
1585
1586 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
1587                                     struct tracing_map_elt *elt, void *rec,
1588                                     struct ring_buffer_event *rbe)
1589 {
1590         struct hist_field *hist_field;
1591         unsigned int i, var_idx;
1592         u64 hist_val;
1593
1594         for_each_hist_val_field(i, hist_data) {
1595                 hist_field = hist_data->fields[i];
1596                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
1597                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
1598                         var_idx = hist_field->var.idx;
1599                         tracing_map_set_var(elt, var_idx, hist_val);
1600                         continue;
1601                 }
1602                 tracing_map_update_sum(elt, i, hist_val);
1603         }
1604
1605         for_each_hist_key_field(i, hist_data) {
1606                 hist_field = hist_data->fields[i];
1607                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
1608                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
1609                         var_idx = hist_field->var.idx;
1610                         tracing_map_set_var(elt, var_idx, hist_val);
1611                 }
1612         }
1613 }
1614
1615 static inline void add_to_key(char *compound_key, void *key,
1616                               struct hist_field *key_field, void *rec)
1617 {
1618         size_t size = key_field->size;
1619
1620         if (key_field->flags & HIST_FIELD_FL_STRING) {
1621                 struct ftrace_event_field *field;
1622
1623                 field = key_field->field;
1624                 if (field->filter_type == FILTER_DYN_STRING)
1625                         size = *(u32 *)(rec + field->offset) >> 16;
1626                 else if (field->filter_type == FILTER_PTR_STRING)
1627                         size = strlen(key);
1628                 else if (field->filter_type == FILTER_STATIC_STRING)
1629                         size = field->size;
1630
1631                 /* ensure NULL-termination */
1632                 if (size > key_field->size - 1)
1633                         size = key_field->size - 1;
1634         }
1635
1636         memcpy(compound_key + key_field->offset, key, size);
1637 }
1638
1639 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
1640                                struct ring_buffer_event *rbe)
1641 {
1642         struct hist_trigger_data *hist_data = data->private_data;
1643         bool use_compound_key = (hist_data->n_keys > 1);
1644         unsigned long entries[HIST_STACKTRACE_DEPTH];
1645         char compound_key[HIST_KEY_SIZE_MAX];
1646         struct tracing_map_elt *elt = NULL;
1647         struct stack_trace stacktrace;
1648         struct hist_field *key_field;
1649         u64 field_contents;
1650         void *key = NULL;
1651         unsigned int i;
1652
1653         memset(compound_key, 0, hist_data->key_size);
1654
1655         for_each_hist_key_field(i, hist_data) {
1656                 key_field = hist_data->fields[i];
1657
1658                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
1659                         stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
1660                         stacktrace.entries = entries;
1661                         stacktrace.nr_entries = 0;
1662                         stacktrace.skip = HIST_STACKTRACE_SKIP;
1663
1664                         memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
1665                         save_stack_trace(&stacktrace);
1666
1667                         key = entries;
1668                 } else {
1669                         field_contents = key_field->fn(key_field, elt, rbe, rec);
1670                         if (key_field->flags & HIST_FIELD_FL_STRING) {
1671                                 key = (void *)(unsigned long)field_contents;
1672                                 use_compound_key = true;
1673                         } else
1674                                 key = (void *)&field_contents;
1675                 }
1676
1677                 if (use_compound_key)
1678                         add_to_key(compound_key, key, key_field, rec);
1679         }
1680
1681         if (use_compound_key)
1682                 key = compound_key;
1683
1684         elt = tracing_map_insert(hist_data->map, key);
1685         if (elt)
1686                 hist_trigger_elt_update(hist_data, elt, rec, rbe);
1687 }
1688
1689 static void hist_trigger_stacktrace_print(struct seq_file *m,
1690                                           unsigned long *stacktrace_entries,
1691                                           unsigned int max_entries)
1692 {
1693         char str[KSYM_SYMBOL_LEN];
1694         unsigned int spaces = 8;
1695         unsigned int i;
1696
1697         for (i = 0; i < max_entries; i++) {
1698                 if (stacktrace_entries[i] == ULONG_MAX)
1699                         return;
1700
1701                 seq_printf(m, "%*c", 1 + spaces, ' ');
1702                 sprint_symbol(str, stacktrace_entries[i]);
1703                 seq_printf(m, "%s\n", str);
1704         }
1705 }
1706
1707 static void
1708 hist_trigger_entry_print(struct seq_file *m,
1709                          struct hist_trigger_data *hist_data, void *key,
1710                          struct tracing_map_elt *elt)
1711 {
1712         struct hist_field *key_field;
1713         char str[KSYM_SYMBOL_LEN];
1714         bool multiline = false;
1715         const char *field_name;
1716         unsigned int i;
1717         u64 uval;
1718
1719         seq_puts(m, "{ ");
1720
1721         for_each_hist_key_field(i, hist_data) {
1722                 key_field = hist_data->fields[i];
1723
1724                 if (i > hist_data->n_vals)
1725                         seq_puts(m, ", ");
1726
1727                 field_name = hist_field_name(key_field, 0);
1728
1729                 if (key_field->flags & HIST_FIELD_FL_HEX) {
1730                         uval = *(u64 *)(key + key_field->offset);
1731                         seq_printf(m, "%s: %llx", field_name, uval);
1732                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
1733                         uval = *(u64 *)(key + key_field->offset);
1734                         sprint_symbol_no_offset(str, uval);
1735                         seq_printf(m, "%s: [%llx] %-45s", field_name,
1736                                    uval, str);
1737                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
1738                         uval = *(u64 *)(key + key_field->offset);
1739                         sprint_symbol(str, uval);
1740                         seq_printf(m, "%s: [%llx] %-55s", field_name,
1741                                    uval, str);
1742                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
1743                         struct hist_elt_data *elt_data = elt->private_data;
1744                         char *comm;
1745
1746                         if (WARN_ON_ONCE(!elt_data))
1747                                 return;
1748
1749                         comm = elt_data->comm;
1750
1751                         uval = *(u64 *)(key + key_field->offset);
1752                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
1753                                    comm, uval);
1754                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
1755                         const char *syscall_name;
1756
1757                         uval = *(u64 *)(key + key_field->offset);
1758                         syscall_name = get_syscall_name(uval);
1759                         if (!syscall_name)
1760                                 syscall_name = "unknown_syscall";
1761
1762                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
1763                                    syscall_name, uval);
1764                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
1765                         seq_puts(m, "stacktrace:\n");
1766                         hist_trigger_stacktrace_print(m,
1767                                                       key + key_field->offset,
1768                                                       HIST_STACKTRACE_DEPTH);
1769                         multiline = true;
1770                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
1771                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
1772                                    *(u64 *)(key + key_field->offset));
1773                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
1774                         seq_printf(m, "%s: %-50s", field_name,
1775                                    (char *)(key + key_field->offset));
1776                 } else {
1777                         uval = *(u64 *)(key + key_field->offset);
1778                         seq_printf(m, "%s: %10llu", field_name, uval);
1779                 }
1780         }
1781
1782         if (!multiline)
1783                 seq_puts(m, " ");
1784
1785         seq_puts(m, "}");
1786
1787         seq_printf(m, " hitcount: %10llu",
1788                    tracing_map_read_sum(elt, HITCOUNT_IDX));
1789
1790         for (i = 1; i < hist_data->n_vals; i++) {
1791                 field_name = hist_field_name(hist_data->fields[i], 0);
1792
1793                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
1794                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
1795                         continue;
1796
1797                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
1798                         seq_printf(m, "  %s: %10llx", field_name,
1799                                    tracing_map_read_sum(elt, i));
1800                 } else {
1801                         seq_printf(m, "  %s: %10llu", field_name,
1802                                    tracing_map_read_sum(elt, i));
1803                 }
1804         }
1805
1806         seq_puts(m, "\n");
1807 }
1808
1809 static int print_entries(struct seq_file *m,
1810                          struct hist_trigger_data *hist_data)
1811 {
1812         struct tracing_map_sort_entry **sort_entries = NULL;
1813         struct tracing_map *map = hist_data->map;
1814         int i, n_entries;
1815
1816         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
1817                                              hist_data->n_sort_keys,
1818                                              &sort_entries);
1819         if (n_entries < 0)
1820                 return n_entries;
1821
1822         for (i = 0; i < n_entries; i++)
1823                 hist_trigger_entry_print(m, hist_data,
1824                                          sort_entries[i]->key,
1825                                          sort_entries[i]->elt);
1826
1827         tracing_map_destroy_sort_entries(sort_entries, n_entries);
1828
1829         return n_entries;
1830 }
1831
1832 static void hist_trigger_show(struct seq_file *m,
1833                               struct event_trigger_data *data, int n)
1834 {
1835         struct hist_trigger_data *hist_data;
1836         int n_entries;
1837
1838         if (n > 0)
1839                 seq_puts(m, "\n\n");
1840
1841         seq_puts(m, "# event histogram\n#\n# trigger info: ");
1842         data->ops->print(m, data->ops, data);
1843         seq_puts(m, "#\n\n");
1844
1845         hist_data = data->private_data;
1846         n_entries = print_entries(m, hist_data);
1847         if (n_entries < 0)
1848                 n_entries = 0;
1849
1850         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
1851                    (u64)atomic64_read(&hist_data->map->hits),
1852                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
1853 }
1854
1855 static int hist_show(struct seq_file *m, void *v)
1856 {
1857         struct event_trigger_data *data;
1858         struct trace_event_file *event_file;
1859         int n = 0, ret = 0;
1860
1861         mutex_lock(&event_mutex);
1862
1863         event_file = event_file_data(m->private);
1864         if (unlikely(!event_file)) {
1865                 ret = -ENODEV;
1866                 goto out_unlock;
1867         }
1868
1869         list_for_each_entry_rcu(data, &event_file->triggers, list) {
1870                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
1871                         hist_trigger_show(m, data, n++);
1872         }
1873
1874  out_unlock:
1875         mutex_unlock(&event_mutex);
1876
1877         return ret;
1878 }
1879
1880 static int event_hist_open(struct inode *inode, struct file *file)
1881 {
1882         return single_open(file, hist_show, file);
1883 }
1884
1885 const struct file_operations event_hist_fops = {
1886         .open = event_hist_open,
1887         .read = seq_read,
1888         .llseek = seq_lseek,
1889         .release = single_release,
1890 };
1891
1892 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
1893 {
1894         const char *field_name = hist_field_name(hist_field, 0);
1895
1896         if (hist_field->var.name)
1897                 seq_printf(m, "%s=", hist_field->var.name);
1898
1899         if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
1900                 seq_puts(m, "common_timestamp");
1901         else if (field_name)
1902                 seq_printf(m, "%s", field_name);
1903
1904         if (hist_field->flags) {
1905                 const char *flags_str = get_hist_field_flags(hist_field);
1906
1907                 if (flags_str)
1908                         seq_printf(m, ".%s", flags_str);
1909         }
1910 }
1911
1912 static int event_hist_trigger_print(struct seq_file *m,
1913                                     struct event_trigger_ops *ops,
1914                                     struct event_trigger_data *data)
1915 {
1916         struct hist_trigger_data *hist_data = data->private_data;
1917         struct hist_field *field;
1918         bool have_var = false;
1919         unsigned int i;
1920
1921         seq_puts(m, "hist:");
1922
1923         if (data->name)
1924                 seq_printf(m, "%s:", data->name);
1925
1926         seq_puts(m, "keys=");
1927
1928         for_each_hist_key_field(i, hist_data) {
1929                 field = hist_data->fields[i];
1930
1931                 if (i > hist_data->n_vals)
1932                         seq_puts(m, ",");
1933
1934                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
1935                         seq_puts(m, "stacktrace");
1936                 else
1937                         hist_field_print(m, field);
1938         }
1939
1940         seq_puts(m, ":vals=");
1941
1942         for_each_hist_val_field(i, hist_data) {
1943                 field = hist_data->fields[i];
1944                 if (field->flags & HIST_FIELD_FL_VAR) {
1945                         have_var = true;
1946                         continue;
1947                 }
1948
1949                 if (i == HITCOUNT_IDX)
1950                         seq_puts(m, "hitcount");
1951                 else {
1952                         seq_puts(m, ",");
1953                         hist_field_print(m, field);
1954                 }
1955         }
1956
1957         if (have_var) {
1958                 unsigned int n = 0;
1959
1960                 seq_puts(m, ":");
1961
1962                 for_each_hist_val_field(i, hist_data) {
1963                         field = hist_data->fields[i];
1964
1965                         if (field->flags & HIST_FIELD_FL_VAR) {
1966                                 if (n++)
1967                                         seq_puts(m, ",");
1968                                 hist_field_print(m, field);
1969                         }
1970                 }
1971         }
1972
1973         seq_puts(m, ":sort=");
1974
1975         for (i = 0; i < hist_data->n_sort_keys; i++) {
1976                 struct tracing_map_sort_key *sort_key;
1977                 unsigned int idx, first_key_idx;
1978
1979                 /* skip VAR vals */
1980                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
1981
1982                 sort_key = &hist_data->sort_keys[i];
1983                 idx = sort_key->field_idx;
1984
1985                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
1986                         return -EINVAL;
1987
1988                 if (i > 0)
1989                         seq_puts(m, ",");
1990
1991                 if (idx == HITCOUNT_IDX)
1992                         seq_puts(m, "hitcount");
1993                 else {
1994                         if (idx >= first_key_idx)
1995                                 idx += hist_data->n_vars;
1996                         hist_field_print(m, hist_data->fields[idx]);
1997                 }
1998
1999                 if (sort_key->descending)
2000                         seq_puts(m, ".descending");
2001         }
2002         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
2003
2004         if (data->filter_str)
2005                 seq_printf(m, " if %s", data->filter_str);
2006
2007         if (data->paused)
2008                 seq_puts(m, " [paused]");
2009         else
2010                 seq_puts(m, " [active]");
2011
2012         seq_putc(m, '\n');
2013
2014         return 0;
2015 }
2016
2017 static int event_hist_trigger_init(struct event_trigger_ops *ops,
2018                                    struct event_trigger_data *data)
2019 {
2020         struct hist_trigger_data *hist_data = data->private_data;
2021
2022         if (!data->ref && hist_data->attrs->name)
2023                 save_named_trigger(hist_data->attrs->name, data);
2024
2025         data->ref++;
2026
2027         return 0;
2028 }
2029
2030 static void event_hist_trigger_free(struct event_trigger_ops *ops,
2031                                     struct event_trigger_data *data)
2032 {
2033         struct hist_trigger_data *hist_data = data->private_data;
2034
2035         if (WARN_ON_ONCE(data->ref <= 0))
2036                 return;
2037
2038         data->ref--;
2039         if (!data->ref) {
2040                 if (data->name)
2041                         del_named_trigger(data);
2042                 trigger_data_free(data);
2043                 destroy_hist_data(hist_data);
2044         }
2045 }
2046
2047 static struct event_trigger_ops event_hist_trigger_ops = {
2048         .func                   = event_hist_trigger,
2049         .print                  = event_hist_trigger_print,
2050         .init                   = event_hist_trigger_init,
2051         .free                   = event_hist_trigger_free,
2052 };
2053
2054 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
2055                                          struct event_trigger_data *data)
2056 {
2057         data->ref++;
2058
2059         save_named_trigger(data->named_data->name, data);
2060
2061         event_hist_trigger_init(ops, data->named_data);
2062
2063         return 0;
2064 }
2065
2066 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
2067                                           struct event_trigger_data *data)
2068 {
2069         if (WARN_ON_ONCE(data->ref <= 0))
2070                 return;
2071
2072         event_hist_trigger_free(ops, data->named_data);
2073
2074         data->ref--;
2075         if (!data->ref) {
2076                 del_named_trigger(data);
2077                 trigger_data_free(data);
2078         }
2079 }
2080
2081 static struct event_trigger_ops event_hist_trigger_named_ops = {
2082         .func                   = event_hist_trigger,
2083         .print                  = event_hist_trigger_print,
2084         .init                   = event_hist_trigger_named_init,
2085         .free                   = event_hist_trigger_named_free,
2086 };
2087
2088 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
2089                                                             char *param)
2090 {
2091         return &event_hist_trigger_ops;
2092 }
2093
2094 static void hist_clear(struct event_trigger_data *data)
2095 {
2096         struct hist_trigger_data *hist_data = data->private_data;
2097
2098         if (data->name)
2099                 pause_named_trigger(data);
2100
2101         synchronize_sched();
2102
2103         tracing_map_clear(hist_data->map);
2104
2105         if (data->name)
2106                 unpause_named_trigger(data);
2107 }
2108
2109 static bool compatible_field(struct ftrace_event_field *field,
2110                              struct ftrace_event_field *test_field)
2111 {
2112         if (field == test_field)
2113                 return true;
2114         if (field == NULL || test_field == NULL)
2115                 return false;
2116         if (strcmp(field->name, test_field->name) != 0)
2117                 return false;
2118         if (strcmp(field->type, test_field->type) != 0)
2119                 return false;
2120         if (field->size != test_field->size)
2121                 return false;
2122         if (field->is_signed != test_field->is_signed)
2123                 return false;
2124
2125         return true;
2126 }
2127
2128 static bool hist_trigger_match(struct event_trigger_data *data,
2129                                struct event_trigger_data *data_test,
2130                                struct event_trigger_data *named_data,
2131                                bool ignore_filter)
2132 {
2133         struct tracing_map_sort_key *sort_key, *sort_key_test;
2134         struct hist_trigger_data *hist_data, *hist_data_test;
2135         struct hist_field *key_field, *key_field_test;
2136         unsigned int i;
2137
2138         if (named_data && (named_data != data_test) &&
2139             (named_data != data_test->named_data))
2140                 return false;
2141
2142         if (!named_data && is_named_trigger(data_test))
2143                 return false;
2144
2145         hist_data = data->private_data;
2146         hist_data_test = data_test->private_data;
2147
2148         if (hist_data->n_vals != hist_data_test->n_vals ||
2149             hist_data->n_fields != hist_data_test->n_fields ||
2150             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
2151                 return false;
2152
2153         if (!ignore_filter) {
2154                 if ((data->filter_str && !data_test->filter_str) ||
2155                    (!data->filter_str && data_test->filter_str))
2156                         return false;
2157         }
2158
2159         for_each_hist_field(i, hist_data) {
2160                 key_field = hist_data->fields[i];
2161                 key_field_test = hist_data_test->fields[i];
2162
2163                 if (key_field->flags != key_field_test->flags)
2164                         return false;
2165                 if (!compatible_field(key_field->field, key_field_test->field))
2166                         return false;
2167                 if (key_field->offset != key_field_test->offset)
2168                         return false;
2169                 if (key_field->size != key_field_test->size)
2170                         return false;
2171                 if (key_field->is_signed != key_field_test->is_signed)
2172                         return false;
2173                 if (!!key_field->var.name != !!key_field_test->var.name)
2174                         return false;
2175                 if (key_field->var.name &&
2176                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
2177                         return false;
2178         }
2179
2180         for (i = 0; i < hist_data->n_sort_keys; i++) {
2181                 sort_key = &hist_data->sort_keys[i];
2182                 sort_key_test = &hist_data_test->sort_keys[i];
2183
2184                 if (sort_key->field_idx != sort_key_test->field_idx ||
2185                     sort_key->descending != sort_key_test->descending)
2186                         return false;
2187         }
2188
2189         if (!ignore_filter && data->filter_str &&
2190             (strcmp(data->filter_str, data_test->filter_str) != 0))
2191                 return false;
2192
2193         return true;
2194 }
2195
2196 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
2197                                  struct event_trigger_data *data,
2198                                  struct trace_event_file *file)
2199 {
2200         struct hist_trigger_data *hist_data = data->private_data;
2201         struct event_trigger_data *test, *named_data = NULL;
2202         int ret = 0;
2203
2204         if (hist_data->attrs->name) {
2205                 named_data = find_named_trigger(hist_data->attrs->name);
2206                 if (named_data) {
2207                         if (!hist_trigger_match(data, named_data, named_data,
2208                                                 true)) {
2209                                 ret = -EINVAL;
2210                                 goto out;
2211                         }
2212                 }
2213         }
2214
2215         if (hist_data->attrs->name && !named_data)
2216                 goto new;
2217
2218         list_for_each_entry_rcu(test, &file->triggers, list) {
2219                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2220                         if (!hist_trigger_match(data, test, named_data, false))
2221                                 continue;
2222                         if (hist_data->attrs->pause)
2223                                 test->paused = true;
2224                         else if (hist_data->attrs->cont)
2225                                 test->paused = false;
2226                         else if (hist_data->attrs->clear)
2227                                 hist_clear(test);
2228                         else
2229                                 ret = -EEXIST;
2230                         goto out;
2231                 }
2232         }
2233  new:
2234         if (hist_data->attrs->cont || hist_data->attrs->clear) {
2235                 ret = -ENOENT;
2236                 goto out;
2237         }
2238
2239         if (hist_data->attrs->pause)
2240                 data->paused = true;
2241
2242         if (named_data) {
2243                 destroy_hist_data(data->private_data);
2244                 data->private_data = named_data->private_data;
2245                 set_named_trigger_data(data, named_data);
2246                 data->ops = &event_hist_trigger_named_ops;
2247         }
2248
2249         if (data->ops->init) {
2250                 ret = data->ops->init(data->ops, data);
2251                 if (ret < 0)
2252                         goto out;
2253         }
2254
2255         list_add_rcu(&data->list, &file->triggers);
2256         ret++;
2257
2258         update_cond_flag(file);
2259
2260         if (hist_data->enable_timestamps)
2261                 tracing_set_time_stamp_abs(file->tr, true);
2262
2263         if (trace_event_trigger_enable_disable(file, 1) < 0) {
2264                 list_del_rcu(&data->list);
2265                 update_cond_flag(file);
2266                 ret--;
2267         }
2268  out:
2269         return ret;
2270 }
2271
2272 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
2273                                     struct event_trigger_data *data,
2274                                     struct trace_event_file *file)
2275 {
2276         struct hist_trigger_data *hist_data = data->private_data;
2277         struct event_trigger_data *test, *named_data = NULL;
2278         bool unregistered = false;
2279
2280         if (hist_data->attrs->name)
2281                 named_data = find_named_trigger(hist_data->attrs->name);
2282
2283         list_for_each_entry_rcu(test, &file->triggers, list) {
2284                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2285                         if (!hist_trigger_match(data, test, named_data, false))
2286                                 continue;
2287                         unregistered = true;
2288                         list_del_rcu(&test->list);
2289                         trace_event_trigger_enable_disable(file, 0);
2290                         update_cond_flag(file);
2291                         break;
2292                 }
2293         }
2294
2295         if (unregistered && test->ops->free)
2296                 test->ops->free(test->ops, test);
2297
2298         if (hist_data->enable_timestamps) {
2299                 if (!hist_data->remove || unregistered)
2300                         tracing_set_time_stamp_abs(file->tr, false);
2301         }
2302 }
2303
2304 static void hist_unreg_all(struct trace_event_file *file)
2305 {
2306         struct event_trigger_data *test, *n;
2307         struct hist_trigger_data *hist_data;
2308
2309         list_for_each_entry_safe(test, n, &file->triggers, list) {
2310                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2311                         hist_data = test->private_data;
2312                         list_del_rcu(&test->list);
2313                         trace_event_trigger_enable_disable(file, 0);
2314                         update_cond_flag(file);
2315                         if (hist_data->enable_timestamps)
2316                                 tracing_set_time_stamp_abs(file->tr, false);
2317                         if (test->ops->free)
2318                                 test->ops->free(test->ops, test);
2319                 }
2320         }
2321 }
2322
2323 static int event_hist_trigger_func(struct event_command *cmd_ops,
2324                                    struct trace_event_file *file,
2325                                    char *glob, char *cmd, char *param)
2326 {
2327         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
2328         struct event_trigger_data *trigger_data;
2329         struct hist_trigger_attrs *attrs;
2330         struct event_trigger_ops *trigger_ops;
2331         struct hist_trigger_data *hist_data;
2332         bool remove = false;
2333         char *trigger;
2334         int ret = 0;
2335
2336         if (!param)
2337                 return -EINVAL;
2338
2339         if (glob[0] == '!')
2340                 remove = true;
2341
2342         /* separate the trigger from the filter (k:v [if filter]) */
2343         trigger = strsep(&param, " \t");
2344         if (!trigger)
2345                 return -EINVAL;
2346
2347         attrs = parse_hist_trigger_attrs(trigger);
2348         if (IS_ERR(attrs))
2349                 return PTR_ERR(attrs);
2350
2351         if (attrs->map_bits)
2352                 hist_trigger_bits = attrs->map_bits;
2353
2354         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
2355         if (IS_ERR(hist_data)) {
2356                 destroy_hist_trigger_attrs(attrs);
2357                 return PTR_ERR(hist_data);
2358         }
2359
2360         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
2361
2362         ret = -ENOMEM;
2363         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
2364         if (!trigger_data)
2365                 goto out_free;
2366
2367         trigger_data->count = -1;
2368         trigger_data->ops = trigger_ops;
2369         trigger_data->cmd_ops = cmd_ops;
2370
2371         INIT_LIST_HEAD(&trigger_data->list);
2372         RCU_INIT_POINTER(trigger_data->filter, NULL);
2373
2374         trigger_data->private_data = hist_data;
2375
2376         /* if param is non-empty, it's supposed to be a filter */
2377         if (param && cmd_ops->set_filter) {
2378                 ret = cmd_ops->set_filter(param, trigger_data, file);
2379                 if (ret < 0)
2380                         goto out_free;
2381         }
2382
2383         if (remove) {
2384                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
2385                 ret = 0;
2386                 goto out_free;
2387         }
2388
2389         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
2390         /*
2391          * The above returns on success the # of triggers registered,
2392          * but if it didn't register any it returns zero.  Consider no
2393          * triggers registered a failure too.
2394          */
2395         if (!ret) {
2396                 if (!(attrs->pause || attrs->cont || attrs->clear))
2397                         ret = -ENOENT;
2398                 goto out_free;
2399         } else if (ret < 0)
2400                 goto out_free;
2401         /* Just return zero, not the number of registered triggers */
2402         ret = 0;
2403  out:
2404         return ret;
2405  out_free:
2406         if (cmd_ops->set_filter)
2407                 cmd_ops->set_filter(NULL, trigger_data, NULL);
2408
2409         kfree(trigger_data);
2410
2411         destroy_hist_data(hist_data);
2412         goto out;
2413 }
2414
2415 static struct event_command trigger_hist_cmd = {
2416         .name                   = "hist",
2417         .trigger_type           = ETT_EVENT_HIST,
2418         .flags                  = EVENT_CMD_FL_NEEDS_REC,
2419         .func                   = event_hist_trigger_func,
2420         .reg                    = hist_register_trigger,
2421         .unreg                  = hist_unregister_trigger,
2422         .unreg_all              = hist_unreg_all,
2423         .get_trigger_ops        = event_hist_get_trigger_ops,
2424         .set_filter             = set_trigger_filter,
2425 };
2426
2427 __init int register_trigger_hist_cmd(void)
2428 {
2429         int ret;
2430
2431         ret = register_event_command(&trigger_hist_cmd);
2432         WARN_ON(ret < 0);
2433
2434         return ret;
2435 }
2436
2437 static void
2438 hist_enable_trigger(struct event_trigger_data *data, void *rec,
2439                     struct ring_buffer_event *event)
2440 {
2441         struct enable_trigger_data *enable_data = data->private_data;
2442         struct event_trigger_data *test;
2443
2444         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
2445                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2446                         if (enable_data->enable)
2447                                 test->paused = false;
2448                         else
2449                                 test->paused = true;
2450                 }
2451         }
2452 }
2453
2454 static void
2455 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
2456                           struct ring_buffer_event *event)
2457 {
2458         if (!data->count)
2459                 return;
2460
2461         if (data->count != -1)
2462                 (data->count)--;
2463
2464         hist_enable_trigger(data, rec, event);
2465 }
2466
2467 static struct event_trigger_ops hist_enable_trigger_ops = {
2468         .func                   = hist_enable_trigger,
2469         .print                  = event_enable_trigger_print,
2470         .init                   = event_trigger_init,
2471         .free                   = event_enable_trigger_free,
2472 };
2473
2474 static struct event_trigger_ops hist_enable_count_trigger_ops = {
2475         .func                   = hist_enable_count_trigger,
2476         .print                  = event_enable_trigger_print,
2477         .init                   = event_trigger_init,
2478         .free                   = event_enable_trigger_free,
2479 };
2480
2481 static struct event_trigger_ops hist_disable_trigger_ops = {
2482         .func                   = hist_enable_trigger,
2483         .print                  = event_enable_trigger_print,
2484         .init                   = event_trigger_init,
2485         .free                   = event_enable_trigger_free,
2486 };
2487
2488 static struct event_trigger_ops hist_disable_count_trigger_ops = {
2489         .func                   = hist_enable_count_trigger,
2490         .print                  = event_enable_trigger_print,
2491         .init                   = event_trigger_init,
2492         .free                   = event_enable_trigger_free,
2493 };
2494
2495 static struct event_trigger_ops *
2496 hist_enable_get_trigger_ops(char *cmd, char *param)
2497 {
2498         struct event_trigger_ops *ops;
2499         bool enable;
2500
2501         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
2502
2503         if (enable)
2504                 ops = param ? &hist_enable_count_trigger_ops :
2505                         &hist_enable_trigger_ops;
2506         else
2507                 ops = param ? &hist_disable_count_trigger_ops :
2508                         &hist_disable_trigger_ops;
2509
2510         return ops;
2511 }
2512
2513 static void hist_enable_unreg_all(struct trace_event_file *file)
2514 {
2515         struct event_trigger_data *test, *n;
2516
2517         list_for_each_entry_safe(test, n, &file->triggers, list) {
2518                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
2519                         list_del_rcu(&test->list);
2520                         update_cond_flag(file);
2521                         trace_event_trigger_enable_disable(file, 0);
2522                         if (test->ops->free)
2523                                 test->ops->free(test->ops, test);
2524                 }
2525         }
2526 }
2527
2528 static struct event_command trigger_hist_enable_cmd = {
2529         .name                   = ENABLE_HIST_STR,
2530         .trigger_type           = ETT_HIST_ENABLE,
2531         .func                   = event_enable_trigger_func,
2532         .reg                    = event_enable_register_trigger,
2533         .unreg                  = event_enable_unregister_trigger,
2534         .unreg_all              = hist_enable_unreg_all,
2535         .get_trigger_ops        = hist_enable_get_trigger_ops,
2536         .set_filter             = set_trigger_filter,
2537 };
2538
2539 static struct event_command trigger_hist_disable_cmd = {
2540         .name                   = DISABLE_HIST_STR,
2541         .trigger_type           = ETT_HIST_ENABLE,
2542         .func                   = event_enable_trigger_func,
2543         .reg                    = event_enable_register_trigger,
2544         .unreg                  = event_enable_unregister_trigger,
2545         .unreg_all              = hist_enable_unreg_all,
2546         .get_trigger_ops        = hist_enable_get_trigger_ops,
2547         .set_filter             = set_trigger_filter,
2548 };
2549
2550 static __init void unregister_trigger_hist_enable_disable_cmds(void)
2551 {
2552         unregister_event_command(&trigger_hist_enable_cmd);
2553         unregister_event_command(&trigger_hist_disable_cmd);
2554 }
2555
2556 __init int register_trigger_hist_enable_disable_cmds(void)
2557 {
2558         int ret;
2559
2560         ret = register_event_command(&trigger_hist_enable_cmd);
2561         if (WARN_ON(ret < 0))
2562                 return ret;
2563         ret = register_event_command(&trigger_hist_disable_cmd);
2564         if (WARN_ON(ret < 0))
2565                 unregister_trigger_hist_enable_disable_cmds();
2566
2567         return ret;
2568 }