kernel/trace/trace_events.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * event tracer
   4  *
   5  * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
   6  *
   7  *  - Added format output of fields of the trace point.
   8  *    This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
   9  *
  10  */
  11
  12 #define pr_fmt(fmt) fmt
  13
  14 #include <linux/workqueue.h>
  15 #include <linux/security.h>
  16 #include <linux/spinlock.h>
  17 #include <linux/kthread.h>
  18 #include <linux/tracefs.h>
  19 #include <linux/uaccess.h>
  20 #include <linux/module.h>
  21 #include <linux/ctype.h>
  22 #include <linux/sort.h>
  23 #include <linux/slab.h>
  24 #include <linux/delay.h>
  25
  26 #include <trace/events/sched.h>
  27 #include <trace/syscall.h>
  28
  29 #include <asm/setup.h>
  30
  31 #include "trace_output.h"
  32
  33 #undef TRACE_SYSTEM
  34 #define TRACE_SYSTEM "TRACE_SYSTEM"
  35
  36 DEFINE_MUTEX(event_mutex);
  37
  38 LIST_HEAD(ftrace_events);
  39 static LIST_HEAD(ftrace_generic_fields);
  40 static LIST_HEAD(ftrace_common_fields);
  41 static bool eventdir_initialized;
  42
  43 static LIST_HEAD(module_strings);
  44
  45 struct module_string {
  46         struct list_head        next;
  47         struct module           *module;
  48         char                    *str;
  49 };
  50
  51 #define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
  52
  53 static struct kmem_cache *field_cachep;
  54 static struct kmem_cache *file_cachep;
  55
  56 static inline int system_refcount(struct event_subsystem *system)
  57 {
  58         return system->ref_count;
  59 }
  60
  61 static int system_refcount_inc(struct event_subsystem *system)
  62 {
  63         return system->ref_count++;
  64 }
  65
  66 static int system_refcount_dec(struct event_subsystem *system)
  67 {
  68         return --system->ref_count;
  69 }
  70
  71 /* Double loops, do not use break, only goto's work */
  72 #define do_for_each_event_file(tr, file)                        \
  73         list_for_each_entry(tr, &ftrace_trace_arrays, list) {   \
  74                 list_for_each_entry(file, &tr->events, list)
  75
  76 #define do_for_each_event_file_safe(tr, file)                   \
  77         list_for_each_entry(tr, &ftrace_trace_arrays, list) {   \
  78                 struct trace_event_file *___n;                          \
  79                 list_for_each_entry_safe(file, ___n, &tr->events, list)
  80
  81 #define while_for_each_event_file()             \
  82         }
  83
  84 static struct ftrace_event_field *
  85 __find_event_field(struct list_head *head, char *name)
  86 {
  87         struct ftrace_event_field *field;
  88
  89         list_for_each_entry(field, head, link) {
  90                 if (!strcmp(field->name, name))
  91                         return field;
  92         }
  93
  94         return NULL;
  95 }
  96
  97 struct ftrace_event_field *
  98 trace_find_event_field(struct trace_event_call *call, char *name)
  99 {
 100         struct ftrace_event_field *field;
 101         struct list_head *head;
 102
 103         head = trace_get_fields(call);
 104         field = __find_event_field(head, name);
 105         if (field)
 106                 return field;
 107
 108         field = __find_event_field(&ftrace_generic_fields, name);
 109         if (field)
 110                 return field;
 111
 112         return __find_event_field(&ftrace_common_fields, name);
 113 }
 114
 115 static int __trace_define_field(struct list_head *head, const char *type,
 116                                 const char *name, int offset, int size,
 117                                 int is_signed, int filter_type, int len)
 118 {
 119         struct ftrace_event_field *field;
 120
 121         field = kmem_cache_alloc(field_cachep, GFP_TRACE);
 122         if (!field)
 123                 return -ENOMEM;
 124
 125         field->name = name;
 126         field->type = type;
 127
 128         if (filter_type == FILTER_OTHER)
 129                 field->filter_type = filter_assign_type(type);
 130         else
 131                 field->filter_type = filter_type;
 132
 133         field->offset = offset;
 134         field->size = size;
 135         field->is_signed = is_signed;
 136         field->len = len;
 137
 138         list_add(&field->link, head);
 139
 140         return 0;
 141 }
 142
 143 int trace_define_field(struct trace_event_call *call, const char *type,
 144                        const char *name, int offset, int size, int is_signed,
 145                        int filter_type)
 146 {
 147         struct list_head *head;
 148
 149         if (WARN_ON(!call->class))
 150                 return 0;
 151
 152         head = trace_get_fields(call);
 153         return __trace_define_field(head, type, name, offset, size,
 154                                     is_signed, filter_type, 0);
 155 }
 156 EXPORT_SYMBOL_GPL(trace_define_field);
 157
 158 static int trace_define_field_ext(struct trace_event_call *call, const char *type,
 159                        const char *name, int offset, int size, int is_signed,
 160                        int filter_type, int len)
 161 {
 162         struct list_head *head;
 163
 164         if (WARN_ON(!call->class))
 165                 return 0;
 166
 167         head = trace_get_fields(call);
 168         return __trace_define_field(head, type, name, offset, size,
 169                                     is_signed, filter_type, len);
 170 }
 171
 172 #define __generic_field(type, item, filter_type)                        \
 173         ret = __trace_define_field(&ftrace_generic_fields, #type,       \
 174                                    #item, 0, 0, is_signed_type(type),   \
 175                                    filter_type, 0);                     \
 176         if (ret)                                                        \
 177                 return ret;
 178
 179 #define __common_field(type, item)                                      \
 180         ret = __trace_define_field(&ftrace_common_fields, #type,        \
 181                                    "common_" #item,                     \
 182                                    offsetof(typeof(ent), item),         \
 183                                    sizeof(ent.item),                    \
 184                                    is_signed_type(type), FILTER_OTHER, 0);      \
 185         if (ret)                                                        \
 186                 return ret;
 187
 188 static int trace_define_generic_fields(void)
 189 {
 190         int ret;
 191
 192         __generic_field(int, CPU, FILTER_CPU);
 193         __generic_field(int, cpu, FILTER_CPU);
 194         __generic_field(int, common_cpu, FILTER_CPU);
 195         __generic_field(char *, COMM, FILTER_COMM);
 196         __generic_field(char *, comm, FILTER_COMM);
 197         __generic_field(char *, stacktrace, FILTER_STACKTRACE);
 198         __generic_field(char *, STACKTRACE, FILTER_STACKTRACE);
 199
 200         return ret;
 201 }
 202
 203 static int trace_define_common_fields(void)
 204 {
 205         int ret;
 206         struct trace_entry ent;
 207
 208         __common_field(unsigned short, type);
 209         __common_field(unsigned char, flags);
 210         /* Holds both preempt_count and migrate_disable */
 211         __common_field(unsigned char, preempt_count);
 212         __common_field(int, pid);
 213
 214         return ret;
 215 }
 216
 217 static void trace_destroy_fields(struct trace_event_call *call)
 218 {
 219         struct ftrace_event_field *field, *next;
 220         struct list_head *head;
 221
 222         head = trace_get_fields(call);
 223         list_for_each_entry_safe(field, next, head, link) {
 224                 list_del(&field->link);
 225                 kmem_cache_free(field_cachep, field);
 226         }
 227 }
 228
 229 /*
 230  * run-time version of trace_event_get_offsets_<call>() that returns the last
 231  * accessible offset of trace fields excluding __dynamic_array bytes
 232  */
 233 int trace_event_get_offsets(struct trace_event_call *call)
 234 {
 235         struct ftrace_event_field *tail;
 236         struct list_head *head;
 237
 238         head = trace_get_fields(call);
 239         /*
 240          * head->next points to the last field with the largest offset,
 241          * since it was added last by trace_define_field()
 242          */
 243         tail = list_first_entry(head, struct ftrace_event_field, link);
 244         return tail->offset + tail->size;
 245 }
 246
 247 /*
 248  * Check if the referenced field is an array and return true,
 249  * as arrays are OK to dereference.
 250  */
 251 static bool test_field(const char *fmt, struct trace_event_call *call)
 252 {
 253         struct trace_event_fields *field = call->class->fields_array;
 254         const char *array_descriptor;
 255         const char *p = fmt;
 256         int len;
 257
 258         if (!(len = str_has_prefix(fmt, "REC->")))
 259                 return false;
 260         fmt += len;
 261         for (p = fmt; *p; p++) {
 262                 if (!isalnum(*p) && *p != '_')
 263                         break;
 264         }
 265         len = p - fmt;
 266
 267         for (; field->type; field++) {
 268                 if (strncmp(field->name, fmt, len) ||
 269                     field->name[len])
 270                         continue;
 271                 array_descriptor = strchr(field->type, '[');
 272                 /* This is an array and is OK to dereference. */
 273                 return array_descriptor != NULL;
 274         }
 275         return false;
 276 }
 277
 278 /*
 279  * Examine the print fmt of the event looking for unsafe dereference
 280  * pointers using %p* that could be recorded in the trace event and
 281  * much later referenced after the pointer was freed. Dereferencing
 282  * pointers are OK, if it is dereferenced into the event itself.
 283  */
 284 static void test_event_printk(struct trace_event_call *call)
 285 {
 286         u64 dereference_flags = 0;
 287         bool first = true;
 288         const char *fmt, *c, *r, *a;
 289         int parens = 0;
 290         char in_quote = 0;
 291         int start_arg = 0;
 292         int arg = 0;
 293         int i;
 294
 295         fmt = call->print_fmt;
 296
 297         if (!fmt)
 298                 return;
 299
 300         for (i = 0; fmt[i]; i++) {
 301                 switch (fmt[i]) {
 302                 case '\\':
 303                         i++;
 304                         if (!fmt[i])
 305                                 return;
 306                         continue;
 307                 case '"':
 308                 case '\'':
 309                         /*
 310                          * The print fmt starts with a string that
 311                          * is processed first to find %p* usage,
 312                          * then after the first string, the print fmt
 313                          * contains arguments that are used to check
 314                          * if the dereferenced %p* usage is safe.
 315                          */
 316                         if (first) {
 317                                 if (fmt[i] == '\'')
 318                                         continue;
 319                                 if (in_quote) {
 320                                         arg = 0;
 321                                         first = false;
 322                                         /*
 323                                          * If there was no %p* uses
 324                                          * the fmt is OK.
 325                                          */
 326                                         if (!dereference_flags)
 327                                                 return;
 328                                 }
 329                         }
 330                         if (in_quote) {
 331                                 if (in_quote == fmt[i])
 332                                         in_quote = 0;
 333                         } else {
 334                                 in_quote = fmt[i];
 335                         }
 336                         continue;
 337                 case '%':
 338                         if (!first || !in_quote)
 339                                 continue;
 340                         i++;
 341                         if (!fmt[i])
 342                                 return;
 343                         switch (fmt[i]) {
 344                         case '%':
 345                                 continue;
 346                         case 'p':
 347                                 /* Find dereferencing fields */
 348                                 switch (fmt[i + 1]) {
 349                                 case 'B': case 'R': case 'r':
 350                                 case 'b': case 'M': case 'm':
 351                                 case 'I': case 'i': case 'E':
 352                                 case 'U': case 'V': case 'N':
 353                                 case 'a': case 'd': case 'D':
 354                                 case 'g': case 't': case 'C':
 355                                 case 'O': case 'f':
 356                                         if (WARN_ONCE(arg == 63,
 357                                                       "Too many args for event: %s",
 358                                                       trace_event_name(call)))
 359                                                 return;
 360                                         dereference_flags |= 1ULL << arg;
 361                                 }
 362                                 break;
 363                         default:
 364                         {
 365                                 bool star = false;
 366                                 int j;
 367
 368                                 /* Increment arg if %*s exists. */
 369                                 for (j = 0; fmt[i + j]; j++) {
 370                                         if (isdigit(fmt[i + j]) ||
 371                                             fmt[i + j] == '.')
 372                                                 continue;
 373                                         if (fmt[i + j] == '*') {
 374                                                 star = true;
 375                                                 continue;
 376                                         }
 377                                         if ((fmt[i + j] == 's') && star)
 378                                                 arg++;
 379                                         break;
 380                                 }
 381                                 break;
 382                         } /* default */
 383
 384                         } /* switch */
 385                         arg++;
 386                         continue;
 387                 case '(':
 388                         if (in_quote)
 389                                 continue;
 390                         parens++;
 391                         continue;
 392                 case ')':
 393                         if (in_quote)
 394                                 continue;
 395                         parens--;
 396                         if (WARN_ONCE(parens < 0,
 397                                       "Paren mismatch for event: %s\narg='%s'\n%*s",
 398                                       trace_event_name(call),
 399                                       fmt + start_arg,
 400                                       (i - start_arg) + 5, "^"))
 401                                 return;
 402                         continue;
 403                 case ',':
 404                         if (in_quote || parens)
 405                                 continue;
 406                         i++;
 407                         while (isspace(fmt[i]))
 408                                 i++;
 409                         start_arg = i;
 410                         if (!(dereference_flags & (1ULL << arg)))
 411                                 goto next_arg;
 412
 413                         /* Find the REC-> in the argument */
 414                         c = strchr(fmt + i, ',');
 415                         r = strstr(fmt + i, "REC->");
 416                         if (r && (!c || r < c)) {
 417                                 /*
 418                                  * Addresses of events on the buffer,
 419                                  * or an array on the buffer is
 420                                  * OK to dereference.
 421                                  * There's ways to fool this, but
 422                                  * this is to catch common mistakes,
 423                                  * not malicious code.
 424                                  */
 425                                 a = strchr(fmt + i, '&');
 426                                 if ((a && (a < r)) || test_field(r, call))
 427                                         dereference_flags &= ~(1ULL << arg);
 428                         } else if ((r = strstr(fmt + i, "__get_dynamic_array(")) &&
 429                                    (!c || r < c)) {
 430                                 dereference_flags &= ~(1ULL << arg);
 431                         } else if ((r = strstr(fmt + i, "__get_sockaddr(")) &&
 432                                    (!c || r < c)) {
 433                                 dereference_flags &= ~(1ULL << arg);
 434                         }
 435
 436                 next_arg:
 437                         i--;
 438                         arg++;
 439                 }
 440         }
 441
 442         /*
 443          * If you triggered the below warning, the trace event reported
 444          * uses an unsafe dereference pointer %p*. As the data stored
 445          * at the trace event time may no longer exist when the trace
 446          * event is printed, dereferencing to the original source is
 447          * unsafe. The source of the dereference must be copied into the
 448          * event itself, and the dereference must access the copy instead.
 449          */
 450         if (WARN_ON_ONCE(dereference_flags)) {
 451                 arg = 1;
 452                 while (!(dereference_flags & 1)) {
 453                         dereference_flags >>= 1;
 454                         arg++;
 455                 }
 456                 pr_warn("event %s has unsafe dereference of argument %d\n",
 457                         trace_event_name(call), arg);
 458                 pr_warn("print_fmt: %s\n", fmt);
 459         }
 460 }
 461
 462 int trace_event_raw_init(struct trace_event_call *call)
 463 {
 464         int id;
 465
 466         id = register_trace_event(&call->event);
 467         if (!id)
 468                 return -ENODEV;
 469
 470         test_event_printk(call);
 471
 472         return 0;
 473 }
 474 EXPORT_SYMBOL_GPL(trace_event_raw_init);
 475
 476 bool trace_event_ignore_this_pid(struct trace_event_file *trace_file)
 477 {
 478         struct trace_array *tr = trace_file->tr;
 479         struct trace_array_cpu *data;
 480         struct trace_pid_list *no_pid_list;
 481         struct trace_pid_list *pid_list;
 482
 483         pid_list = rcu_dereference_raw(tr->filtered_pids);
 484         no_pid_list = rcu_dereference_raw(tr->filtered_no_pids);
 485
 486         if (!pid_list && !no_pid_list)
 487                 return false;
 488
 489         data = this_cpu_ptr(tr->array_buffer.data);
 490
 491         return data->ignore_pid;
 492 }
 493 EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid);
 494
 495 void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
 496                                  struct trace_event_file *trace_file,
 497                                  unsigned long len)
 498 {
 499         struct trace_event_call *event_call = trace_file->event_call;
 500
 501         if ((trace_file->flags & EVENT_FILE_FL_PID_FILTER) &&
 502             trace_event_ignore_this_pid(trace_file))
 503                 return NULL;
 504
 505         /*
 506          * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables
 507          * preemption (adding one to the preempt_count). Since we are
 508          * interested in the preempt_count at the time the tracepoint was
 509          * hit, we need to subtract one to offset the increment.
 510          */
 511         fbuffer->trace_ctx = tracing_gen_ctx_dec();
 512         fbuffer->trace_file = trace_file;
 513
 514         fbuffer->event =
 515                 trace_event_buffer_lock_reserve(&fbuffer->buffer, trace_file,
 516                                                 event_call->event.type, len,
 517                                                 fbuffer->trace_ctx);
 518         if (!fbuffer->event)
 519                 return NULL;
 520
 521         fbuffer->regs = NULL;
 522         fbuffer->entry = ring_buffer_event_data(fbuffer->event);
 523         return fbuffer->entry;
 524 }
 525 EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
 526
 527 int trace_event_reg(struct trace_event_call *call,
 528                     enum trace_reg type, void *data)
 529 {
 530         struct trace_event_file *file = data;
 531
 532         WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
 533         switch (type) {
 534         case TRACE_REG_REGISTER:
 535                 return tracepoint_probe_register(call->tp,
 536                                                  call->class->probe,
 537                                                  file);
 538         case TRACE_REG_UNREGISTER:
 539                 tracepoint_probe_unregister(call->tp,
 540                                             call->class->probe,
 541                                             file);
 542                 return 0;
 543
 544 #ifdef CONFIG_PERF_EVENTS
 545         case TRACE_REG_PERF_REGISTER:
 546                 return tracepoint_probe_register(call->tp,
 547                                                  call->class->perf_probe,
 548                                                  call);
 549         case TRACE_REG_PERF_UNREGISTER:
 550                 tracepoint_probe_unregister(call->tp,
 551                                             call->class->perf_probe,
 552                                             call);
 553                 return 0;
 554         case TRACE_REG_PERF_OPEN:
 555         case TRACE_REG_PERF_CLOSE:
 556         case TRACE_REG_PERF_ADD:
 557         case TRACE_REG_PERF_DEL:
 558                 return 0;
 559 #endif
 560         }
 561         return 0;
 562 }
 563 EXPORT_SYMBOL_GPL(trace_event_reg);
 564
 565 void trace_event_enable_cmd_record(bool enable)
 566 {
 567         struct trace_event_file *file;
 568         struct trace_array *tr;
 569
 570         lockdep_assert_held(&event_mutex);
 571
 572         do_for_each_event_file(tr, file) {
 573
 574                 if (!(file->flags & EVENT_FILE_FL_ENABLED))
 575                         continue;
 576
 577                 if (enable) {
 578                         tracing_start_cmdline_record();
 579                         set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 580                 } else {
 581                         tracing_stop_cmdline_record();
 582                         clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 583                 }
 584         } while_for_each_event_file();
 585 }
 586
 587 void trace_event_enable_tgid_record(bool enable)
 588 {
 589         struct trace_event_file *file;
 590         struct trace_array *tr;
 591
 592         lockdep_assert_held(&event_mutex);
 593
 594         do_for_each_event_file(tr, file) {
 595                 if (!(file->flags & EVENT_FILE_FL_ENABLED))
 596                         continue;
 597
 598                 if (enable) {
 599                         tracing_start_tgid_record();
 600                         set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
 601                 } else {
 602                         tracing_stop_tgid_record();
 603                         clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT,
 604                                   &file->flags);
 605                 }
 606         } while_for_each_event_file();
 607 }
 608
 609 static int __ftrace_event_enable_disable(struct trace_event_file *file,
 610                                          int enable, int soft_disable)
 611 {
 612         struct trace_event_call *call = file->event_call;
 613         struct trace_array *tr = file->tr;
 614         unsigned long file_flags = file->flags;
 615         int ret = 0;
 616         int disable;
 617
 618         switch (enable) {
 619         case 0:
 620                 /*
 621                  * When soft_disable is set and enable is cleared, the sm_ref
 622                  * reference counter is decremented. If it reaches 0, we want
 623                  * to clear the SOFT_DISABLED flag but leave the event in the
 624                  * state that it was. That is, if the event was enabled and
 625                  * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
 626                  * is set we do not want the event to be enabled before we
 627                  * clear the bit.
 628                  *
 629                  * When soft_disable is not set but the SOFT_MODE flag is,
 630                  * we do nothing. Do not disable the tracepoint, otherwise
 631                  * "soft enable"s (clearing the SOFT_DISABLED bit) wont work.
 632                  */
 633                 if (soft_disable) {
 634                         if (atomic_dec_return(&file->sm_ref) > 0)
 635                                 break;
 636                         disable = file->flags & EVENT_FILE_FL_SOFT_DISABLED;
 637                         clear_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
 638                 } else
 639                         disable = !(file->flags & EVENT_FILE_FL_SOFT_MODE);
 640
 641                 if (disable && (file->flags & EVENT_FILE_FL_ENABLED)) {
 642                         clear_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
 643                         if (file->flags & EVENT_FILE_FL_RECORDED_CMD) {
 644                                 tracing_stop_cmdline_record();
 645                                 clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 646                         }
 647
 648                         if (file->flags & EVENT_FILE_FL_RECORDED_TGID) {
 649                                 tracing_stop_tgid_record();
 650                                 clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
 651                         }
 652
 653                         call->class->reg(call, TRACE_REG_UNREGISTER, file);
 654                 }
 655                 /* If in SOFT_MODE, just set the SOFT_DISABLE_BIT, else clear it */
 656                 if (file->flags & EVENT_FILE_FL_SOFT_MODE)
 657                         set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 658                 else
 659                         clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 660                 break;
 661         case 1:
 662                 /*
 663                  * When soft_disable is set and enable is set, we want to
 664                  * register the tracepoint for the event, but leave the event
 665                  * as is. That means, if the event was already enabled, we do
 666                  * nothing (but set SOFT_MODE). If the event is disabled, we
 667                  * set SOFT_DISABLED before enabling the event tracepoint, so
 668                  * it still seems to be disabled.
 669                  */
 670                 if (!soft_disable)
 671                         clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 672                 else {
 673                         if (atomic_inc_return(&file->sm_ref) > 1)
 674                                 break;
 675                         set_bit(EVENT_FILE_FL_SOFT_MODE_BIT, &file->flags);
 676                 }
 677
 678                 if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
 679                         bool cmd = false, tgid = false;
 680
 681                         /* Keep the event disabled, when going to SOFT_MODE. */
 682                         if (soft_disable)
 683                                 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 684
 685                         if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {
 686                                 cmd = true;
 687                                 tracing_start_cmdline_record();
 688                                 set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 689                         }
 690
 691                         if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {
 692                                 tgid = true;
 693                                 tracing_start_tgid_record();
 694                                 set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
 695                         }
 696
 697                         ret = call->class->reg(call, TRACE_REG_REGISTER, file);
 698                         if (ret) {
 699                                 if (cmd)
 700                                         tracing_stop_cmdline_record();
 701                                 if (tgid)
 702                                         tracing_stop_tgid_record();
 703                                 pr_info("event trace: Could not enable event "
 704                                         "%s\n", trace_event_name(call));
 705                                 break;
 706                         }
 707                         set_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
 708
 709                         /* WAS_ENABLED gets set but never cleared. */
 710                         set_bit(EVENT_FILE_FL_WAS_ENABLED_BIT, &file->flags);
 711                 }
 712                 break;
 713         }
 714
 715         /* Enable or disable use of trace_buffered_event */
 716         if ((file_flags & EVENT_FILE_FL_SOFT_DISABLED) !=
 717             (file->flags & EVENT_FILE_FL_SOFT_DISABLED)) {
 718                 if (file->flags & EVENT_FILE_FL_SOFT_DISABLED)
 719                         trace_buffered_event_enable();
 720                 else
 721                         trace_buffered_event_disable();
 722         }
 723
 724         return ret;
 725 }
 726
 727 int trace_event_enable_disable(struct trace_event_file *file,
 728                                int enable, int soft_disable)
 729 {
 730         return __ftrace_event_enable_disable(file, enable, soft_disable);
 731 }
 732
 733 static int ftrace_event_enable_disable(struct trace_event_file *file,
 734                                        int enable)
 735 {
 736         return __ftrace_event_enable_disable(file, enable, 0);
 737 }
 738
 739 static void ftrace_clear_events(struct trace_array *tr)
 740 {
 741         struct trace_event_file *file;
 742
 743         mutex_lock(&event_mutex);
 744         list_for_each_entry(file, &tr->events, list) {
 745                 ftrace_event_enable_disable(file, 0);
 746         }
 747         mutex_unlock(&event_mutex);
 748 }
 749
 750 static void
 751 event_filter_pid_sched_process_exit(void *data, struct task_struct *task)
 752 {
 753         struct trace_pid_list *pid_list;
 754         struct trace_array *tr = data;
 755
 756         pid_list = rcu_dereference_raw(tr->filtered_pids);
 757         trace_filter_add_remove_task(pid_list, NULL, task);
 758
 759         pid_list = rcu_dereference_raw(tr->filtered_no_pids);
 760         trace_filter_add_remove_task(pid_list, NULL, task);
 761 }
 762
 763 static void
 764 event_filter_pid_sched_process_fork(void *data,
 765                                     struct task_struct *self,
 766                                     struct task_struct *task)
 767 {
 768         struct trace_pid_list *pid_list;
 769         struct trace_array *tr = data;
 770
 771         pid_list = rcu_dereference_sched(tr->filtered_pids);
 772         trace_filter_add_remove_task(pid_list, self, task);
 773
 774         pid_list = rcu_dereference_sched(tr->filtered_no_pids);
 775         trace_filter_add_remove_task(pid_list, self, task);
 776 }
 777
 778 void trace_event_follow_fork(struct trace_array *tr, bool enable)
 779 {
 780         if (enable) {
 781                 register_trace_prio_sched_process_fork(event_filter_pid_sched_process_fork,
 782                                                        tr, INT_MIN);
 783                 register_trace_prio_sched_process_free(event_filter_pid_sched_process_exit,
 784                                                        tr, INT_MAX);
 785         } else {
 786                 unregister_trace_sched_process_fork(event_filter_pid_sched_process_fork,
 787                                                     tr);
 788                 unregister_trace_sched_process_free(event_filter_pid_sched_process_exit,
 789                                                     tr);
 790         }
 791 }
 792
 793 static void
 794 event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
 795                                         struct task_struct *prev,
 796                                         struct task_struct *next,
 797                                         unsigned int prev_state)
 798 {
 799         struct trace_array *tr = data;
 800         struct trace_pid_list *no_pid_list;
 801         struct trace_pid_list *pid_list;
 802         bool ret;
 803
 804         pid_list = rcu_dereference_sched(tr->filtered_pids);
 805         no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
 806
 807         /*
 808          * Sched switch is funny, as we only want to ignore it
 809          * in the notrace case if both prev and next should be ignored.
 810          */
 811         ret = trace_ignore_this_task(NULL, no_pid_list, prev) &&
 812                 trace_ignore_this_task(NULL, no_pid_list, next);
 813
 814         this_cpu_write(tr->array_buffer.data->ignore_pid, ret ||
 815                        (trace_ignore_this_task(pid_list, NULL, prev) &&
 816                         trace_ignore_this_task(pid_list, NULL, next)));
 817 }
 818
 819 static void
 820 event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
 821                                          struct task_struct *prev,
 822                                          struct task_struct *next,
 823                                          unsigned int prev_state)
 824 {
 825         struct trace_array *tr = data;
 826         struct trace_pid_list *no_pid_list;
 827         struct trace_pid_list *pid_list;
 828
 829         pid_list = rcu_dereference_sched(tr->filtered_pids);
 830         no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
 831
 832         this_cpu_write(tr->array_buffer.data->ignore_pid,
 833                        trace_ignore_this_task(pid_list, no_pid_list, next));
 834 }
 835
 836 static void
 837 event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
 838 {
 839         struct trace_array *tr = data;
 840         struct trace_pid_list *no_pid_list;
 841         struct trace_pid_list *pid_list;
 842
 843         /* Nothing to do if we are already tracing */
 844         if (!this_cpu_read(tr->array_buffer.data->ignore_pid))
 845                 return;
 846
 847         pid_list = rcu_dereference_sched(tr->filtered_pids);
 848         no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
 849
 850         this_cpu_write(tr->array_buffer.data->ignore_pid,
 851                        trace_ignore_this_task(pid_list, no_pid_list, task));
 852 }
 853
 854 static void
 855 event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
 856 {
 857         struct trace_array *tr = data;
 858         struct trace_pid_list *no_pid_list;
 859         struct trace_pid_list *pid_list;
 860
 861         /* Nothing to do if we are not tracing */
 862         if (this_cpu_read(tr->array_buffer.data->ignore_pid))
 863                 return;
 864
 865         pid_list = rcu_dereference_sched(tr->filtered_pids);
 866         no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
 867
 868         /* Set tracing if current is enabled */
 869         this_cpu_write(tr->array_buffer.data->ignore_pid,
 870                        trace_ignore_this_task(pid_list, no_pid_list, current));
 871 }
 872
 873 static void unregister_pid_events(struct trace_array *tr)
 874 {
 875         unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);
 876         unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);
 877
 878         unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);
 879         unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);
 880
 881         unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);
 882         unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);
 883
 884         unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);
 885         unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);
 886 }
 887
 888 static void __ftrace_clear_event_pids(struct trace_array *tr, int type)
 889 {
 890         struct trace_pid_list *pid_list;
 891         struct trace_pid_list *no_pid_list;
 892         struct trace_event_file *file;
 893         int cpu;
 894
 895         pid_list = rcu_dereference_protected(tr->filtered_pids,
 896                                              lockdep_is_held(&event_mutex));
 897         no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
 898                                              lockdep_is_held(&event_mutex));
 899
 900         /* Make sure there's something to do */
 901         if (!pid_type_enabled(type, pid_list, no_pid_list))
 902                 return;
 903
 904         if (!still_need_pid_events(type, pid_list, no_pid_list)) {
 905                 unregister_pid_events(tr);
 906
 907                 list_for_each_entry(file, &tr->events, list) {
 908                         clear_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
 909                 }
 910
 911                 for_each_possible_cpu(cpu)
 912                         per_cpu_ptr(tr->array_buffer.data, cpu)->ignore_pid = false;
 913         }
 914
 915         if (type & TRACE_PIDS)
 916                 rcu_assign_pointer(tr->filtered_pids, NULL);
 917
 918         if (type & TRACE_NO_PIDS)
 919                 rcu_assign_pointer(tr->filtered_no_pids, NULL);
 920
 921         /* Wait till all users are no longer using pid filtering */
 922         tracepoint_synchronize_unregister();
 923
 924         if ((type & TRACE_PIDS) && pid_list)
 925                 trace_pid_list_free(pid_list);
 926
 927         if ((type & TRACE_NO_PIDS) && no_pid_list)
 928                 trace_pid_list_free(no_pid_list);
 929 }
 930
 931 static void ftrace_clear_event_pids(struct trace_array *tr, int type)
 932 {
 933         mutex_lock(&event_mutex);
 934         __ftrace_clear_event_pids(tr, type);
 935         mutex_unlock(&event_mutex);
 936 }
 937
 938 static void __put_system(struct event_subsystem *system)
 939 {
 940         struct event_filter *filter = system->filter;
 941
 942         WARN_ON_ONCE(system_refcount(system) == 0);
 943         if (system_refcount_dec(system))
 944                 return;
 945
 946         list_del(&system->list);
 947
 948         if (filter) {
 949                 kfree(filter->filter_string);
 950                 kfree(filter);
 951         }
 952         kfree_const(system->name);
 953         kfree(system);
 954 }
 955
 956 static void __get_system(struct event_subsystem *system)
 957 {
 958         WARN_ON_ONCE(system_refcount(system) == 0);
 959         system_refcount_inc(system);
 960 }
 961
 962 static void __get_system_dir(struct trace_subsystem_dir *dir)
 963 {
 964         WARN_ON_ONCE(dir->ref_count == 0);
 965         dir->ref_count++;
 966         __get_system(dir->subsystem);
 967 }
 968
 969 static void __put_system_dir(struct trace_subsystem_dir *dir)
 970 {
 971         WARN_ON_ONCE(dir->ref_count == 0);
 972         /* If the subsystem is about to be freed, the dir must be too */
 973         WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
 974
 975         __put_system(dir->subsystem);
 976         if (!--dir->ref_count)
 977                 kfree(dir);
 978 }
 979
 980 static void put_system(struct trace_subsystem_dir *dir)
 981 {
 982         mutex_lock(&event_mutex);
 983         __put_system_dir(dir);
 984         mutex_unlock(&event_mutex);
 985 }
 986
 987 static void remove_subsystem(struct trace_subsystem_dir *dir)
 988 {
 989         if (!dir)
 990                 return;
 991
 992         if (!--dir->nr_events) {
 993                 tracefs_remove(dir->entry);
 994                 list_del(&dir->list);
 995                 __put_system_dir(dir);
 996         }
 997 }
 998
 999 static void remove_event_file_dir(struct trace_event_file *file)
1000 {
1001         struct dentry *dir = file->dir;
1002         struct dentry *child;
1003
1004         if (dir) {
1005                 spin_lock(&dir->d_lock);        /* probably unneeded */
1006                 list_for_each_entry(child, &dir->d_subdirs, d_child) {
1007                         if (d_really_is_positive(child))        /* probably unneeded */
1008                                 d_inode(child)->i_private = NULL;
1009                 }
1010                 spin_unlock(&dir->d_lock);
1011
1012                 tracefs_remove(dir);
1013         }
1014
1015         list_del(&file->list);
1016         remove_subsystem(file->system);
1017         free_event_filter(file->filter);
1018         kmem_cache_free(file_cachep, file);
1019 }
1020
1021 /*
1022  * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
1023  */
1024 static int
1025 __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
1026                               const char *sub, const char *event, int set)
1027 {
1028         struct trace_event_file *file;
1029         struct trace_event_call *call;
1030         const char *name;
1031         int ret = -EINVAL;
1032         int eret = 0;
1033
1034         list_for_each_entry(file, &tr->events, list) {
1035
1036                 call = file->event_call;
1037                 name = trace_event_name(call);
1038
1039                 if (!name || !call->class || !call->class->reg)
1040                         continue;
1041
1042                 if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
1043                         continue;
1044
1045                 if (match &&
1046                     strcmp(match, name) != 0 &&
1047                     strcmp(match, call->class->system) != 0)
1048                         continue;
1049
1050                 if (sub && strcmp(sub, call->class->system) != 0)
1051                         continue;
1052
1053                 if (event && strcmp(event, name) != 0)
1054                         continue;
1055
1056                 ret = ftrace_event_enable_disable(file, set);
1057
1058                 /*
1059                  * Save the first error and return that. Some events
1060                  * may still have been enabled, but let the user
1061                  * know that something went wrong.
1062                  */
1063                 if (ret && !eret)
1064                         eret = ret;
1065
1066                 ret = eret;
1067         }
1068
1069         return ret;
1070 }
1071
1072 static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,
1073                                   const char *sub, const char *event, int set)
1074 {
1075         int ret;
1076
1077         mutex_lock(&event_mutex);
1078         ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set);
1079         mutex_unlock(&event_mutex);
1080
1081         return ret;
1082 }
1083
1084 int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)
1085 {
1086         char *event = NULL, *sub = NULL, *match;
1087         int ret;
1088
1089         if (!tr)
1090                 return -ENOENT;
1091         /*
1092          * The buf format can be <subsystem>:<event-name>
1093          *  *:<event-name> means any event by that name.
1094          *  :<event-name> is the same.
1095          *
1096          *  <subsystem>:* means all events in that subsystem
1097          *  <subsystem>: means the same.
1098          *
1099          *  <name> (no ':') means all events in a subsystem with
1100          *  the name <name> or any event that matches <name>
1101          */
1102
1103         match = strsep(&buf, ":");
1104         if (buf) {
1105                 sub = match;
1106                 event = buf;
1107                 match = NULL;
1108
1109                 if (!strlen(sub) || strcmp(sub, "*") == 0)
1110                         sub = NULL;
1111                 if (!strlen(event) || strcmp(event, "*") == 0)
1112                         event = NULL;
1113         }
1114
1115         ret = __ftrace_set_clr_event(tr, match, sub, event, set);
1116
1117         /* Put back the colon to allow this to be called again */
1118         if (buf)
1119                 *(buf - 1) = ':';
1120
1121         return ret;
1122 }
1123
1124 /**
1125  * trace_set_clr_event - enable or disable an event
1126  * @system: system name to match (NULL for any system)
1127  * @event: event name to match (NULL for all events, within system)
1128  * @set: 1 to enable, 0 to disable
1129  *
1130  * This is a way for other parts of the kernel to enable or disable
1131  * event recording.
1132  *
1133  * Returns 0 on success, -EINVAL if the parameters do not match any
1134  * registered events.
1135  */
1136 int trace_set_clr_event(const char *system, const char *event, int set)
1137 {
1138         struct trace_array *tr = top_trace_array();
1139
1140         if (!tr)
1141                 return -ENODEV;
1142
1143         return __ftrace_set_clr_event(tr, NULL, system, event, set);
1144 }
1145 EXPORT_SYMBOL_GPL(trace_set_clr_event);
1146
1147 /**
1148  * trace_array_set_clr_event - enable or disable an event for a trace array.
1149  * @tr: concerned trace array.
1150  * @system: system name to match (NULL for any system)
1151  * @event: event name to match (NULL for all events, within system)
1152  * @enable: true to enable, false to disable
1153  *
1154  * This is a way for other parts of the kernel to enable or disable
1155  * event recording.
1156  *
1157  * Returns 0 on success, -EINVAL if the parameters do not match any
1158  * registered events.
1159  */
1160 int trace_array_set_clr_event(struct trace_array *tr, const char *system,
1161                 const char *event, bool enable)
1162 {
1163         int set;
1164
1165         if (!tr)
1166                 return -ENOENT;
1167
1168         set = (enable == true) ? 1 : 0;
1169         return __ftrace_set_clr_event(tr, NULL, system, event, set);
1170 }
1171 EXPORT_SYMBOL_GPL(trace_array_set_clr_event);
1172
1173 /* 128 should be much more than enough */
1174 #define EVENT_BUF_SIZE          127
1175
1176 static ssize_t
1177 ftrace_event_write(struct file *file, const char __user *ubuf,
1178                    size_t cnt, loff_t *ppos)
1179 {
1180         struct trace_parser parser;
1181         struct seq_file *m = file->private_data;
1182         struct trace_array *tr = m->private;
1183         ssize_t read, ret;
1184
1185         if (!cnt)
1186                 return 0;
1187
1188         ret = tracing_update_buffers();
1189         if (ret < 0)
1190                 return ret;
1191
1192         if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
1193                 return -ENOMEM;
1194
1195         read = trace_get_user(&parser, ubuf, cnt, ppos);
1196
1197         if (read >= 0 && trace_parser_loaded((&parser))) {
1198                 int set = 1;
1199
1200                 if (*parser.buffer == '!')
1201                         set = 0;
1202
1203                 ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);
1204                 if (ret)
1205                         goto out_put;
1206         }
1207
1208         ret = read;
1209
1210  out_put:
1211         trace_parser_put(&parser);
1212
1213         return ret;
1214 }
1215
1216 static void *
1217 t_next(struct seq_file *m, void *v, loff_t *pos)
1218 {
1219         struct trace_event_file *file = v;
1220         struct trace_event_call *call;
1221         struct trace_array *tr = m->private;
1222
1223         (*pos)++;
1224
1225         list_for_each_entry_continue(file, &tr->events, list) {
1226                 call = file->event_call;
1227                 /*
1228                  * The ftrace subsystem is for showing formats only.
1229                  * They can not be enabled or disabled via the event files.
1230                  */
1231                 if (call->class && call->class->reg &&
1232                     !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
1233                         return file;
1234         }
1235
1236         return NULL;
1237 }
1238
1239 static void *t_start(struct seq_file *m, loff_t *pos)
1240 {
1241         struct trace_event_file *file;
1242         struct trace_array *tr = m->private;
1243         loff_t l;
1244
1245         mutex_lock(&event_mutex);
1246
1247         file = list_entry(&tr->events, struct trace_event_file, list);
1248         for (l = 0; l <= *pos; ) {
1249                 file = t_next(m, file, &l);
1250                 if (!file)
1251                         break;
1252         }
1253         return file;
1254 }
1255
1256 static void *
1257 s_next(struct seq_file *m, void *v, loff_t *pos)
1258 {
1259         struct trace_event_file *file = v;
1260         struct trace_array *tr = m->private;
1261
1262         (*pos)++;
1263
1264         list_for_each_entry_continue(file, &tr->events, list) {
1265                 if (file->flags & EVENT_FILE_FL_ENABLED)
1266                         return file;
1267         }
1268
1269         return NULL;
1270 }
1271
1272 static void *s_start(struct seq_file *m, loff_t *pos)
1273 {
1274         struct trace_event_file *file;
1275         struct trace_array *tr = m->private;
1276         loff_t l;
1277
1278         mutex_lock(&event_mutex);
1279
1280         file = list_entry(&tr->events, struct trace_event_file, list);
1281         for (l = 0; l <= *pos; ) {
1282                 file = s_next(m, file, &l);
1283                 if (!file)
1284                         break;
1285         }
1286         return file;
1287 }
1288
1289 static int t_show(struct seq_file *m, void *v)
1290 {
1291         struct trace_event_file *file = v;
1292         struct trace_event_call *call = file->event_call;
1293
1294         if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
1295                 seq_printf(m, "%s:", call->class->system);
1296         seq_printf(m, "%s\n", trace_event_name(call));
1297
1298         return 0;
1299 }
1300
1301 static void t_stop(struct seq_file *m, void *p)
1302 {
1303         mutex_unlock(&event_mutex);
1304 }
1305
1306 static void *
1307 __next(struct seq_file *m, void *v, loff_t *pos, int type)
1308 {
1309         struct trace_array *tr = m->private;
1310         struct trace_pid_list *pid_list;
1311
1312         if (type == TRACE_PIDS)
1313                 pid_list = rcu_dereference_sched(tr->filtered_pids);
1314         else
1315                 pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1316
1317         return trace_pid_next(pid_list, v, pos);
1318 }
1319
1320 static void *
1321 p_next(struct seq_file *m, void *v, loff_t *pos)
1322 {
1323         return __next(m, v, pos, TRACE_PIDS);
1324 }
1325
1326 static void *
1327 np_next(struct seq_file *m, void *v, loff_t *pos)
1328 {
1329         return __next(m, v, pos, TRACE_NO_PIDS);
1330 }
1331
1332 static void *__start(struct seq_file *m, loff_t *pos, int type)
1333         __acquires(RCU)
1334 {
1335         struct trace_pid_list *pid_list;
1336         struct trace_array *tr = m->private;
1337
1338         /*
1339          * Grab the mutex, to keep calls to p_next() having the same
1340          * tr->filtered_pids as p_start() has.
1341          * If we just passed the tr->filtered_pids around, then RCU would
1342          * have been enough, but doing that makes things more complex.
1343          */
1344         mutex_lock(&event_mutex);
1345         rcu_read_lock_sched();
1346
1347         if (type == TRACE_PIDS)
1348                 pid_list = rcu_dereference_sched(tr->filtered_pids);
1349         else
1350                 pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1351
1352         if (!pid_list)
1353                 return NULL;
1354
1355         return trace_pid_start(pid_list, pos);
1356 }
1357
1358 static void *p_start(struct seq_file *m, loff_t *pos)
1359         __acquires(RCU)
1360 {
1361         return __start(m, pos, TRACE_PIDS);
1362 }
1363
1364 static void *np_start(struct seq_file *m, loff_t *pos)
1365         __acquires(RCU)
1366 {
1367         return __start(m, pos, TRACE_NO_PIDS);
1368 }
1369
1370 static void p_stop(struct seq_file *m, void *p)
1371         __releases(RCU)
1372 {
1373         rcu_read_unlock_sched();
1374         mutex_unlock(&event_mutex);
1375 }
1376
1377 static ssize_t
1378 event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1379                   loff_t *ppos)
1380 {
1381         struct trace_event_file *file;
1382         unsigned long flags;
1383         char buf[4] = "0";
1384
1385         mutex_lock(&event_mutex);
1386         file = event_file_data(filp);
1387         if (likely(file))
1388                 flags = file->flags;
1389         mutex_unlock(&event_mutex);
1390
1391         if (!file)
1392                 return -ENODEV;
1393
1394         if (flags & EVENT_FILE_FL_ENABLED &&
1395             !(flags & EVENT_FILE_FL_SOFT_DISABLED))
1396                 strcpy(buf, "1");
1397
1398         if (flags & EVENT_FILE_FL_SOFT_DISABLED ||
1399             flags & EVENT_FILE_FL_SOFT_MODE)
1400                 strcat(buf, "*");
1401
1402         strcat(buf, "\n");
1403
1404         return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));
1405 }
1406
1407 static ssize_t
1408 event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1409                    loff_t *ppos)
1410 {
1411         struct trace_event_file *file;
1412         unsigned long val;
1413         int ret;
1414
1415         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1416         if (ret)
1417                 return ret;
1418
1419         ret = tracing_update_buffers();
1420         if (ret < 0)
1421                 return ret;
1422
1423         switch (val) {
1424         case 0:
1425         case 1:
1426                 ret = -ENODEV;
1427                 mutex_lock(&event_mutex);
1428                 file = event_file_data(filp);
1429                 if (likely(file))
1430                         ret = ftrace_event_enable_disable(file, val);
1431                 mutex_unlock(&event_mutex);
1432                 break;
1433
1434         default:
1435                 return -EINVAL;
1436         }
1437
1438         *ppos += cnt;
1439
1440         return ret ? ret : cnt;
1441 }
1442
1443 static ssize_t
1444 system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1445                    loff_t *ppos)
1446 {
1447         const char set_to_char[4] = { '?', '0', '1', 'X' };
1448         struct trace_subsystem_dir *dir = filp->private_data;
1449         struct event_subsystem *system = dir->subsystem;
1450         struct trace_event_call *call;
1451         struct trace_event_file *file;
1452         struct trace_array *tr = dir->tr;
1453         char buf[2];
1454         int set = 0;
1455         int ret;
1456
1457         mutex_lock(&event_mutex);
1458         list_for_each_entry(file, &tr->events, list) {
1459                 call = file->event_call;
1460                 if ((call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
1461                     !trace_event_name(call) || !call->class || !call->class->reg)
1462                         continue;
1463
1464                 if (system && strcmp(call->class->system, system->name) != 0)
1465                         continue;
1466
1467                 /*
1468                  * We need to find out if all the events are set
1469                  * or if all events or cleared, or if we have
1470                  * a mixture.
1471                  */
1472                 set |= (1 << !!(file->flags & EVENT_FILE_FL_ENABLED));
1473
1474                 /*
1475                  * If we have a mixture, no need to look further.
1476                  */
1477                 if (set == 3)
1478                         break;
1479         }
1480         mutex_unlock(&event_mutex);
1481
1482         buf[0] = set_to_char[set];
1483         buf[1] = '\n';
1484
1485         ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
1486
1487         return ret;
1488 }
1489
1490 static ssize_t
1491 system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1492                     loff_t *ppos)
1493 {
1494         struct trace_subsystem_dir *dir = filp->private_data;
1495         struct event_subsystem *system = dir->subsystem;
1496         const char *name = NULL;
1497         unsigned long val;
1498         ssize_t ret;
1499
1500         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1501         if (ret)
1502                 return ret;
1503
1504         ret = tracing_update_buffers();
1505         if (ret < 0)
1506                 return ret;
1507
1508         if (val != 0 && val != 1)
1509                 return -EINVAL;
1510
1511         /*
1512          * Opening of "enable" adds a ref count to system,
1513          * so the name is safe to use.
1514          */
1515         if (system)
1516                 name = system->name;
1517
1518         ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val);
1519         if (ret)
1520                 goto out;
1521
1522         ret = cnt;
1523
1524 out:
1525         *ppos += cnt;
1526
1527         return ret;
1528 }
1529
1530 enum {
1531         FORMAT_HEADER           = 1,
1532         FORMAT_FIELD_SEPERATOR  = 2,
1533         FORMAT_PRINTFMT         = 3,
1534 };
1535
1536 static void *f_next(struct seq_file *m, void *v, loff_t *pos)
1537 {
1538         struct trace_event_call *call = event_file_data(m->private);
1539         struct list_head *common_head = &ftrace_common_fields;
1540         struct list_head *head = trace_get_fields(call);
1541         struct list_head *node = v;
1542
1543         (*pos)++;
1544
1545         switch ((unsigned long)v) {
1546         case FORMAT_HEADER:
1547                 node = common_head;
1548                 break;
1549
1550         case FORMAT_FIELD_SEPERATOR:
1551                 node = head;
1552                 break;
1553
1554         case FORMAT_PRINTFMT:
1555                 /* all done */
1556                 return NULL;
1557         }
1558
1559         node = node->prev;
1560         if (node == common_head)
1561                 return (void *)FORMAT_FIELD_SEPERATOR;
1562         else if (node == head)
1563                 return (void *)FORMAT_PRINTFMT;
1564         else
1565                 return node;
1566 }
1567
1568 static int f_show(struct seq_file *m, void *v)
1569 {
1570         struct trace_event_call *call = event_file_data(m->private);
1571         struct ftrace_event_field *field;
1572         const char *array_descriptor;
1573
1574         switch ((unsigned long)v) {
1575         case FORMAT_HEADER:
1576                 seq_printf(m, "name: %s\n", trace_event_name(call));
1577                 seq_printf(m, "ID: %d\n", call->event.type);
1578                 seq_puts(m, "format:\n");
1579                 return 0;
1580
1581         case FORMAT_FIELD_SEPERATOR:
1582                 seq_putc(m, '\n');
1583                 return 0;
1584
1585         case FORMAT_PRINTFMT:
1586                 seq_printf(m, "\nprint fmt: %s\n",
1587                            call->print_fmt);
1588                 return 0;
1589         }
1590
1591         field = list_entry(v, struct ftrace_event_field, link);
1592         /*
1593          * Smartly shows the array type(except dynamic array).
1594          * Normal:
1595          *      field:TYPE VAR
1596          * If TYPE := TYPE[LEN], it is shown:
1597          *      field:TYPE VAR[LEN]
1598          */
1599         array_descriptor = strchr(field->type, '[');
1600
1601         if (str_has_prefix(field->type, "__data_loc"))
1602                 array_descriptor = NULL;
1603
1604         if (!array_descriptor)
1605                 seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1606                            field->type, field->name, field->offset,
1607                            field->size, !!field->is_signed);
1608         else if (field->len)
1609                 seq_printf(m, "\tfield:%.*s %s[%d];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1610                            (int)(array_descriptor - field->type),
1611                            field->type, field->name,
1612                            field->len, field->offset,
1613                            field->size, !!field->is_signed);
1614         else
1615                 seq_printf(m, "\tfield:%.*s %s[];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
1616                                 (int)(array_descriptor - field->type),
1617                                 field->type, field->name,
1618                                 field->offset, field->size, !!field->is_signed);
1619
1620         return 0;
1621 }
1622
1623 static void *f_start(struct seq_file *m, loff_t *pos)
1624 {
1625         void *p = (void *)FORMAT_HEADER;
1626         loff_t l = 0;
1627
1628         /* ->stop() is called even if ->start() fails */
1629         mutex_lock(&event_mutex);
1630         if (!event_file_data(m->private))
1631                 return ERR_PTR(-ENODEV);
1632
1633         while (l < *pos && p)
1634                 p = f_next(m, p, &l);
1635
1636         return p;
1637 }
1638
1639 static void f_stop(struct seq_file *m, void *p)
1640 {
1641         mutex_unlock(&event_mutex);
1642 }
1643
1644 static const struct seq_operations trace_format_seq_ops = {
1645         .start          = f_start,
1646         .next           = f_next,
1647         .stop           = f_stop,
1648         .show           = f_show,
1649 };
1650
1651 static int trace_format_open(struct inode *inode, struct file *file)
1652 {
1653         struct seq_file *m;
1654         int ret;
1655
1656         /* Do we want to hide event format files on tracefs lockdown? */
1657
1658         ret = seq_open(file, &trace_format_seq_ops);
1659         if (ret < 0)
1660                 return ret;
1661
1662         m = file->private_data;
1663         m->private = file;
1664
1665         return 0;
1666 }
1667
1668 static ssize_t
1669 event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1670 {
1671         int id = (long)event_file_data(filp);
1672         char buf[32];
1673         int len;
1674
1675         if (unlikely(!id))
1676                 return -ENODEV;
1677
1678         len = sprintf(buf, "%d\n", id);
1679
1680         return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
1681 }
1682
1683 static ssize_t
1684 event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1685                   loff_t *ppos)
1686 {
1687         struct trace_event_file *file;
1688         struct trace_seq *s;
1689         int r = -ENODEV;
1690
1691         if (*ppos)
1692                 return 0;
1693
1694         s = kmalloc(sizeof(*s), GFP_KERNEL);
1695
1696         if (!s)
1697                 return -ENOMEM;
1698
1699         trace_seq_init(s);
1700
1701         mutex_lock(&event_mutex);
1702         file = event_file_data(filp);
1703         if (file)
1704                 print_event_filter(file, s);
1705         mutex_unlock(&event_mutex);
1706
1707         if (file)
1708                 r = simple_read_from_buffer(ubuf, cnt, ppos,
1709                                             s->buffer, trace_seq_used(s));
1710
1711         kfree(s);
1712
1713         return r;
1714 }
1715
1716 static ssize_t
1717 event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1718                    loff_t *ppos)
1719 {
1720         struct trace_event_file *file;
1721         char *buf;
1722         int err = -ENODEV;
1723
1724         if (cnt >= PAGE_SIZE)
1725                 return -EINVAL;
1726
1727         buf = memdup_user_nul(ubuf, cnt);
1728         if (IS_ERR(buf))
1729                 return PTR_ERR(buf);
1730
1731         mutex_lock(&event_mutex);
1732         file = event_file_data(filp);
1733         if (file)
1734                 err = apply_event_filter(file, buf);
1735         mutex_unlock(&event_mutex);
1736
1737         kfree(buf);
1738         if (err < 0)
1739                 return err;
1740
1741         *ppos += cnt;
1742
1743         return cnt;
1744 }
1745
1746 static LIST_HEAD(event_subsystems);
1747
1748 static int subsystem_open(struct inode *inode, struct file *filp)
1749 {
1750         struct trace_subsystem_dir *dir = NULL, *iter_dir;
1751         struct trace_array *tr = NULL, *iter_tr;
1752         struct event_subsystem *system = NULL;
1753         int ret;
1754
1755         if (tracing_is_disabled())
1756                 return -ENODEV;
1757
1758         /* Make sure the system still exists */
1759         mutex_lock(&event_mutex);
1760         mutex_lock(&trace_types_lock);
1761         list_for_each_entry(iter_tr, &ftrace_trace_arrays, list) {
1762                 list_for_each_entry(iter_dir, &iter_tr->systems, list) {
1763                         if (iter_dir == inode->i_private) {
1764                                 /* Don't open systems with no events */
1765                                 tr = iter_tr;
1766                                 dir = iter_dir;
1767                                 if (dir->nr_events) {
1768                                         __get_system_dir(dir);
1769                                         system = dir->subsystem;
1770                                 }
1771                                 goto exit_loop;
1772                         }
1773                 }
1774         }
1775  exit_loop:
1776         mutex_unlock(&trace_types_lock);
1777         mutex_unlock(&event_mutex);
1778
1779         if (!system)
1780                 return -ENODEV;
1781
1782         /* Still need to increment the ref count of the system */
1783         if (trace_array_get(tr) < 0) {
1784                 put_system(dir);
1785                 return -ENODEV;
1786         }
1787
1788         ret = tracing_open_generic(inode, filp);
1789         if (ret < 0) {
1790                 trace_array_put(tr);
1791                 put_system(dir);
1792         }
1793
1794         return ret;
1795 }
1796
1797 static int system_tr_open(struct inode *inode, struct file *filp)
1798 {
1799         struct trace_subsystem_dir *dir;
1800         struct trace_array *tr = inode->i_private;
1801         int ret;
1802
1803         /* Make a temporary dir that has no system but points to tr */
1804         dir = kzalloc(sizeof(*dir), GFP_KERNEL);
1805         if (!dir)
1806                 return -ENOMEM;
1807
1808         ret = tracing_open_generic_tr(inode, filp);
1809         if (ret < 0) {
1810                 kfree(dir);
1811                 return ret;
1812         }
1813         dir->tr = tr;
1814         filp->private_data = dir;
1815
1816         return 0;
1817 }
1818
1819 static int subsystem_release(struct inode *inode, struct file *file)
1820 {
1821         struct trace_subsystem_dir *dir = file->private_data;
1822
1823         trace_array_put(dir->tr);
1824
1825         /*
1826          * If dir->subsystem is NULL, then this is a temporary
1827          * descriptor that was made for a trace_array to enable
1828          * all subsystems.
1829          */
1830         if (dir->subsystem)
1831                 put_system(dir);
1832         else
1833                 kfree(dir);
1834
1835         return 0;
1836 }
1837
1838 static ssize_t
1839 subsystem_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
1840                       loff_t *ppos)
1841 {
1842         struct trace_subsystem_dir *dir = filp->private_data;
1843         struct event_subsystem *system = dir->subsystem;
1844         struct trace_seq *s;
1845         int r;
1846
1847         if (*ppos)
1848                 return 0;
1849
1850         s = kmalloc(sizeof(*s), GFP_KERNEL);
1851         if (!s)
1852                 return -ENOMEM;
1853
1854         trace_seq_init(s);
1855
1856         print_subsystem_event_filter(system, s);
1857         r = simple_read_from_buffer(ubuf, cnt, ppos,
1858                                     s->buffer, trace_seq_used(s));
1859
1860         kfree(s);
1861
1862         return r;
1863 }
1864
1865 static ssize_t
1866 subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
1867                        loff_t *ppos)
1868 {
1869         struct trace_subsystem_dir *dir = filp->private_data;
1870         char *buf;
1871         int err;
1872
1873         if (cnt >= PAGE_SIZE)
1874                 return -EINVAL;
1875
1876         buf = memdup_user_nul(ubuf, cnt);
1877         if (IS_ERR(buf))
1878                 return PTR_ERR(buf);
1879
1880         err = apply_subsystem_event_filter(dir, buf);
1881         kfree(buf);
1882         if (err < 0)
1883                 return err;
1884
1885         *ppos += cnt;
1886
1887         return cnt;
1888 }
1889
1890 static ssize_t
1891 show_header(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1892 {
1893         int (*func)(struct trace_seq *s) = filp->private_data;
1894         struct trace_seq *s;
1895         int r;
1896
1897         if (*ppos)
1898                 return 0;
1899
1900         s = kmalloc(sizeof(*s), GFP_KERNEL);
1901         if (!s)
1902                 return -ENOMEM;
1903
1904         trace_seq_init(s);
1905
1906         func(s);
1907         r = simple_read_from_buffer(ubuf, cnt, ppos,
1908                                     s->buffer, trace_seq_used(s));
1909
1910         kfree(s);
1911
1912         return r;
1913 }
1914
1915 static void ignore_task_cpu(void *data)
1916 {
1917         struct trace_array *tr = data;
1918         struct trace_pid_list *pid_list;
1919         struct trace_pid_list *no_pid_list;
1920
1921         /*
1922          * This function is called by on_each_cpu() while the
1923          * event_mutex is held.
1924          */
1925         pid_list = rcu_dereference_protected(tr->filtered_pids,
1926                                              mutex_is_locked(&event_mutex));
1927         no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
1928                                              mutex_is_locked(&event_mutex));
1929
1930         this_cpu_write(tr->array_buffer.data->ignore_pid,
1931                        trace_ignore_this_task(pid_list, no_pid_list, current));
1932 }
1933
1934 static void register_pid_events(struct trace_array *tr)
1935 {
1936         /*
1937          * Register a probe that is called before all other probes
1938          * to set ignore_pid if next or prev do not match.
1939          * Register a probe this is called after all other probes
1940          * to only keep ignore_pid set if next pid matches.
1941          */
1942         register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_pre,
1943                                          tr, INT_MAX);
1944         register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_post,
1945                                          tr, 0);
1946
1947         register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre,
1948                                          tr, INT_MAX);
1949         register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_post,
1950                                          tr, 0);
1951
1952         register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre,
1953                                              tr, INT_MAX);
1954         register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post,
1955                                              tr, 0);
1956
1957         register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_pre,
1958                                          tr, INT_MAX);
1959         register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_post,
1960                                          tr, 0);
1961 }
1962
1963 static ssize_t
1964 event_pid_write(struct file *filp, const char __user *ubuf,
1965                 size_t cnt, loff_t *ppos, int type)
1966 {
1967         struct seq_file *m = filp->private_data;
1968         struct trace_array *tr = m->private;
1969         struct trace_pid_list *filtered_pids = NULL;
1970         struct trace_pid_list *other_pids = NULL;
1971         struct trace_pid_list *pid_list;
1972         struct trace_event_file *file;
1973         ssize_t ret;
1974
1975         if (!cnt)
1976                 return 0;
1977
1978         ret = tracing_update_buffers();
1979         if (ret < 0)
1980                 return ret;
1981
1982         mutex_lock(&event_mutex);
1983
1984         if (type == TRACE_PIDS) {
1985                 filtered_pids = rcu_dereference_protected(tr->filtered_pids,
1986                                                           lockdep_is_held(&event_mutex));
1987                 other_pids = rcu_dereference_protected(tr->filtered_no_pids,
1988                                                           lockdep_is_held(&event_mutex));
1989         } else {
1990                 filtered_pids = rcu_dereference_protected(tr->filtered_no_pids,
1991                                                           lockdep_is_held(&event_mutex));
1992                 other_pids = rcu_dereference_protected(tr->filtered_pids,
1993                                                           lockdep_is_held(&event_mutex));
1994         }
1995
1996         ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
1997         if (ret < 0)
1998                 goto out;
1999
2000         if (type == TRACE_PIDS)
2001                 rcu_assign_pointer(tr->filtered_pids, pid_list);
2002         else
2003                 rcu_assign_pointer(tr->filtered_no_pids, pid_list);
2004
2005         list_for_each_entry(file, &tr->events, list) {
2006                 set_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
2007         }
2008
2009         if (filtered_pids) {
2010                 tracepoint_synchronize_unregister();
2011                 trace_pid_list_free(filtered_pids);
2012         } else if (pid_list && !other_pids) {
2013                 register_pid_events(tr);
2014         }
2015
2016         /*
2017          * Ignoring of pids is done at task switch. But we have to
2018          * check for those tasks that are currently running.
2019          * Always do this in case a pid was appended or removed.
2020          */
2021         on_each_cpu(ignore_task_cpu, tr, 1);
2022
2023  out:
2024         mutex_unlock(&event_mutex);
2025
2026         if (ret > 0)
2027                 *ppos += ret;
2028
2029         return ret;
2030 }
2031
2032 static ssize_t
2033 ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
2034                        size_t cnt, loff_t *ppos)
2035 {
2036         return event_pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
2037 }
2038
2039 static ssize_t
2040 ftrace_event_npid_write(struct file *filp, const char __user *ubuf,
2041                         size_t cnt, loff_t *ppos)
2042 {
2043         return event_pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
2044 }
2045
2046 static int ftrace_event_avail_open(struct inode *inode, struct file *file);
2047 static int ftrace_event_set_open(struct inode *inode, struct file *file);
2048 static int ftrace_event_set_pid_open(struct inode *inode, struct file *file);
2049 static int ftrace_event_set_npid_open(struct inode *inode, struct file *file);
2050 static int ftrace_event_release(struct inode *inode, struct file *file);
2051
2052 static const struct seq_operations show_event_seq_ops = {
2053         .start = t_start,
2054         .next = t_next,
2055         .show = t_show,
2056         .stop = t_stop,
2057 };
2058
2059 static const struct seq_operations show_set_event_seq_ops = {
2060         .start = s_start,
2061         .next = s_next,
2062         .show = t_show,
2063         .stop = t_stop,
2064 };
2065
2066 static const struct seq_operations show_set_pid_seq_ops = {
2067         .start = p_start,
2068         .next = p_next,
2069         .show = trace_pid_show,
2070         .stop = p_stop,
2071 };
2072
2073 static const struct seq_operations show_set_no_pid_seq_ops = {
2074         .start = np_start,
2075         .next = np_next,
2076         .show = trace_pid_show,
2077         .stop = p_stop,
2078 };
2079
2080 static const struct file_operations ftrace_avail_fops = {
2081         .open = ftrace_event_avail_open,
2082         .read = seq_read,
2083         .llseek = seq_lseek,
2084         .release = seq_release,
2085 };
2086
2087 static const struct file_operations ftrace_set_event_fops = {
2088         .open = ftrace_event_set_open,
2089         .read = seq_read,
2090         .write = ftrace_event_write,
2091         .llseek = seq_lseek,
2092         .release = ftrace_event_release,
2093 };
2094
2095 static const struct file_operations ftrace_set_event_pid_fops = {
2096         .open = ftrace_event_set_pid_open,
2097         .read = seq_read,
2098         .write = ftrace_event_pid_write,
2099         .llseek = seq_lseek,
2100         .release = ftrace_event_release,
2101 };
2102
2103 static const struct file_operations ftrace_set_event_notrace_pid_fops = {
2104         .open = ftrace_event_set_npid_open,
2105         .read = seq_read,
2106         .write = ftrace_event_npid_write,
2107         .llseek = seq_lseek,
2108         .release = ftrace_event_release,
2109 };
2110
2111 static const struct file_operations ftrace_enable_fops = {
2112         .open = tracing_open_generic,
2113         .read = event_enable_read,
2114         .write = event_enable_write,
2115         .llseek = default_llseek,
2116 };
2117
2118 static const struct file_operations ftrace_event_format_fops = {
2119         .open = trace_format_open,
2120         .read = seq_read,
2121         .llseek = seq_lseek,
2122         .release = seq_release,
2123 };
2124
2125 static const struct file_operations ftrace_event_id_fops = {
2126         .read = event_id_read,
2127         .llseek = default_llseek,
2128 };
2129
2130 static const struct file_operations ftrace_event_filter_fops = {
2131         .open = tracing_open_generic,
2132         .read = event_filter_read,
2133         .write = event_filter_write,
2134         .llseek = default_llseek,
2135 };
2136
2137 static const struct file_operations ftrace_subsystem_filter_fops = {
2138         .open = subsystem_open,
2139         .read = subsystem_filter_read,
2140         .write = subsystem_filter_write,
2141         .llseek = default_llseek,
2142         .release = subsystem_release,
2143 };
2144
2145 static const struct file_operations ftrace_system_enable_fops = {
2146         .open = subsystem_open,
2147         .read = system_enable_read,
2148         .write = system_enable_write,
2149         .llseek = default_llseek,
2150         .release = subsystem_release,
2151 };
2152
2153 static const struct file_operations ftrace_tr_enable_fops = {
2154         .open = system_tr_open,
2155         .read = system_enable_read,
2156         .write = system_enable_write,
2157         .llseek = default_llseek,
2158         .release = subsystem_release,
2159 };
2160
2161 static const struct file_operations ftrace_show_header_fops = {
2162         .open = tracing_open_generic,
2163         .read = show_header,
2164         .llseek = default_llseek,
2165 };
2166
2167 static int
2168 ftrace_event_open(struct inode *inode, struct file *file,
2169                   const struct seq_operations *seq_ops)
2170 {
2171         struct seq_file *m;
2172         int ret;
2173
2174         ret = security_locked_down(LOCKDOWN_TRACEFS);
2175         if (ret)
2176                 return ret;
2177
2178         ret = seq_open(file, seq_ops);
2179         if (ret < 0)
2180                 return ret;
2181         m = file->private_data;
2182         /* copy tr over to seq ops */
2183         m->private = inode->i_private;
2184
2185         return ret;
2186 }
2187
2188 static int ftrace_event_release(struct inode *inode, struct file *file)
2189 {
2190         struct trace_array *tr = inode->i_private;
2191
2192         trace_array_put(tr);
2193
2194         return seq_release(inode, file);
2195 }
2196
2197 static int
2198 ftrace_event_avail_open(struct inode *inode, struct file *file)
2199 {
2200         const struct seq_operations *seq_ops = &show_event_seq_ops;
2201
2202         /* Checks for tracefs lockdown */
2203         return ftrace_event_open(inode, file, seq_ops);
2204 }
2205
2206 static int
2207 ftrace_event_set_open(struct inode *inode, struct file *file)
2208 {
2209         const struct seq_operations *seq_ops = &show_set_event_seq_ops;
2210         struct trace_array *tr = inode->i_private;
2211         int ret;
2212
2213         ret = tracing_check_open_get_tr(tr);
2214         if (ret)
2215                 return ret;
2216
2217         if ((file->f_mode & FMODE_WRITE) &&
2218             (file->f_flags & O_TRUNC))
2219                 ftrace_clear_events(tr);
2220
2221         ret = ftrace_event_open(inode, file, seq_ops);
2222         if (ret < 0)
2223                 trace_array_put(tr);
2224         return ret;
2225 }
2226
2227 static int
2228 ftrace_event_set_pid_open(struct inode *inode, struct file *file)
2229 {
2230         const struct seq_operations *seq_ops = &show_set_pid_seq_ops;
2231         struct trace_array *tr = inode->i_private;
2232         int ret;
2233
2234         ret = tracing_check_open_get_tr(tr);
2235         if (ret)
2236                 return ret;
2237
2238         if ((file->f_mode & FMODE_WRITE) &&
2239             (file->f_flags & O_TRUNC))
2240                 ftrace_clear_event_pids(tr, TRACE_PIDS);
2241
2242         ret = ftrace_event_open(inode, file, seq_ops);
2243         if (ret < 0)
2244                 trace_array_put(tr);
2245         return ret;
2246 }
2247
2248 static int
2249 ftrace_event_set_npid_open(struct inode *inode, struct file *file)
2250 {
2251         const struct seq_operations *seq_ops = &show_set_no_pid_seq_ops;
2252         struct trace_array *tr = inode->i_private;
2253         int ret;
2254
2255         ret = tracing_check_open_get_tr(tr);
2256         if (ret)
2257                 return ret;
2258
2259         if ((file->f_mode & FMODE_WRITE) &&
2260             (file->f_flags & O_TRUNC))
2261                 ftrace_clear_event_pids(tr, TRACE_NO_PIDS);
2262
2263         ret = ftrace_event_open(inode, file, seq_ops);
2264         if (ret < 0)
2265                 trace_array_put(tr);
2266         return ret;
2267 }
2268
2269 static struct event_subsystem *
2270 create_new_subsystem(const char *name)
2271 {
2272         struct event_subsystem *system;
2273
2274         /* need to create new entry */
2275         system = kmalloc(sizeof(*system), GFP_KERNEL);
2276         if (!system)
2277                 return NULL;
2278
2279         system->ref_count = 1;
2280
2281         /* Only allocate if dynamic (kprobes and modules) */
2282         system->name = kstrdup_const(name, GFP_KERNEL);
2283         if (!system->name)
2284                 goto out_free;
2285
2286         system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);
2287         if (!system->filter)
2288                 goto out_free;
2289
2290         list_add(&system->list, &event_subsystems);
2291
2292         return system;
2293
2294  out_free:
2295         kfree_const(system->name);
2296         kfree(system);
2297         return NULL;
2298 }
2299
2300 static struct dentry *
2301 event_subsystem_dir(struct trace_array *tr, const char *name,
2302                     struct trace_event_file *file, struct dentry *parent)
2303 {
2304         struct event_subsystem *system, *iter;
2305         struct trace_subsystem_dir *dir;
2306         struct dentry *entry;
2307
2308         /* First see if we did not already create this dir */
2309         list_for_each_entry(dir, &tr->systems, list) {
2310                 system = dir->subsystem;
2311                 if (strcmp(system->name, name) == 0) {
2312                         dir->nr_events++;
2313                         file->system = dir;
2314                         return dir->entry;
2315                 }
2316         }
2317
2318         /* Now see if the system itself exists. */
2319         system = NULL;
2320         list_for_each_entry(iter, &event_subsystems, list) {
2321                 if (strcmp(iter->name, name) == 0) {
2322                         system = iter;
2323                         break;
2324                 }
2325         }
2326
2327         dir = kmalloc(sizeof(*dir), GFP_KERNEL);
2328         if (!dir)
2329                 goto out_fail;
2330
2331         if (!system) {
2332                 system = create_new_subsystem(name);
2333                 if (!system)
2334                         goto out_free;
2335         } else
2336                 __get_system(system);
2337
2338         dir->entry = tracefs_create_dir(name, parent);
2339         if (!dir->entry) {
2340                 pr_warn("Failed to create system directory %s\n", name);
2341                 __put_system(system);
2342                 goto out_free;
2343         }
2344
2345         dir->tr = tr;
2346         dir->ref_count = 1;
2347         dir->nr_events = 1;
2348         dir->subsystem = system;
2349         file->system = dir;
2350
2351         /* the ftrace system is special, do not create enable or filter files */
2352         if (strcmp(name, "ftrace") != 0) {
2353
2354                 entry = tracefs_create_file("filter", TRACE_MODE_WRITE,
2355                                             dir->entry, dir,
2356                                             &ftrace_subsystem_filter_fops);
2357                 if (!entry) {
2358                         kfree(system->filter);
2359                         system->filter = NULL;
2360                         pr_warn("Could not create tracefs '%s/filter' entry\n", name);
2361                 }
2362
2363                 trace_create_file("enable", TRACE_MODE_WRITE, dir->entry, dir,
2364                                   &ftrace_system_enable_fops);
2365         }
2366
2367         list_add(&dir->list, &tr->systems);
2368
2369         return dir->entry;
2370
2371  out_free:
2372         kfree(dir);
2373  out_fail:
2374         /* Only print this message if failed on memory allocation */
2375         if (!dir || !system)
2376                 pr_warn("No memory to create event subsystem %s\n", name);
2377         return NULL;
2378 }
2379
2380 static int
2381 event_define_fields(struct trace_event_call *call)
2382 {
2383         struct list_head *head;
2384         int ret = 0;
2385
2386         /*
2387          * Other events may have the same class. Only update
2388          * the fields if they are not already defined.
2389          */
2390         head = trace_get_fields(call);
2391         if (list_empty(head)) {
2392                 struct trace_event_fields *field = call->class->fields_array;
2393                 unsigned int offset = sizeof(struct trace_entry);
2394
2395                 for (; field->type; field++) {
2396                         if (field->type == TRACE_FUNCTION_TYPE) {
2397                                 field->define_fields(call);
2398                                 break;
2399                         }
2400
2401                         offset = ALIGN(offset, field->align);
2402                         ret = trace_define_field_ext(call, field->type, field->name,
2403                                                  offset, field->size,
2404                                                  field->is_signed, field->filter_type,
2405                                                  field->len);
2406                         if (WARN_ON_ONCE(ret)) {
2407                                 pr_err("error code is %d\n", ret);
2408                                 break;
2409                         }
2410
2411                         offset += field->size;
2412                 }
2413         }
2414
2415         return ret;
2416 }
2417
2418 static int
2419 event_create_dir(struct dentry *parent, struct trace_event_file *file)
2420 {
2421         struct trace_event_call *call = file->event_call;
2422         struct trace_array *tr = file->tr;
2423         struct dentry *d_events;
2424         const char *name;
2425         int ret;
2426
2427         /*
2428          * If the trace point header did not define TRACE_SYSTEM
2429          * then the system would be called "TRACE_SYSTEM".
2430          */
2431         if (strcmp(call->class->system, TRACE_SYSTEM) != 0) {
2432                 d_events = event_subsystem_dir(tr, call->class->system, file, parent);
2433                 if (!d_events)
2434                         return -ENOMEM;
2435         } else
2436                 d_events = parent;
2437
2438         name = trace_event_name(call);
2439         file->dir = tracefs_create_dir(name, d_events);
2440         if (!file->dir) {
2441                 pr_warn("Could not create tracefs '%s' directory\n", name);
2442                 return -1;
2443         }
2444
2445         if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
2446                 trace_create_file("enable", TRACE_MODE_WRITE, file->dir, file,
2447                                   &ftrace_enable_fops);
2448
2449 #ifdef CONFIG_PERF_EVENTS
2450         if (call->event.type && call->class->reg)
2451                 trace_create_file("id", TRACE_MODE_READ, file->dir,
2452                                   (void *)(long)call->event.type,
2453                                   &ftrace_event_id_fops);
2454 #endif
2455
2456         ret = event_define_fields(call);
2457         if (ret < 0) {
2458                 pr_warn("Could not initialize trace point events/%s\n", name);
2459                 return ret;
2460         }
2461
2462         /*
2463          * Only event directories that can be enabled should have
2464          * triggers or filters.
2465          */
2466         if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
2467                 trace_create_file("filter", TRACE_MODE_WRITE, file->dir,
2468                                   file, &ftrace_event_filter_fops);
2469
2470                 trace_create_file("trigger", TRACE_MODE_WRITE, file->dir,
2471                                   file, &event_trigger_fops);
2472         }
2473
2474 #ifdef CONFIG_HIST_TRIGGERS
2475         trace_create_file("hist", TRACE_MODE_READ, file->dir, file,
2476                           &event_hist_fops);
2477 #endif
2478 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
2479         trace_create_file("hist_debug", TRACE_MODE_READ, file->dir, file,
2480                           &event_hist_debug_fops);
2481 #endif
2482         trace_create_file("format", TRACE_MODE_READ, file->dir, call,
2483                           &ftrace_event_format_fops);
2484
2485 #ifdef CONFIG_TRACE_EVENT_INJECT
2486         if (call->event.type && call->class->reg)
2487                 trace_create_file("inject", 0200, file->dir, file,
2488                                   &event_inject_fops);
2489 #endif
2490
2491         return 0;
2492 }
2493
2494 static void remove_event_from_tracers(struct trace_event_call *call)
2495 {
2496         struct trace_event_file *file;
2497         struct trace_array *tr;
2498
2499         do_for_each_event_file_safe(tr, file) {
2500                 if (file->event_call != call)
2501                         continue;
2502
2503                 remove_event_file_dir(file);
2504                 /*
2505                  * The do_for_each_event_file_safe() is
2506                  * a double loop. After finding the call for this
2507                  * trace_array, we use break to jump to the next
2508                  * trace_array.
2509                  */
2510                 break;
2511         } while_for_each_event_file();
2512 }
2513
2514 static void event_remove(struct trace_event_call *call)
2515 {
2516         struct trace_array *tr;
2517         struct trace_event_file *file;
2518
2519         do_for_each_event_file(tr, file) {
2520                 if (file->event_call != call)
2521                         continue;
2522
2523                 if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
2524                         tr->clear_trace = true;
2525
2526                 ftrace_event_enable_disable(file, 0);
2527                 /*
2528                  * The do_for_each_event_file() is
2529                  * a double loop. After finding the call for this
2530                  * trace_array, we use break to jump to the next
2531                  * trace_array.
2532                  */
2533                 break;
2534         } while_for_each_event_file();
2535
2536         if (call->event.funcs)
2537                 __unregister_trace_event(&call->event);
2538         remove_event_from_tracers(call);
2539         list_del(&call->list);
2540 }
2541
2542 static int event_init(struct trace_event_call *call)
2543 {
2544         int ret = 0;
2545         const char *name;
2546
2547         name = trace_event_name(call);
2548         if (WARN_ON(!name))
2549                 return -EINVAL;
2550
2551         if (call->class->raw_init) {
2552                 ret = call->class->raw_init(call);
2553                 if (ret < 0 && ret != -ENOSYS)
2554                         pr_warn("Could not initialize trace events/%s\n", name);
2555         }
2556
2557         return ret;
2558 }
2559
2560 static int
2561 __register_event(struct trace_event_call *call, struct module *mod)
2562 {
2563         int ret;
2564
2565         ret = event_init(call);
2566         if (ret < 0)
2567                 return ret;
2568
2569         list_add(&call->list, &ftrace_events);
2570         if (call->flags & TRACE_EVENT_FL_DYNAMIC)
2571                 atomic_set(&call->refcnt, 0);
2572         else
2573                 call->module = mod;
2574
2575         return 0;
2576 }
2577
2578 static char *eval_replace(char *ptr, struct trace_eval_map *map, int len)
2579 {
2580         int rlen;
2581         int elen;
2582
2583         /* Find the length of the eval value as a string */
2584         elen = snprintf(ptr, 0, "%ld", map->eval_value);
2585         /* Make sure there's enough room to replace the string with the value */
2586         if (len < elen)
2587                 return NULL;
2588
2589         snprintf(ptr, elen + 1, "%ld", map->eval_value);
2590
2591         /* Get the rest of the string of ptr */
2592         rlen = strlen(ptr + len);
2593         memmove(ptr + elen, ptr + len, rlen);
2594         /* Make sure we end the new string */
2595         ptr[elen + rlen] = 0;
2596
2597         return ptr + elen;
2598 }
2599
2600 static void update_event_printk(struct trace_event_call *call,
2601                                 struct trace_eval_map *map)
2602 {
2603         char *ptr;
2604         int quote = 0;
2605         int len = strlen(map->eval_string);
2606
2607         for (ptr = call->print_fmt; *ptr; ptr++) {
2608                 if (*ptr == '\\') {
2609                         ptr++;
2610                         /* paranoid */
2611                         if (!*ptr)
2612                                 break;
2613                         continue;
2614                 }
2615                 if (*ptr == '"') {
2616                         quote ^= 1;
2617                         continue;
2618                 }
2619                 if (quote)
2620                         continue;
2621                 if (isdigit(*ptr)) {
2622                         /* skip numbers */
2623                         do {
2624                                 ptr++;
2625                                 /* Check for alpha chars like ULL */
2626                         } while (isalnum(*ptr));
2627                         if (!*ptr)
2628                                 break;
2629                         /*
2630                          * A number must have some kind of delimiter after
2631                          * it, and we can ignore that too.
2632                          */
2633                         continue;
2634                 }
2635                 if (isalpha(*ptr) || *ptr == '_') {
2636                         if (strncmp(map->eval_string, ptr, len) == 0 &&
2637                             !isalnum(ptr[len]) && ptr[len] != '_') {
2638                                 ptr = eval_replace(ptr, map, len);
2639                                 /* enum/sizeof string smaller than value */
2640                                 if (WARN_ON_ONCE(!ptr))
2641                                         return;
2642                                 /*
2643                                  * No need to decrement here, as eval_replace()
2644                                  * returns the pointer to the character passed
2645                                  * the eval, and two evals can not be placed
2646                                  * back to back without something in between.
2647                                  * We can skip that something in between.
2648                                  */
2649                                 continue;
2650                         }
2651                 skip_more:
2652                         do {
2653                                 ptr++;
2654                         } while (isalnum(*ptr) || *ptr == '_');
2655                         if (!*ptr)
2656                                 break;
2657                         /*
2658                          * If what comes after this variable is a '.' or
2659                          * '->' then we can continue to ignore that string.
2660                          */
2661                         if (*ptr == '.' || (ptr[0] == '-' && ptr[1] == '>')) {
2662                                 ptr += *ptr == '.' ? 1 : 2;
2663                                 if (!*ptr)
2664                                         break;
2665                                 goto skip_more;
2666                         }
2667                         /*
2668                          * Once again, we can skip the delimiter that came
2669                          * after the string.
2670                          */
2671                         continue;
2672                 }
2673         }
2674 }
2675
2676 static void add_str_to_module(struct module *module, char *str)
2677 {
2678         struct module_string *modstr;
2679
2680         modstr = kmalloc(sizeof(*modstr), GFP_KERNEL);
2681
2682         /*
2683          * If we failed to allocate memory here, then we'll just
2684          * let the str memory leak when the module is removed.
2685          * If this fails to allocate, there's worse problems than
2686          * a leaked string on module removal.
2687          */
2688         if (WARN_ON_ONCE(!modstr))
2689                 return;
2690
2691         modstr->module = module;
2692         modstr->str = str;
2693
2694         list_add(&modstr->next, &module_strings);
2695 }
2696
2697 static void update_event_fields(struct trace_event_call *call,
2698                                 struct trace_eval_map *map)
2699 {
2700         struct ftrace_event_field *field;
2701         struct list_head *head;
2702         char *ptr;
2703         char *str;
2704         int len = strlen(map->eval_string);
2705
2706         /* Dynamic events should never have field maps */
2707         if (WARN_ON_ONCE(call->flags & TRACE_EVENT_FL_DYNAMIC))
2708                 return;
2709
2710         head = trace_get_fields(call);
2711         list_for_each_entry(field, head, link) {
2712                 ptr = strchr(field->type, '[');
2713                 if (!ptr)
2714                         continue;
2715                 ptr++;
2716
2717                 if (!isalpha(*ptr) && *ptr != '_')
2718                         continue;
2719
2720                 if (strncmp(map->eval_string, ptr, len) != 0)
2721                         continue;
2722
2723                 str = kstrdup(field->type, GFP_KERNEL);
2724                 if (WARN_ON_ONCE(!str))
2725                         return;
2726                 ptr = str + (ptr - field->type);
2727                 ptr = eval_replace(ptr, map, len);
2728                 /* enum/sizeof string smaller than value */
2729                 if (WARN_ON_ONCE(!ptr)) {
2730                         kfree(str);
2731                         continue;
2732                 }
2733
2734                 /*
2735                  * If the event is part of a module, then we need to free the string
2736                  * when the module is removed. Otherwise, it will stay allocated
2737                  * until a reboot.
2738                  */
2739                 if (call->module)
2740                         add_str_to_module(call->module, str);
2741
2742                 field->type = str;
2743         }
2744 }
2745
2746 void trace_event_eval_update(struct trace_eval_map **map, int len)
2747 {
2748         struct trace_event_call *call, *p;
2749         const char *last_system = NULL;
2750         bool first = false;
2751         int last_i;
2752         int i;
2753
2754         down_write(&trace_event_sem);
2755         list_for_each_entry_safe(call, p, &ftrace_events, list) {
2756                 /* events are usually grouped together with systems */
2757                 if (!last_system || call->class->system != last_system) {
2758                         first = true;
2759                         last_i = 0;
2760                         last_system = call->class->system;
2761                 }
2762
2763                 /*
2764                  * Since calls are grouped by systems, the likelihood that the
2765                  * next call in the iteration belongs to the same system as the
2766                  * previous call is high. As an optimization, we skip searching
2767                  * for a map[] that matches the call's system if the last call
2768                  * was from the same system. That's what last_i is for. If the
2769                  * call has the same system as the previous call, then last_i
2770                  * will be the index of the first map[] that has a matching
2771                  * system.
2772                  */
2773                 for (i = last_i; i < len; i++) {
2774                         if (call->class->system == map[i]->system) {
2775                                 /* Save the first system if need be */
2776                                 if (first) {
2777                                         last_i = i;
2778                                         first = false;
2779                                 }
2780                                 update_event_printk(call, map[i]);
2781                                 update_event_fields(call, map[i]);
2782                         }
2783                 }
2784         }
2785         up_write(&trace_event_sem);
2786 }
2787
2788 static struct trace_event_file *
2789 trace_create_new_event(struct trace_event_call *call,
2790                        struct trace_array *tr)
2791 {
2792         struct trace_pid_list *no_pid_list;
2793         struct trace_pid_list *pid_list;
2794         struct trace_event_file *file;
2795         unsigned int first;
2796
2797         file = kmem_cache_alloc(file_cachep, GFP_TRACE);
2798         if (!file)
2799                 return NULL;
2800
2801         pid_list = rcu_dereference_protected(tr->filtered_pids,
2802                                              lockdep_is_held(&event_mutex));
2803         no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
2804                                              lockdep_is_held(&event_mutex));
2805
2806         if (!trace_pid_list_first(pid_list, &first) ||
2807             !trace_pid_list_first(no_pid_list, &first))
2808                 file->flags |= EVENT_FILE_FL_PID_FILTER;
2809
2810         file->event_call = call;
2811         file->tr = tr;
2812         atomic_set(&file->sm_ref, 0);
2813         atomic_set(&file->tm_ref, 0);
2814         INIT_LIST_HEAD(&file->triggers);
2815         list_add(&file->list, &tr->events);
2816
2817         return file;
2818 }
2819
2820 #define MAX_BOOT_TRIGGERS 32
2821
2822 static struct boot_triggers {
2823         const char              *event;
2824         char                    *trigger;
2825 } bootup_triggers[MAX_BOOT_TRIGGERS];
2826
2827 static char bootup_trigger_buf[COMMAND_LINE_SIZE];
2828 static int nr_boot_triggers;
2829
2830 static __init int setup_trace_triggers(char *str)
2831 {
2832         char *trigger;
2833         char *buf;
2834         int i;
2835
2836         strscpy(bootup_trigger_buf, str, COMMAND_LINE_SIZE);
2837         ring_buffer_expanded = true;
2838         disable_tracing_selftest("running event triggers");
2839
2840         buf = bootup_trigger_buf;
2841         for (i = 0; i < MAX_BOOT_TRIGGERS; i++) {
2842                 trigger = strsep(&buf, ",");
2843                 if (!trigger)
2844                         break;
2845                 bootup_triggers[i].event = strsep(&trigger, ".");
2846                 bootup_triggers[i].trigger = trigger;
2847                 if (!bootup_triggers[i].trigger)
2848                         break;
2849         }
2850
2851         nr_boot_triggers = i;
2852         return 1;
2853 }
2854 __setup("trace_trigger=", setup_trace_triggers);
2855
2856 /* Add an event to a trace directory */
2857 static int
2858 __trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)
2859 {
2860         struct trace_event_file *file;
2861
2862         file = trace_create_new_event(call, tr);
2863         if (!file)
2864                 return -ENOMEM;
2865
2866         if (eventdir_initialized)
2867                 return event_create_dir(tr->event_dir, file);
2868         else
2869                 return event_define_fields(call);
2870 }
2871
2872 static void trace_early_triggers(struct trace_event_file *file, const char *name)
2873 {
2874         int ret;
2875         int i;
2876
2877         for (i = 0; i < nr_boot_triggers; i++) {
2878                 if (strcmp(name, bootup_triggers[i].event))
2879                         continue;
2880                 mutex_lock(&event_mutex);
2881                 ret = trigger_process_regex(file, bootup_triggers[i].trigger);
2882                 mutex_unlock(&event_mutex);
2883                 if (ret)
2884                         pr_err("Failed to register trigger '%s' on event %s\n",
2885                                bootup_triggers[i].trigger,
2886                                bootup_triggers[i].event);
2887         }
2888 }
2889
2890 /*
2891  * Just create a descriptor for early init. A descriptor is required
2892  * for enabling events at boot. We want to enable events before
2893  * the filesystem is initialized.
2894  */
2895 static int
2896 __trace_early_add_new_event(struct trace_event_call *call,
2897                             struct trace_array *tr)
2898 {
2899         struct trace_event_file *file;
2900         int ret;
2901
2902         file = trace_create_new_event(call, tr);
2903         if (!file)
2904                 return -ENOMEM;
2905
2906         ret = event_define_fields(call);
2907         if (ret)
2908                 return ret;
2909
2910         trace_early_triggers(file, trace_event_name(call));
2911
2912         return 0;
2913 }
2914
2915 struct ftrace_module_file_ops;
2916 static void __add_event_to_tracers(struct trace_event_call *call);
2917
2918 /* Add an additional event_call dynamically */
2919 int trace_add_event_call(struct trace_event_call *call)
2920 {
2921         int ret;
2922         lockdep_assert_held(&event_mutex);
2923
2924         mutex_lock(&trace_types_lock);
2925
2926         ret = __register_event(call, NULL);
2927         if (ret >= 0)
2928                 __add_event_to_tracers(call);
2929
2930         mutex_unlock(&trace_types_lock);
2931         return ret;
2932 }
2933 EXPORT_SYMBOL_GPL(trace_add_event_call);
2934
2935 /*
2936  * Must be called under locking of trace_types_lock, event_mutex and
2937  * trace_event_sem.
2938  */
2939 static void __trace_remove_event_call(struct trace_event_call *call)
2940 {
2941         event_remove(call);
2942         trace_destroy_fields(call);
2943         free_event_filter(call->filter);
2944         call->filter = NULL;
2945 }
2946
2947 static int probe_remove_event_call(struct trace_event_call *call)
2948 {
2949         struct trace_array *tr;
2950         struct trace_event_file *file;
2951
2952 #ifdef CONFIG_PERF_EVENTS
2953         if (call->perf_refcount)
2954                 return -EBUSY;
2955 #endif
2956         do_for_each_event_file(tr, file) {
2957                 if (file->event_call != call)
2958                         continue;
2959                 /*
2960                  * We can't rely on ftrace_event_enable_disable(enable => 0)
2961                  * we are going to do, EVENT_FILE_FL_SOFT_MODE can suppress
2962                  * TRACE_REG_UNREGISTER.
2963                  */
2964                 if (file->flags & EVENT_FILE_FL_ENABLED)
2965                         goto busy;
2966
2967                 if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
2968                         tr->clear_trace = true;
2969                 /*
2970                  * The do_for_each_event_file_safe() is
2971                  * a double loop. After finding the call for this
2972                  * trace_array, we use break to jump to the next
2973                  * trace_array.
2974                  */
2975                 break;
2976         } while_for_each_event_file();
2977
2978         __trace_remove_event_call(call);
2979
2980         return 0;
2981  busy:
2982         /* No need to clear the trace now */
2983         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2984                 tr->clear_trace = false;
2985         }
2986         return -EBUSY;
2987 }
2988
2989 /* Remove an event_call */
2990 int trace_remove_event_call(struct trace_event_call *call)
2991 {
2992         int ret;
2993
2994         lockdep_assert_held(&event_mutex);
2995
2996         mutex_lock(&trace_types_lock);
2997         down_write(&trace_event_sem);
2998         ret = probe_remove_event_call(call);
2999         up_write(&trace_event_sem);
3000         mutex_unlock(&trace_types_lock);
3001
3002         return ret;
3003 }
3004 EXPORT_SYMBOL_GPL(trace_remove_event_call);
3005
3006 #define for_each_event(event, start, end)                       \
3007         for (event = start;                                     \
3008              (unsigned long)event < (unsigned long)end;         \
3009              event++)
3010
3011 #ifdef CONFIG_MODULES
3012
3013 static void trace_module_add_events(struct module *mod)
3014 {
3015         struct trace_event_call **call, **start, **end;
3016
3017         if (!mod->num_trace_events)
3018                 return;
3019
3020         /* Don't add infrastructure for mods without tracepoints */
3021         if (trace_module_has_bad_taint(mod)) {
3022                 pr_err("%s: module has bad taint, not creating trace events\n",
3023                        mod->name);
3024                 return;
3025         }
3026
3027         start = mod->trace_events;
3028         end = mod->trace_events + mod->num_trace_events;
3029
3030         for_each_event(call, start, end) {
3031                 __register_event(*call, mod);
3032                 __add_event_to_tracers(*call);
3033         }
3034 }
3035
3036 static void trace_module_remove_events(struct module *mod)
3037 {
3038         struct trace_event_call *call, *p;
3039         struct module_string *modstr, *m;
3040
3041         down_write(&trace_event_sem);
3042         list_for_each_entry_safe(call, p, &ftrace_events, list) {
3043                 if ((call->flags & TRACE_EVENT_FL_DYNAMIC) || !call->module)
3044                         continue;
3045                 if (call->module == mod)
3046                         __trace_remove_event_call(call);
3047         }
3048         /* Check for any strings allocade for this module */
3049         list_for_each_entry_safe(modstr, m, &module_strings, next) {
3050                 if (modstr->module != mod)
3051                         continue;
3052                 list_del(&modstr->next);
3053                 kfree(modstr->str);
3054                 kfree(modstr);
3055         }
3056         up_write(&trace_event_sem);
3057
3058         /*
3059          * It is safest to reset the ring buffer if the module being unloaded
3060          * registered any events that were used. The only worry is if
3061          * a new module gets loaded, and takes on the same id as the events
3062          * of this module. When printing out the buffer, traced events left
3063          * over from this module may be passed to the new module events and
3064          * unexpected results may occur.
3065          */
3066         tracing_reset_all_online_cpus_unlocked();
3067 }
3068
3069 static int trace_module_notify(struct notifier_block *self,
3070                                unsigned long val, void *data)
3071 {
3072         struct module *mod = data;
3073
3074         mutex_lock(&event_mutex);
3075         mutex_lock(&trace_types_lock);
3076         switch (val) {
3077         case MODULE_STATE_COMING:
3078                 trace_module_add_events(mod);
3079                 break;
3080         case MODULE_STATE_GOING:
3081                 trace_module_remove_events(mod);
3082                 break;
3083         }
3084         mutex_unlock(&trace_types_lock);
3085         mutex_unlock(&event_mutex);
3086
3087         return NOTIFY_OK;
3088 }
3089
3090 static struct notifier_block trace_module_nb = {
3091         .notifier_call = trace_module_notify,
3092         .priority = 1, /* higher than trace.c module notify */
3093 };
3094 #endif /* CONFIG_MODULES */
3095
3096 /* Create a new event directory structure for a trace directory. */
3097 static void
3098 __trace_add_event_dirs(struct trace_array *tr)
3099 {
3100         struct trace_event_call *call;
3101         int ret;
3102
3103         list_for_each_entry(call, &ftrace_events, list) {
3104                 ret = __trace_add_new_event(call, tr);
3105                 if (ret < 0)
3106                         pr_warn("Could not create directory for event %s\n",
3107                                 trace_event_name(call));
3108         }
3109 }
3110
3111 /* Returns any file that matches the system and event */
3112 struct trace_event_file *
3113 __find_event_file(struct trace_array *tr, const char *system, const char *event)
3114 {
3115         struct trace_event_file *file;
3116         struct trace_event_call *call;
3117         const char *name;
3118
3119         list_for_each_entry(file, &tr->events, list) {
3120
3121                 call = file->event_call;
3122                 name = trace_event_name(call);
3123
3124                 if (!name || !call->class)
3125                         continue;
3126
3127                 if (strcmp(event, name) == 0 &&
3128                     strcmp(system, call->class->system) == 0)
3129                         return file;
3130         }
3131         return NULL;
3132 }
3133
3134 /* Returns valid trace event files that match system and event */
3135 struct trace_event_file *
3136 find_event_file(struct trace_array *tr, const char *system, const char *event)
3137 {
3138         struct trace_event_file *file;
3139
3140         file = __find_event_file(tr, system, event);
3141         if (!file || !file->event_call->class->reg ||
3142             file->event_call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
3143                 return NULL;
3144
3145         return file;
3146 }
3147
3148 /**
3149  * trace_get_event_file - Find and return a trace event file
3150  * @instance: The name of the trace instance containing the event
3151  * @system: The name of the system containing the event
3152  * @event: The name of the event
3153  *
3154  * Return a trace event file given the trace instance name, trace
3155  * system, and trace event name.  If the instance name is NULL, it
3156  * refers to the top-level trace array.
3157  *
3158  * This function will look it up and return it if found, after calling
3159  * trace_array_get() to prevent the instance from going away, and
3160  * increment the event's module refcount to prevent it from being
3161  * removed.
3162  *
3163  * To release the file, call trace_put_event_file(), which will call
3164  * trace_array_put() and decrement the event's module refcount.
3165  *
3166  * Return: The trace event on success, ERR_PTR otherwise.
3167  */
3168 struct trace_event_file *trace_get_event_file(const char *instance,
3169                                               const char *system,
3170                                               const char *event)
3171 {
3172         struct trace_array *tr = top_trace_array();
3173         struct trace_event_file *file = NULL;
3174         int ret = -EINVAL;
3175
3176         if (instance) {
3177                 tr = trace_array_find_get(instance);
3178                 if (!tr)
3179                         return ERR_PTR(-ENOENT);
3180         } else {
3181                 ret = trace_array_get(tr);
3182                 if (ret)
3183                         return ERR_PTR(ret);
3184         }
3185
3186         mutex_lock(&event_mutex);
3187
3188         file = find_event_file(tr, system, event);
3189         if (!file) {
3190                 trace_array_put(tr);
3191                 ret = -EINVAL;
3192                 goto out;
3193         }
3194
3195         /* Don't let event modules unload while in use */
3196         ret = trace_event_try_get_ref(file->event_call);
3197         if (!ret) {
3198                 trace_array_put(tr);
3199                 ret = -EBUSY;
3200                 goto out;
3201         }
3202
3203         ret = 0;
3204  out:
3205         mutex_unlock(&event_mutex);
3206
3207         if (ret)
3208                 file = ERR_PTR(ret);
3209
3210         return file;
3211 }
3212 EXPORT_SYMBOL_GPL(trace_get_event_file);
3213
3214 /**
3215  * trace_put_event_file - Release a file from trace_get_event_file()
3216  * @file: The trace event file
3217  *
3218  * If a file was retrieved using trace_get_event_file(), this should
3219  * be called when it's no longer needed.  It will cancel the previous
3220  * trace_array_get() called by that function, and decrement the
3221  * event's module refcount.
3222  */
3223 void trace_put_event_file(struct trace_event_file *file)
3224 {
3225         mutex_lock(&event_mutex);
3226         trace_event_put_ref(file->event_call);
3227         mutex_unlock(&event_mutex);
3228
3229         trace_array_put(file->tr);
3230 }
3231 EXPORT_SYMBOL_GPL(trace_put_event_file);
3232
3233 #ifdef CONFIG_DYNAMIC_FTRACE
3234
3235 /* Avoid typos */
3236 #define ENABLE_EVENT_STR        "enable_event"
3237 #define DISABLE_EVENT_STR       "disable_event"
3238
3239 struct event_probe_data {
3240         struct trace_event_file *file;
3241         unsigned long                   count;
3242         int                             ref;
3243         bool                            enable;
3244 };
3245
3246 static void update_event_probe(struct event_probe_data *data)
3247 {
3248         if (data->enable)
3249                 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3250         else
3251                 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3252 }
3253
3254 static void
3255 event_enable_probe(unsigned long ip, unsigned long parent_ip,
3256                    struct trace_array *tr, struct ftrace_probe_ops *ops,
3257                    void *data)
3258 {
3259         struct ftrace_func_mapper *mapper = data;
3260         struct event_probe_data *edata;
3261         void **pdata;
3262
3263         pdata = ftrace_func_mapper_find_ip(mapper, ip);
3264         if (!pdata || !*pdata)
3265                 return;
3266
3267         edata = *pdata;
3268         update_event_probe(edata);
3269 }
3270
3271 static void
3272 event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
3273                          struct trace_array *tr, struct ftrace_probe_ops *ops,
3274                          void *data)
3275 {
3276         struct ftrace_func_mapper *mapper = data;
3277         struct event_probe_data *edata;
3278         void **pdata;
3279
3280         pdata = ftrace_func_mapper_find_ip(mapper, ip);
3281         if (!pdata || !*pdata)
3282                 return;
3283
3284         edata = *pdata;
3285
3286         if (!edata->count)
3287                 return;
3288
3289         /* Skip if the event is in a state we want to switch to */
3290         if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
3291                 return;
3292
3293         if (edata->count != -1)
3294                 (edata->count)--;
3295
3296         update_event_probe(edata);
3297 }
3298
3299 static int
3300 event_enable_print(struct seq_file *m, unsigned long ip,
3301                    struct ftrace_probe_ops *ops, void *data)
3302 {
3303         struct ftrace_func_mapper *mapper = data;
3304         struct event_probe_data *edata;
3305         void **pdata;
3306
3307         pdata = ftrace_func_mapper_find_ip(mapper, ip);
3308
3309         if (WARN_ON_ONCE(!pdata || !*pdata))
3310                 return 0;
3311
3312         edata = *pdata;
3313
3314         seq_printf(m, "%ps:", (void *)ip);
3315
3316         seq_printf(m, "%s:%s:%s",
3317                    edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
3318                    edata->file->event_call->class->system,
3319                    trace_event_name(edata->file->event_call));
3320
3321         if (edata->count == -1)
3322                 seq_puts(m, ":unlimited\n");
3323         else
3324                 seq_printf(m, ":count=%ld\n", edata->count);
3325
3326         return 0;
3327 }
3328
3329 static int
3330 event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
3331                   unsigned long ip, void *init_data, void **data)
3332 {
3333         struct ftrace_func_mapper *mapper = *data;
3334         struct event_probe_data *edata = init_data;
3335         int ret;
3336
3337         if (!mapper) {
3338                 mapper = allocate_ftrace_func_mapper();
3339                 if (!mapper)
3340                         return -ENODEV;
3341                 *data = mapper;
3342         }
3343
3344         ret = ftrace_func_mapper_add_ip(mapper, ip, edata);
3345         if (ret < 0)
3346                 return ret;
3347
3348         edata->ref++;
3349
3350         return 0;
3351 }
3352
3353 static int free_probe_data(void *data)
3354 {
3355         struct event_probe_data *edata = data;
3356
3357         edata->ref--;
3358         if (!edata->ref) {
3359                 /* Remove the SOFT_MODE flag */
3360                 __ftrace_event_enable_disable(edata->file, 0, 1);
3361                 trace_event_put_ref(edata->file->event_call);
3362                 kfree(edata);
3363         }
3364         return 0;
3365 }
3366
3367 static void
3368 event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
3369                   unsigned long ip, void *data)
3370 {
3371         struct ftrace_func_mapper *mapper = data;
3372         struct event_probe_data *edata;
3373
3374         if (!ip) {
3375                 if (!mapper)
3376                         return;
3377                 free_ftrace_func_mapper(mapper, free_probe_data);
3378                 return;
3379         }
3380
3381         edata = ftrace_func_mapper_remove_ip(mapper, ip);
3382
3383         if (WARN_ON_ONCE(!edata))
3384                 return;
3385
3386         if (WARN_ON_ONCE(edata->ref <= 0))
3387                 return;
3388
3389         free_probe_data(edata);
3390 }
3391
3392 static struct ftrace_probe_ops event_enable_probe_ops = {
3393         .func                   = event_enable_probe,
3394         .print                  = event_enable_print,
3395         .init                   = event_enable_init,
3396         .free                   = event_enable_free,
3397 };
3398
3399 static struct ftrace_probe_ops event_enable_count_probe_ops = {
3400         .func                   = event_enable_count_probe,
3401         .print                  = event_enable_print,
3402         .init                   = event_enable_init,
3403         .free                   = event_enable_free,
3404 };
3405
3406 static struct ftrace_probe_ops event_disable_probe_ops = {
3407         .func                   = event_enable_probe,
3408         .print                  = event_enable_print,
3409         .init                   = event_enable_init,
3410         .free                   = event_enable_free,
3411 };
3412
3413 static struct ftrace_probe_ops event_disable_count_probe_ops = {
3414         .func                   = event_enable_count_probe,
3415         .print                  = event_enable_print,
3416         .init                   = event_enable_init,
3417         .free                   = event_enable_free,
3418 };
3419
3420 static int
3421 event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,
3422                   char *glob, char *cmd, char *param, int enabled)
3423 {
3424         struct trace_event_file *file;
3425         struct ftrace_probe_ops *ops;
3426         struct event_probe_data *data;
3427         const char *system;
3428         const char *event;
3429         char *number;
3430         bool enable;
3431         int ret;
3432
3433         if (!tr)
3434                 return -ENODEV;
3435
3436         /* hash funcs only work with set_ftrace_filter */
3437         if (!enabled || !param)
3438                 return -EINVAL;
3439
3440         system = strsep(&param, ":");
3441         if (!param)
3442                 return -EINVAL;
3443
3444         event = strsep(&param, ":");
3445
3446         mutex_lock(&event_mutex);
3447
3448         ret = -EINVAL;
3449         file = find_event_file(tr, system, event);
3450         if (!file)
3451                 goto out;
3452
3453         enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
3454
3455         if (enable)
3456                 ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
3457         else
3458                 ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
3459
3460         if (glob[0] == '!') {
3461                 ret = unregister_ftrace_function_probe_func(glob+1, tr, ops);
3462                 goto out;
3463         }
3464
3465         ret = -ENOMEM;
3466
3467         data = kzalloc(sizeof(*data), GFP_KERNEL);
3468         if (!data)
3469                 goto out;
3470
3471         data->enable = enable;
3472         data->count = -1;
3473         data->file = file;
3474
3475         if (!param)
3476                 goto out_reg;
3477
3478         number = strsep(&param, ":");
3479
3480         ret = -EINVAL;
3481         if (!strlen(number))
3482                 goto out_free;
3483
3484         /*
3485          * We use the callback data field (which is a pointer)
3486          * as our counter.
3487          */
3488         ret = kstrtoul(number, 0, &data->count);
3489         if (ret)
3490                 goto out_free;
3491
3492  out_reg:
3493         /* Don't let event modules unload while probe registered */
3494         ret = trace_event_try_get_ref(file->event_call);
3495         if (!ret) {
3496                 ret = -EBUSY;
3497                 goto out_free;
3498         }
3499
3500         ret = __ftrace_event_enable_disable(file, 1, 1);
3501         if (ret < 0)
3502                 goto out_put;
3503
3504         ret = register_ftrace_function_probe(glob, tr, ops, data);
3505         /*
3506          * The above returns on success the # of functions enabled,
3507          * but if it didn't find any functions it returns zero.
3508          * Consider no functions a failure too.
3509          */
3510         if (!ret) {
3511                 ret = -ENOENT;
3512                 goto out_disable;
3513         } else if (ret < 0)
3514                 goto out_disable;
3515         /* Just return zero, not the number of enabled functions */
3516         ret = 0;
3517  out:
3518         mutex_unlock(&event_mutex);
3519         return ret;
3520
3521  out_disable:
3522         __ftrace_event_enable_disable(file, 0, 1);
3523  out_put:
3524         trace_event_put_ref(file->event_call);
3525  out_free:
3526         kfree(data);
3527         goto out;
3528 }
3529
3530 static struct ftrace_func_command event_enable_cmd = {
3531         .name                   = ENABLE_EVENT_STR,
3532         .func                   = event_enable_func,
3533 };
3534
3535 static struct ftrace_func_command event_disable_cmd = {
3536         .name                   = DISABLE_EVENT_STR,
3537         .func                   = event_enable_func,
3538 };
3539
3540 static __init int register_event_cmds(void)
3541 {
3542         int ret;
3543
3544         ret = register_ftrace_command(&event_enable_cmd);
3545         if (WARN_ON(ret < 0))
3546                 return ret;
3547         ret = register_ftrace_command(&event_disable_cmd);
3548         if (WARN_ON(ret < 0))
3549                 unregister_ftrace_command(&event_enable_cmd);
3550         return ret;
3551 }
3552 #else
3553 static inline int register_event_cmds(void) { return 0; }
3554 #endif /* CONFIG_DYNAMIC_FTRACE */
3555
3556 /*
3557  * The top level array and trace arrays created by boot-time tracing
3558  * have already had its trace_event_file descriptors created in order
3559  * to allow for early events to be recorded.
3560  * This function is called after the tracefs has been initialized,
3561  * and we now have to create the files associated to the events.
3562  */
3563 static void __trace_early_add_event_dirs(struct trace_array *tr)
3564 {
3565         struct trace_event_file *file;
3566         int ret;
3567
3568
3569         list_for_each_entry(file, &tr->events, list) {
3570                 ret = event_create_dir(tr->event_dir, file);
3571                 if (ret < 0)
3572                         pr_warn("Could not create directory for event %s\n",
3573                                 trace_event_name(file->event_call));
3574         }
3575 }
3576
3577 /*
3578  * For early boot up, the top trace array and the trace arrays created
3579  * by boot-time tracing require to have a list of events that can be
3580  * enabled. This must be done before the filesystem is set up in order
3581  * to allow events to be traced early.
3582  */
3583 void __trace_early_add_events(struct trace_array *tr)
3584 {
3585         struct trace_event_call *call;
3586         int ret;
3587
3588         list_for_each_entry(call, &ftrace_events, list) {
3589                 /* Early boot up should not have any modules loaded */
3590                 if (!(call->flags & TRACE_EVENT_FL_DYNAMIC) &&
3591                     WARN_ON_ONCE(call->module))
3592                         continue;
3593
3594                 ret = __trace_early_add_new_event(call, tr);
3595                 if (ret < 0)
3596                         pr_warn("Could not create early event %s\n",
3597                                 trace_event_name(call));
3598         }
3599 }
3600
3601 /* Remove the event directory structure for a trace directory. */
3602 static void
3603 __trace_remove_event_dirs(struct trace_array *tr)
3604 {
3605         struct trace_event_file *file, *next;
3606
3607         list_for_each_entry_safe(file, next, &tr->events, list)
3608                 remove_event_file_dir(file);
3609 }
3610
3611 static void __add_event_to_tracers(struct trace_event_call *call)
3612 {
3613         struct trace_array *tr;
3614
3615         list_for_each_entry(tr, &ftrace_trace_arrays, list)
3616                 __trace_add_new_event(call, tr);
3617 }
3618
3619 extern struct trace_event_call *__start_ftrace_events[];
3620 extern struct trace_event_call *__stop_ftrace_events[];
3621
3622 static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
3623
3624 static __init int setup_trace_event(char *str)
3625 {
3626         strscpy(bootup_event_buf, str, COMMAND_LINE_SIZE);
3627         ring_buffer_expanded = true;
3628         disable_tracing_selftest("running event tracing");
3629
3630         return 1;
3631 }
3632 __setup("trace_event=", setup_trace_event);
3633
3634 /* Expects to have event_mutex held when called */
3635 static int
3636 create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
3637 {
3638         struct dentry *d_events;
3639         struct dentry *entry;
3640
3641         entry = trace_create_file("set_event", TRACE_MODE_WRITE, parent,
3642                                   tr, &ftrace_set_event_fops);
3643         if (!entry)
3644                 return -ENOMEM;
3645
3646         d_events = tracefs_create_dir("events", parent);
3647         if (!d_events) {
3648                 pr_warn("Could not create tracefs 'events' directory\n");
3649                 return -ENOMEM;
3650         }
3651
3652         entry = trace_create_file("enable", TRACE_MODE_WRITE, d_events,
3653                                   tr, &ftrace_tr_enable_fops);
3654         if (!entry)
3655                 return -ENOMEM;
3656
3657         /* There are not as crucial, just warn if they are not created */
3658
3659         trace_create_file("set_event_pid", TRACE_MODE_WRITE, parent,
3660                           tr, &ftrace_set_event_pid_fops);
3661
3662         trace_create_file("set_event_notrace_pid",
3663                           TRACE_MODE_WRITE, parent, tr,
3664                           &ftrace_set_event_notrace_pid_fops);
3665
3666         /* ring buffer internal formats */
3667         trace_create_file("header_page", TRACE_MODE_READ, d_events,
3668                                   ring_buffer_print_page_header,
3669                                   &ftrace_show_header_fops);
3670
3671         trace_create_file("header_event", TRACE_MODE_READ, d_events,
3672                                   ring_buffer_print_entry_header,
3673                                   &ftrace_show_header_fops);
3674
3675         tr->event_dir = d_events;
3676
3677         return 0;
3678 }
3679
3680 /**
3681  * event_trace_add_tracer - add a instance of a trace_array to events
3682  * @parent: The parent dentry to place the files/directories for events in
3683  * @tr: The trace array associated with these events
3684  *
3685  * When a new instance is created, it needs to set up its events
3686  * directory, as well as other files associated with events. It also
3687  * creates the event hierarchy in the @parent/events directory.
3688  *
3689  * Returns 0 on success.
3690  *
3691  * Must be called with event_mutex held.
3692  */
3693 int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)
3694 {
3695         int ret;
3696
3697         lockdep_assert_held(&event_mutex);
3698
3699         ret = create_event_toplevel_files(parent, tr);
3700         if (ret)
3701                 goto out;
3702
3703         down_write(&trace_event_sem);
3704         /* If tr already has the event list, it is initialized in early boot. */
3705         if (unlikely(!list_empty(&tr->events)))
3706                 __trace_early_add_event_dirs(tr);
3707         else
3708                 __trace_add_event_dirs(tr);
3709         up_write(&trace_event_sem);
3710
3711  out:
3712         return ret;
3713 }
3714
3715 /*
3716  * The top trace array already had its file descriptors created.
3717  * Now the files themselves need to be created.
3718  */
3719 static __init int
3720 early_event_add_tracer(struct dentry *parent, struct trace_array *tr)
3721 {
3722         int ret;
3723
3724         mutex_lock(&event_mutex);
3725
3726         ret = create_event_toplevel_files(parent, tr);
3727         if (ret)
3728                 goto out_unlock;
3729
3730         down_write(&trace_event_sem);
3731         __trace_early_add_event_dirs(tr);
3732         up_write(&trace_event_sem);
3733
3734  out_unlock:
3735         mutex_unlock(&event_mutex);
3736
3737         return ret;
3738 }
3739
3740 /* Must be called with event_mutex held */
3741 int event_trace_del_tracer(struct trace_array *tr)
3742 {
3743         lockdep_assert_held(&event_mutex);
3744
3745         /* Disable any event triggers and associated soft-disabled events */
3746         clear_event_triggers(tr);
3747
3748         /* Clear the pid list */
3749         __ftrace_clear_event_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
3750
3751         /* Disable any running events */
3752         __ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0);
3753
3754         /* Make sure no more events are being executed */
3755         tracepoint_synchronize_unregister();
3756
3757         down_write(&trace_event_sem);
3758         __trace_remove_event_dirs(tr);
3759         tracefs_remove(tr->event_dir);
3760         up_write(&trace_event_sem);
3761
3762         tr->event_dir = NULL;
3763
3764         return 0;
3765 }
3766
3767 static __init int event_trace_memsetup(void)
3768 {
3769         field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
3770         file_cachep = KMEM_CACHE(trace_event_file, SLAB_PANIC);
3771         return 0;
3772 }
3773
3774 __init void
3775 early_enable_events(struct trace_array *tr, char *buf, bool disable_first)
3776 {
3777         char *token;
3778         int ret;
3779
3780         while (true) {
3781                 token = strsep(&buf, ",");
3782
3783                 if (!token)
3784                         break;
3785
3786                 if (*token) {
3787                         /* Restarting syscalls requires that we stop them first */
3788                         if (disable_first)
3789                                 ftrace_set_clr_event(tr, token, 0);
3790
3791                         ret = ftrace_set_clr_event(tr, token, 1);
3792                         if (ret)
3793                                 pr_warn("Failed to enable trace event: %s\n", token);
3794                 }
3795
3796                 /* Put back the comma to allow this to be called again */
3797                 if (buf)
3798                         *(buf - 1) = ',';
3799         }
3800 }
3801
3802 static __init int event_trace_enable(void)
3803 {
3804         struct trace_array *tr = top_trace_array();
3805         struct trace_event_call **iter, *call;
3806         int ret;
3807
3808         if (!tr)
3809                 return -ENODEV;
3810
3811         for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
3812
3813                 call = *iter;
3814                 ret = event_init(call);
3815                 if (!ret)
3816                         list_add(&call->list, &ftrace_events);
3817         }
3818
3819         register_trigger_cmds();
3820
3821         /*
3822          * We need the top trace array to have a working set of trace
3823          * points at early init, before the debug files and directories
3824          * are created. Create the file entries now, and attach them
3825          * to the actual file dentries later.
3826          */
3827         __trace_early_add_events(tr);
3828
3829         early_enable_events(tr, bootup_event_buf, false);
3830
3831         trace_printk_start_comm();
3832
3833         register_event_cmds();
3834
3835
3836         return 0;
3837 }
3838
3839 /*
3840  * event_trace_enable() is called from trace_event_init() first to
3841  * initialize events and perhaps start any events that are on the
3842  * command line. Unfortunately, there are some events that will not
3843  * start this early, like the system call tracepoints that need
3844  * to set the %SYSCALL_WORK_SYSCALL_TRACEPOINT flag of pid 1. But
3845  * event_trace_enable() is called before pid 1 starts, and this flag
3846  * is never set, making the syscall tracepoint never get reached, but
3847  * the event is enabled regardless (and not doing anything).
3848  */
3849 static __init int event_trace_enable_again(void)
3850 {
3851         struct trace_array *tr;
3852
3853         tr = top_trace_array();
3854         if (!tr)
3855                 return -ENODEV;
3856
3857         early_enable_events(tr, bootup_event_buf, true);
3858
3859         return 0;
3860 }
3861
3862 early_initcall(event_trace_enable_again);
3863
3864 /* Init fields which doesn't related to the tracefs */
3865 static __init int event_trace_init_fields(void)
3866 {
3867         if (trace_define_generic_fields())
3868                 pr_warn("tracing: Failed to allocated generic fields");
3869
3870         if (trace_define_common_fields())
3871                 pr_warn("tracing: Failed to allocate common fields");
3872
3873         return 0;
3874 }
3875
3876 __init int event_trace_init(void)
3877 {
3878         struct trace_array *tr;
3879         int ret;
3880
3881         tr = top_trace_array();
3882         if (!tr)
3883                 return -ENODEV;
3884
3885         trace_create_file("available_events", TRACE_MODE_READ,
3886                           NULL, tr, &ftrace_avail_fops);
3887
3888         ret = early_event_add_tracer(NULL, tr);
3889         if (ret)
3890                 return ret;
3891
3892 #ifdef CONFIG_MODULES
3893         ret = register_module_notifier(&trace_module_nb);
3894         if (ret)
3895                 pr_warn("Failed to register trace events module notifier\n");
3896 #endif
3897
3898         eventdir_initialized = true;
3899
3900         return 0;
3901 }
3902
3903 void __init trace_event_init(void)
3904 {
3905         event_trace_memsetup();
3906         init_ftrace_syscalls();
3907         event_trace_enable();
3908         event_trace_init_fields();
3909 }
3910
3911 #ifdef CONFIG_EVENT_TRACE_STARTUP_TEST
3912
3913 static DEFINE_SPINLOCK(test_spinlock);
3914 static DEFINE_SPINLOCK(test_spinlock_irq);
3915 static DEFINE_MUTEX(test_mutex);
3916
3917 static __init void test_work(struct work_struct *dummy)
3918 {
3919         spin_lock(&test_spinlock);
3920         spin_lock_irq(&test_spinlock_irq);
3921         udelay(1);
3922         spin_unlock_irq(&test_spinlock_irq);
3923         spin_unlock(&test_spinlock);
3924
3925         mutex_lock(&test_mutex);
3926         msleep(1);
3927         mutex_unlock(&test_mutex);
3928 }
3929
3930 static __init int event_test_thread(void *unused)
3931 {
3932         void *test_malloc;
3933
3934         test_malloc = kmalloc(1234, GFP_KERNEL);
3935         if (!test_malloc)
3936                 pr_info("failed to kmalloc\n");
3937
3938         schedule_on_each_cpu(test_work);
3939
3940         kfree(test_malloc);
3941
3942         set_current_state(TASK_INTERRUPTIBLE);
3943         while (!kthread_should_stop()) {
3944                 schedule();
3945                 set_current_state(TASK_INTERRUPTIBLE);
3946         }
3947         __set_current_state(TASK_RUNNING);
3948
3949         return 0;
3950 }
3951
3952 /*
3953  * Do various things that may trigger events.
3954  */
3955 static __init void event_test_stuff(void)
3956 {
3957         struct task_struct *test_thread;
3958
3959         test_thread = kthread_run(event_test_thread, NULL, "test-events");
3960         msleep(1);
3961         kthread_stop(test_thread);
3962 }
3963
3964 /*
3965  * For every trace event defined, we will test each trace point separately,
3966  * and then by groups, and finally all trace points.
3967  */
3968 static __init void event_trace_self_tests(void)
3969 {
3970         struct trace_subsystem_dir *dir;
3971         struct trace_event_file *file;
3972         struct trace_event_call *call;
3973         struct event_subsystem *system;
3974         struct trace_array *tr;
3975         int ret;
3976
3977         tr = top_trace_array();
3978         if (!tr)
3979                 return;
3980
3981         pr_info("Running tests on trace events:\n");
3982
3983         list_for_each_entry(file, &tr->events, list) {
3984
3985                 call = file->event_call;
3986
3987                 /* Only test those that have a probe */
3988                 if (!call->class || !call->class->probe)
3989                         continue;
3990
3991 /*
3992  * Testing syscall events here is pretty useless, but
3993  * we still do it if configured. But this is time consuming.
3994  * What we really need is a user thread to perform the
3995  * syscalls as we test.
3996  */
3997 #ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
3998                 if (call->class->system &&
3999                     strcmp(call->class->system, "syscalls") == 0)
4000                         continue;
4001 #endif
4002
4003                 pr_info("Testing event %s: ", trace_event_name(call));
4004
4005                 /*
4006                  * If an event is already enabled, someone is using
4007                  * it and the self test should not be on.
4008                  */
4009                 if (file->flags & EVENT_FILE_FL_ENABLED) {
4010                         pr_warn("Enabled event during self test!\n");
4011                         WARN_ON_ONCE(1);
4012                         continue;
4013                 }
4014
4015                 ftrace_event_enable_disable(file, 1);
4016                 event_test_stuff();
4017                 ftrace_event_enable_disable(file, 0);
4018
4019                 pr_cont("OK\n");
4020         }
4021
4022         /* Now test at the sub system level */
4023
4024         pr_info("Running tests on trace event systems:\n");
4025
4026         list_for_each_entry(dir, &tr->systems, list) {
4027
4028                 system = dir->subsystem;
4029
4030                 /* the ftrace system is special, skip it */
4031                 if (strcmp(system->name, "ftrace") == 0)
4032                         continue;
4033
4034                 pr_info("Testing event system %s: ", system->name);
4035
4036                 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1);
4037                 if (WARN_ON_ONCE(ret)) {
4038                         pr_warn("error enabling system %s\n",
4039                                 system->name);
4040                         continue;
4041                 }
4042
4043                 event_test_stuff();
4044
4045                 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0);
4046                 if (WARN_ON_ONCE(ret)) {
4047                         pr_warn("error disabling system %s\n",
4048                                 system->name);
4049                         continue;
4050                 }
4051
4052                 pr_cont("OK\n");
4053         }
4054
4055         /* Test with all events enabled */
4056
4057         pr_info("Running tests on all trace events:\n");
4058         pr_info("Testing all events: ");
4059
4060         ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1);
4061         if (WARN_ON_ONCE(ret)) {
4062                 pr_warn("error enabling all events\n");
4063                 return;
4064         }
4065
4066         event_test_stuff();
4067
4068         /* reset sysname */
4069         ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0);
4070         if (WARN_ON_ONCE(ret)) {
4071                 pr_warn("error disabling all events\n");
4072                 return;
4073         }
4074
4075         pr_cont("OK\n");
4076 }
4077
4078 #ifdef CONFIG_FUNCTION_TRACER
4079
4080 static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
4081
4082 static struct trace_event_file event_trace_file __initdata;
4083
4084 static void __init
4085 function_test_events_call(unsigned long ip, unsigned long parent_ip,
4086                           struct ftrace_ops *op, struct ftrace_regs *regs)
4087 {
4088         struct trace_buffer *buffer;
4089         struct ring_buffer_event *event;
4090         struct ftrace_entry *entry;
4091         unsigned int trace_ctx;
4092         long disabled;
4093         int cpu;
4094
4095         trace_ctx = tracing_gen_ctx();
4096         preempt_disable_notrace();
4097         cpu = raw_smp_processor_id();
4098         disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
4099
4100         if (disabled != 1)
4101                 goto out;
4102
4103         event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file,
4104                                                 TRACE_FN, sizeof(*entry),
4105                                                 trace_ctx);
4106         if (!event)
4107                 goto out;
4108         entry   = ring_buffer_event_data(event);
4109         entry->ip                       = ip;
4110         entry->parent_ip                = parent_ip;
4111
4112         event_trigger_unlock_commit(&event_trace_file, buffer, event,
4113                                     entry, trace_ctx);
4114  out:
4115         atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
4116         preempt_enable_notrace();
4117 }
4118
4119 static struct ftrace_ops trace_ops __initdata  =
4120 {
4121         .func = function_test_events_call,
4122 };
4123
4124 static __init void event_trace_self_test_with_function(void)
4125 {
4126         int ret;
4127
4128         event_trace_file.tr = top_trace_array();
4129         if (WARN_ON(!event_trace_file.tr))
4130                 return;
4131
4132         ret = register_ftrace_function(&trace_ops);
4133         if (WARN_ON(ret < 0)) {
4134                 pr_info("Failed to enable function tracer for event tests\n");
4135                 return;
4136         }
4137         pr_info("Running tests again, along with the function tracer\n");
4138         event_trace_self_tests();
4139         unregister_ftrace_function(&trace_ops);
4140 }
4141 #else
4142 static __init void event_trace_self_test_with_function(void)
4143 {
4144 }
4145 #endif
4146
4147 static __init int event_trace_self_tests_init(void)
4148 {
4149         if (!tracing_selftest_disabled) {
4150                 event_trace_self_tests();
4151                 event_trace_self_test_with_function();
4152         }
4153
4154         return 0;
4155 }
4156
4157 late_initcall(event_trace_self_tests_init);
4158
4159 #endif