kernel/trace/trace_eprobe.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * event probes
   4  *
   5  * Part of this code was copied from kernel/trace/trace_kprobe.c written by
   6  * Masami Hiramatsu <mhiramat@kernel.org>
   7  *
   8  * Copyright (C) 2021, VMware Inc, Steven Rostedt <rostedt@goodmis.org>
   9  * Copyright (C) 2021, VMware Inc, Tzvetomir Stoyanov tz.stoyanov@gmail.com>
  10  *
  11  */
  12 #include <linux/module.h>
  13 #include <linux/mutex.h>
  14 #include <linux/ftrace.h>
  15
  16 #include "trace_dynevent.h"
  17 #include "trace_probe.h"
  18 #include "trace_probe_tmpl.h"
  19 #include "trace_probe_kernel.h"
  20
  21 #define EPROBE_EVENT_SYSTEM "eprobes"
  22
  23 struct trace_eprobe {
  24         /* tracepoint system */
  25         const char *event_system;
  26
  27         /* tracepoint event */
  28         const char *event_name;
  29
  30         /* filter string for the tracepoint */
  31         char *filter_str;
  32
  33         struct trace_event_call *event;
  34
  35         struct dyn_event        devent;
  36         struct trace_probe      tp;
  37 };
  38
  39 struct eprobe_data {
  40         struct trace_event_file *file;
  41         struct trace_eprobe     *ep;
  42 };
  43
  44 static int __trace_eprobe_create(int argc, const char *argv[]);
  45
  46 static void trace_event_probe_cleanup(struct trace_eprobe *ep)
  47 {
  48         if (!ep)
  49                 return;
  50         trace_probe_cleanup(&ep->tp);
  51         kfree(ep->event_name);
  52         kfree(ep->event_system);
  53         if (ep->event)
  54                 trace_event_put_ref(ep->event);
  55         kfree(ep->filter_str);
  56         kfree(ep);
  57 }
  58
  59 static struct trace_eprobe *to_trace_eprobe(struct dyn_event *ev)
  60 {
  61         return container_of(ev, struct trace_eprobe, devent);
  62 }
  63
  64 static int eprobe_dyn_event_create(const char *raw_command)
  65 {
  66         return trace_probe_create(raw_command, __trace_eprobe_create);
  67 }
  68
  69 static int eprobe_dyn_event_show(struct seq_file *m, struct dyn_event *ev)
  70 {
  71         struct trace_eprobe *ep = to_trace_eprobe(ev);
  72         int i;
  73
  74         seq_printf(m, "e:%s/%s", trace_probe_group_name(&ep->tp),
  75                                 trace_probe_name(&ep->tp));
  76         seq_printf(m, " %s.%s", ep->event_system, ep->event_name);
  77
  78         for (i = 0; i < ep->tp.nr_args; i++)
  79                 seq_printf(m, " %s=%s", ep->tp.args[i].name, ep->tp.args[i].comm);
  80         seq_putc(m, '\n');
  81
  82         return 0;
  83 }
  84
  85 static int unregister_trace_eprobe(struct trace_eprobe *ep)
  86 {
  87         /* If other probes are on the event, just unregister eprobe */
  88         if (trace_probe_has_sibling(&ep->tp))
  89                 goto unreg;
  90
  91         /* Enabled event can not be unregistered */
  92         if (trace_probe_is_enabled(&ep->tp))
  93                 return -EBUSY;
  94
  95         /* Will fail if probe is being used by ftrace or perf */
  96         if (trace_probe_unregister_event_call(&ep->tp))
  97                 return -EBUSY;
  98
  99 unreg:
 100         dyn_event_remove(&ep->devent);
 101         trace_probe_unlink(&ep->tp);
 102
 103         return 0;
 104 }
 105
 106 static int eprobe_dyn_event_release(struct dyn_event *ev)
 107 {
 108         struct trace_eprobe *ep = to_trace_eprobe(ev);
 109         int ret = unregister_trace_eprobe(ep);
 110
 111         if (!ret)
 112                 trace_event_probe_cleanup(ep);
 113         return ret;
 114 }
 115
 116 static bool eprobe_dyn_event_is_busy(struct dyn_event *ev)
 117 {
 118         struct trace_eprobe *ep = to_trace_eprobe(ev);
 119
 120         return trace_probe_is_enabled(&ep->tp);
 121 }
 122
 123 static bool eprobe_dyn_event_match(const char *system, const char *event,
 124                         int argc, const char **argv, struct dyn_event *ev)
 125 {
 126         struct trace_eprobe *ep = to_trace_eprobe(ev);
 127         const char *slash;
 128
 129         /*
 130          * We match the following:
 131          *  event only                  - match all eprobes with event name
 132          *  system and event only       - match all system/event probes
 133          *  system only                 - match all system probes
 134          *
 135          * The below has the above satisfied with more arguments:
 136          *
 137          *  attached system/event       - If the arg has the system and event
 138          *                                the probe is attached to, match
 139          *                                probes with the attachment.
 140          *
 141          *  If any more args are given, then it requires a full match.
 142          */
 143
 144         /*
 145          * If system exists, but this probe is not part of that system
 146          * do not match.
 147          */
 148         if (system && strcmp(trace_probe_group_name(&ep->tp), system) != 0)
 149                 return false;
 150
 151         /* Must match the event name */
 152         if (event[0] != '\0' && strcmp(trace_probe_name(&ep->tp), event) != 0)
 153                 return false;
 154
 155         /* No arguments match all */
 156         if (argc < 1)
 157                 return true;
 158
 159         /* First argument is the system/event the probe is attached to */
 160
 161         slash = strchr(argv[0], '/');
 162         if (!slash)
 163                 slash = strchr(argv[0], '.');
 164         if (!slash)
 165                 return false;
 166
 167         if (strncmp(ep->event_system, argv[0], slash - argv[0]))
 168                 return false;
 169         if (strcmp(ep->event_name, slash + 1))
 170                 return false;
 171
 172         argc--;
 173         argv++;
 174
 175         /* If there are no other args, then match */
 176         if (argc < 1)
 177                 return true;
 178
 179         return trace_probe_match_command_args(&ep->tp, argc, argv);
 180 }
 181
 182 static struct dyn_event_operations eprobe_dyn_event_ops = {
 183         .create = eprobe_dyn_event_create,
 184         .show = eprobe_dyn_event_show,
 185         .is_busy = eprobe_dyn_event_is_busy,
 186         .free = eprobe_dyn_event_release,
 187         .match = eprobe_dyn_event_match,
 188 };
 189
 190 static struct trace_eprobe *alloc_event_probe(const char *group,
 191                                               const char *this_event,
 192                                               struct trace_event_call *event,
 193                                               int nargs)
 194 {
 195         struct trace_eprobe *ep;
 196         const char *event_name;
 197         const char *sys_name;
 198         int ret = -ENOMEM;
 199
 200         if (!event)
 201                 return ERR_PTR(-ENODEV);
 202
 203         sys_name = event->class->system;
 204         event_name = trace_event_name(event);
 205
 206         ep = kzalloc(struct_size(ep, tp.args, nargs), GFP_KERNEL);
 207         if (!ep) {
 208                 trace_event_put_ref(event);
 209                 goto error;
 210         }
 211         ep->event = event;
 212         ep->event_name = kstrdup(event_name, GFP_KERNEL);
 213         if (!ep->event_name)
 214                 goto error;
 215         ep->event_system = kstrdup(sys_name, GFP_KERNEL);
 216         if (!ep->event_system)
 217                 goto error;
 218
 219         ret = trace_probe_init(&ep->tp, this_event, group, false);
 220         if (ret < 0)
 221                 goto error;
 222
 223         dyn_event_init(&ep->devent, &eprobe_dyn_event_ops);
 224         return ep;
 225 error:
 226         trace_event_probe_cleanup(ep);
 227         return ERR_PTR(ret);
 228 }
 229
 230 static int trace_eprobe_tp_arg_update(struct trace_eprobe *ep, int i)
 231 {
 232         struct probe_arg *parg = &ep->tp.args[i];
 233         struct ftrace_event_field *field;
 234         struct list_head *head;
 235         int ret = -ENOENT;
 236
 237         head = trace_get_fields(ep->event);
 238         list_for_each_entry(field, head, link) {
 239                 if (!strcmp(parg->code->data, field->name)) {
 240                         kfree(parg->code->data);
 241                         parg->code->data = field;
 242                         return 0;
 243                 }
 244         }
 245
 246         /*
 247          * Argument not found on event. But allow for comm and COMM
 248          * to be used to get the current->comm.
 249          */
 250         if (strcmp(parg->code->data, "COMM") == 0 ||
 251             strcmp(parg->code->data, "comm") == 0) {
 252                 parg->code->op = FETCH_OP_COMM;
 253                 ret = 0;
 254         }
 255
 256         kfree(parg->code->data);
 257         parg->code->data = NULL;
 258         return ret;
 259 }
 260
 261 static int eprobe_event_define_fields(struct trace_event_call *event_call)
 262 {
 263         struct eprobe_trace_entry_head field;
 264         struct trace_probe *tp;
 265
 266         tp = trace_probe_primary_from_call(event_call);
 267         if (WARN_ON_ONCE(!tp))
 268                 return -ENOENT;
 269
 270         return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
 271 }
 272
 273 static struct trace_event_fields eprobe_fields_array[] = {
 274         { .type = TRACE_FUNCTION_TYPE,
 275           .define_fields = eprobe_event_define_fields },
 276         {}
 277 };
 278
 279 /* Event entry printers */
 280 static enum print_line_t
 281 print_eprobe_event(struct trace_iterator *iter, int flags,
 282                    struct trace_event *event)
 283 {
 284         struct eprobe_trace_entry_head *field;
 285         struct trace_event_call *pevent;
 286         struct trace_event *probed_event;
 287         struct trace_seq *s = &iter->seq;
 288         struct trace_eprobe *ep;
 289         struct trace_probe *tp;
 290         unsigned int type;
 291
 292         field = (struct eprobe_trace_entry_head *)iter->ent;
 293         tp = trace_probe_primary_from_call(
 294                 container_of(event, struct trace_event_call, event));
 295         if (WARN_ON_ONCE(!tp))
 296                 goto out;
 297
 298         ep = container_of(tp, struct trace_eprobe, tp);
 299         type = ep->event->event.type;
 300
 301         trace_seq_printf(s, "%s: (", trace_probe_name(tp));
 302
 303         probed_event = ftrace_find_event(type);
 304         if (probed_event) {
 305                 pevent = container_of(probed_event, struct trace_event_call, event);
 306                 trace_seq_printf(s, "%s.%s", pevent->class->system,
 307                                  trace_event_name(pevent));
 308         } else {
 309                 trace_seq_printf(s, "%u", type);
 310         }
 311
 312         trace_seq_putc(s, ')');
 313
 314         if (print_probe_args(s, tp->args, tp->nr_args,
 315                              (u8 *)&field[1], field) < 0)
 316                 goto out;
 317
 318         trace_seq_putc(s, '\n');
 319  out:
 320         return trace_handle_return(s);
 321 }
 322
 323 static unsigned long get_event_field(struct fetch_insn *code, void *rec)
 324 {
 325         struct ftrace_event_field *field = code->data;
 326         unsigned long val;
 327         void *addr;
 328
 329         addr = rec + field->offset;
 330
 331         if (is_string_field(field)) {
 332                 switch (field->filter_type) {
 333                 case FILTER_DYN_STRING:
 334                         val = (unsigned long)(rec + (*(unsigned int *)addr & 0xffff));
 335                         break;
 336                 case FILTER_RDYN_STRING:
 337                         val = (unsigned long)(addr + (*(unsigned int *)addr & 0xffff));
 338                         break;
 339                 case FILTER_STATIC_STRING:
 340                         val = (unsigned long)addr;
 341                         break;
 342                 case FILTER_PTR_STRING:
 343                         val = (unsigned long)(*(char *)addr);
 344                         break;
 345                 default:
 346                         WARN_ON_ONCE(1);
 347                         return 0;
 348                 }
 349                 return val;
 350         }
 351
 352         switch (field->size) {
 353         case 1:
 354                 if (field->is_signed)
 355                         val = *(char *)addr;
 356                 else
 357                         val = *(unsigned char *)addr;
 358                 break;
 359         case 2:
 360                 if (field->is_signed)
 361                         val = *(short *)addr;
 362                 else
 363                         val = *(unsigned short *)addr;
 364                 break;
 365         case 4:
 366                 if (field->is_signed)
 367                         val = *(int *)addr;
 368                 else
 369                         val = *(unsigned int *)addr;
 370                 break;
 371         default:
 372                 if (field->is_signed)
 373                         val = *(long *)addr;
 374                 else
 375                         val = *(unsigned long *)addr;
 376                 break;
 377         }
 378         return val;
 379 }
 380
 381 static int get_eprobe_size(struct trace_probe *tp, void *rec)
 382 {
 383         struct fetch_insn *code;
 384         struct probe_arg *arg;
 385         int i, len, ret = 0;
 386
 387         for (i = 0; i < tp->nr_args; i++) {
 388                 arg = tp->args + i;
 389                 if (arg->dynamic) {
 390                         unsigned long val;
 391
 392                         code = arg->code;
 393  retry:
 394                         switch (code->op) {
 395                         case FETCH_OP_TP_ARG:
 396                                 val = get_event_field(code, rec);
 397                                 break;
 398                         case FETCH_OP_IMM:
 399                                 val = code->immediate;
 400                                 break;
 401                         case FETCH_OP_COMM:
 402                                 val = (unsigned long)current->comm;
 403                                 break;
 404                         case FETCH_OP_DATA:
 405                                 val = (unsigned long)code->data;
 406                                 break;
 407                         case FETCH_NOP_SYMBOL:  /* Ignore a place holder */
 408                                 code++;
 409                                 goto retry;
 410                         default:
 411                                 continue;
 412                         }
 413                         code++;
 414                         len = process_fetch_insn_bottom(code, val, NULL, NULL);
 415                         if (len > 0)
 416                                 ret += len;
 417                 }
 418         }
 419
 420         return ret;
 421 }
 422
 423 /* Kprobe specific fetch functions */
 424
 425 /* Note that we don't verify it, since the code does not come from user space */
 426 static int
 427 process_fetch_insn(struct fetch_insn *code, void *rec, void *dest,
 428                    void *base)
 429 {
 430         unsigned long val;
 431
 432  retry:
 433         switch (code->op) {
 434         case FETCH_OP_TP_ARG:
 435                 val = get_event_field(code, rec);
 436                 break;
 437         case FETCH_OP_IMM:
 438                 val = code->immediate;
 439                 break;
 440         case FETCH_OP_COMM:
 441                 val = (unsigned long)current->comm;
 442                 break;
 443         case FETCH_OP_DATA:
 444                 val = (unsigned long)code->data;
 445                 break;
 446         case FETCH_NOP_SYMBOL:  /* Ignore a place holder */
 447                 code++;
 448                 goto retry;
 449         default:
 450                 return -EILSEQ;
 451         }
 452         code++;
 453         return process_fetch_insn_bottom(code, val, dest, base);
 454 }
 455 NOKPROBE_SYMBOL(process_fetch_insn)
 456
 457 /* Return the length of string -- including null terminal byte */
 458 static nokprobe_inline int
 459 fetch_store_strlen_user(unsigned long addr)
 460 {
 461         return kern_fetch_store_strlen_user(addr);
 462 }
 463
 464 /* Return the length of string -- including null terminal byte */
 465 static nokprobe_inline int
 466 fetch_store_strlen(unsigned long addr)
 467 {
 468         return kern_fetch_store_strlen(addr);
 469 }
 470
 471 /*
 472  * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf
 473  * with max length and relative data location.
 474  */
 475 static nokprobe_inline int
 476 fetch_store_string_user(unsigned long addr, void *dest, void *base)
 477 {
 478         return kern_fetch_store_string_user(addr, dest, base);
 479 }
 480
 481 /*
 482  * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max
 483  * length and relative data location.
 484  */
 485 static nokprobe_inline int
 486 fetch_store_string(unsigned long addr, void *dest, void *base)
 487 {
 488         return kern_fetch_store_string(addr, dest, base);
 489 }
 490
 491 static nokprobe_inline int
 492 probe_mem_read_user(void *dest, void *src, size_t size)
 493 {
 494         const void __user *uaddr =  (__force const void __user *)src;
 495
 496         return copy_from_user_nofault(dest, uaddr, size);
 497 }
 498
 499 static nokprobe_inline int
 500 probe_mem_read(void *dest, void *src, size_t size)
 501 {
 502 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
 503         if ((unsigned long)src < TASK_SIZE)
 504                 return probe_mem_read_user(dest, src, size);
 505 #endif
 506         return copy_from_kernel_nofault(dest, src, size);
 507 }
 508
 509 /* eprobe handler */
 510 static inline void
 511 __eprobe_trace_func(struct eprobe_data *edata, void *rec)
 512 {
 513         struct eprobe_trace_entry_head *entry;
 514         struct trace_event_call *call = trace_probe_event_call(&edata->ep->tp);
 515         struct trace_event_buffer fbuffer;
 516         int dsize;
 517
 518         if (WARN_ON_ONCE(call != edata->file->event_call))
 519                 return;
 520
 521         if (trace_trigger_soft_disabled(edata->file))
 522                 return;
 523
 524         dsize = get_eprobe_size(&edata->ep->tp, rec);
 525
 526         entry = trace_event_buffer_reserve(&fbuffer, edata->file,
 527                                            sizeof(*entry) + edata->ep->tp.size + dsize);
 528
 529         if (!entry)
 530                 return;
 531
 532         entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
 533         store_trace_args(&entry[1], &edata->ep->tp, rec, sizeof(*entry), dsize);
 534
 535         trace_event_buffer_commit(&fbuffer);
 536 }
 537
 538 /*
 539  * The event probe implementation uses event triggers to get access to
 540  * the event it is attached to, but is not an actual trigger. The below
 541  * functions are just stubs to fulfill what is needed to use the trigger
 542  * infrastructure.
 543  */
 544 static int eprobe_trigger_init(struct event_trigger_data *data)
 545 {
 546         return 0;
 547 }
 548
 549 static void eprobe_trigger_free(struct event_trigger_data *data)
 550 {
 551
 552 }
 553
 554 static int eprobe_trigger_print(struct seq_file *m,
 555                                 struct event_trigger_data *data)
 556 {
 557         /* Do not print eprobe event triggers */
 558         return 0;
 559 }
 560
 561 static void eprobe_trigger_func(struct event_trigger_data *data,
 562                                 struct trace_buffer *buffer, void *rec,
 563                                 struct ring_buffer_event *rbe)
 564 {
 565         struct eprobe_data *edata = data->private_data;
 566
 567         if (unlikely(!rec))
 568                 return;
 569
 570         if (unlikely(!rec))
 571                 return;
 572
 573         __eprobe_trace_func(edata, rec);
 574 }
 575
 576 static struct event_trigger_ops eprobe_trigger_ops = {
 577         .trigger                = eprobe_trigger_func,
 578         .print                  = eprobe_trigger_print,
 579         .init                   = eprobe_trigger_init,
 580         .free                   = eprobe_trigger_free,
 581 };
 582
 583 static int eprobe_trigger_cmd_parse(struct event_command *cmd_ops,
 584                                     struct trace_event_file *file,
 585                                     char *glob, char *cmd,
 586                                     char *param_and_filter)
 587 {
 588         return -1;
 589 }
 590
 591 static int eprobe_trigger_reg_func(char *glob,
 592                                    struct event_trigger_data *data,
 593                                    struct trace_event_file *file)
 594 {
 595         return -1;
 596 }
 597
 598 static void eprobe_trigger_unreg_func(char *glob,
 599                                       struct event_trigger_data *data,
 600                                       struct trace_event_file *file)
 601 {
 602
 603 }
 604
 605 static struct event_trigger_ops *eprobe_trigger_get_ops(char *cmd,
 606                                                         char *param)
 607 {
 608         return &eprobe_trigger_ops;
 609 }
 610
 611 static struct event_command event_trigger_cmd = {
 612         .name                   = "eprobe",
 613         .trigger_type           = ETT_EVENT_EPROBE,
 614         .flags                  = EVENT_CMD_FL_NEEDS_REC,
 615         .parse                  = eprobe_trigger_cmd_parse,
 616         .reg                    = eprobe_trigger_reg_func,
 617         .unreg                  = eprobe_trigger_unreg_func,
 618         .unreg_all              = NULL,
 619         .get_trigger_ops        = eprobe_trigger_get_ops,
 620         .set_filter             = NULL,
 621 };
 622
 623 static struct event_trigger_data *
 624 new_eprobe_trigger(struct trace_eprobe *ep, struct trace_event_file *file)
 625 {
 626         struct event_trigger_data *trigger;
 627         struct event_filter *filter = NULL;
 628         struct eprobe_data *edata;
 629         int ret;
 630
 631         edata = kzalloc(sizeof(*edata), GFP_KERNEL);
 632         trigger = kzalloc(sizeof(*trigger), GFP_KERNEL);
 633         if (!trigger || !edata) {
 634                 ret = -ENOMEM;
 635                 goto error;
 636         }
 637
 638         trigger->flags = EVENT_TRIGGER_FL_PROBE;
 639         trigger->count = -1;
 640         trigger->ops = &eprobe_trigger_ops;
 641
 642         /*
 643          * EVENT PROBE triggers are not registered as commands with
 644          * register_event_command(), as they are not controlled by the user
 645          * from the trigger file
 646          */
 647         trigger->cmd_ops = &event_trigger_cmd;
 648
 649         INIT_LIST_HEAD(&trigger->list);
 650
 651         if (ep->filter_str) {
 652                 ret = create_event_filter(file->tr, ep->event,
 653                                         ep->filter_str, false, &filter);
 654                 if (ret)
 655                         goto error;
 656         }
 657         RCU_INIT_POINTER(trigger->filter, filter);
 658
 659         edata->file = file;
 660         edata->ep = ep;
 661         trigger->private_data = edata;
 662
 663         return trigger;
 664 error:
 665         free_event_filter(filter);
 666         kfree(edata);
 667         kfree(trigger);
 668         return ERR_PTR(ret);
 669 }
 670
 671 static int enable_eprobe(struct trace_eprobe *ep,
 672                          struct trace_event_file *eprobe_file)
 673 {
 674         struct event_trigger_data *trigger;
 675         struct trace_event_file *file;
 676         struct trace_array *tr = eprobe_file->tr;
 677
 678         file = find_event_file(tr, ep->event_system, ep->event_name);
 679         if (!file)
 680                 return -ENOENT;
 681         trigger = new_eprobe_trigger(ep, eprobe_file);
 682         if (IS_ERR(trigger))
 683                 return PTR_ERR(trigger);
 684
 685         list_add_tail_rcu(&trigger->list, &file->triggers);
 686
 687         trace_event_trigger_enable_disable(file, 1);
 688         update_cond_flag(file);
 689
 690         return 0;
 691 }
 692
 693 static struct trace_event_functions eprobe_funcs = {
 694         .trace          = print_eprobe_event
 695 };
 696
 697 static int disable_eprobe(struct trace_eprobe *ep,
 698                           struct trace_array *tr)
 699 {
 700         struct event_trigger_data *trigger = NULL, *iter;
 701         struct trace_event_file *file;
 702         struct event_filter *filter;
 703         struct eprobe_data *edata;
 704
 705         file = find_event_file(tr, ep->event_system, ep->event_name);
 706         if (!file)
 707                 return -ENOENT;
 708
 709         list_for_each_entry(iter, &file->triggers, list) {
 710                 if (!(iter->flags & EVENT_TRIGGER_FL_PROBE))
 711                         continue;
 712                 edata = iter->private_data;
 713                 if (edata->ep == ep) {
 714                         trigger = iter;
 715                         break;
 716                 }
 717         }
 718         if (!trigger)
 719                 return -ENODEV;
 720
 721         list_del_rcu(&trigger->list);
 722
 723         trace_event_trigger_enable_disable(file, 0);
 724         update_cond_flag(file);
 725
 726         /* Make sure nothing is using the edata or trigger */
 727         tracepoint_synchronize_unregister();
 728
 729         filter = rcu_access_pointer(trigger->filter);
 730
 731         if (filter)
 732                 free_event_filter(filter);
 733         kfree(edata);
 734         kfree(trigger);
 735
 736         return 0;
 737 }
 738
 739 static int enable_trace_eprobe(struct trace_event_call *call,
 740                                struct trace_event_file *file)
 741 {
 742         struct trace_probe *pos, *tp;
 743         struct trace_eprobe *ep;
 744         bool enabled;
 745         int ret = 0;
 746
 747         tp = trace_probe_primary_from_call(call);
 748         if (WARN_ON_ONCE(!tp))
 749                 return -ENODEV;
 750         enabled = trace_probe_is_enabled(tp);
 751
 752         /* This also changes "enabled" state */
 753         if (file) {
 754                 ret = trace_probe_add_file(tp, file);
 755                 if (ret)
 756                         return ret;
 757         } else
 758                 trace_probe_set_flag(tp, TP_FLAG_PROFILE);
 759
 760         if (enabled)
 761                 return 0;
 762
 763         list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
 764                 ep = container_of(pos, struct trace_eprobe, tp);
 765                 ret = enable_eprobe(ep, file);
 766                 if (ret)
 767                         break;
 768                 enabled = true;
 769         }
 770
 771         if (ret) {
 772                 /* Failed to enable one of them. Roll back all */
 773                 if (enabled)
 774                         disable_eprobe(ep, file->tr);
 775                 if (file)
 776                         trace_probe_remove_file(tp, file);
 777                 else
 778                         trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
 779         }
 780
 781         return ret;
 782 }
 783
 784 static int disable_trace_eprobe(struct trace_event_call *call,
 785                                 struct trace_event_file *file)
 786 {
 787         struct trace_probe *pos, *tp;
 788         struct trace_eprobe *ep;
 789
 790         tp = trace_probe_primary_from_call(call);
 791         if (WARN_ON_ONCE(!tp))
 792                 return -ENODEV;
 793
 794         if (file) {
 795                 if (!trace_probe_get_file_link(tp, file))
 796                         return -ENOENT;
 797                 if (!trace_probe_has_single_file(tp))
 798                         goto out;
 799                 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
 800         } else
 801                 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
 802
 803         if (!trace_probe_is_enabled(tp)) {
 804                 list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
 805                         ep = container_of(pos, struct trace_eprobe, tp);
 806                         disable_eprobe(ep, file->tr);
 807                 }
 808         }
 809
 810  out:
 811         if (file)
 812                 /*
 813                  * Synchronization is done in below function. For perf event,
 814                  * file == NULL and perf_trace_event_unreg() calls
 815                  * tracepoint_synchronize_unregister() to ensure synchronize
 816                  * event. We don't need to care about it.
 817                  */
 818                 trace_probe_remove_file(tp, file);
 819
 820         return 0;
 821 }
 822
 823 static int eprobe_register(struct trace_event_call *event,
 824                            enum trace_reg type, void *data)
 825 {
 826         struct trace_event_file *file = data;
 827
 828         switch (type) {
 829         case TRACE_REG_REGISTER:
 830                 return enable_trace_eprobe(event, file);
 831         case TRACE_REG_UNREGISTER:
 832                 return disable_trace_eprobe(event, file);
 833 #ifdef CONFIG_PERF_EVENTS
 834         case TRACE_REG_PERF_REGISTER:
 835         case TRACE_REG_PERF_UNREGISTER:
 836         case TRACE_REG_PERF_OPEN:
 837         case TRACE_REG_PERF_CLOSE:
 838         case TRACE_REG_PERF_ADD:
 839         case TRACE_REG_PERF_DEL:
 840                 return 0;
 841 #endif
 842         }
 843         return 0;
 844 }
 845
 846 static inline void init_trace_eprobe_call(struct trace_eprobe *ep)
 847 {
 848         struct trace_event_call *call = trace_probe_event_call(&ep->tp);
 849
 850         call->flags = TRACE_EVENT_FL_EPROBE;
 851         call->event.funcs = &eprobe_funcs;
 852         call->class->fields_array = eprobe_fields_array;
 853         call->class->reg = eprobe_register;
 854 }
 855
 856 static struct trace_event_call *
 857 find_and_get_event(const char *system, const char *event_name)
 858 {
 859         struct trace_event_call *tp_event;
 860         const char *name;
 861
 862         list_for_each_entry(tp_event, &ftrace_events, list) {
 863                 /* Skip other probes and ftrace events */
 864                 if (tp_event->flags &
 865                     (TRACE_EVENT_FL_IGNORE_ENABLE |
 866                      TRACE_EVENT_FL_KPROBE |
 867                      TRACE_EVENT_FL_UPROBE |
 868                      TRACE_EVENT_FL_EPROBE))
 869                         continue;
 870                 if (!tp_event->class->system ||
 871                     strcmp(system, tp_event->class->system))
 872                         continue;
 873                 name = trace_event_name(tp_event);
 874                 if (!name || strcmp(event_name, name))
 875                         continue;
 876                 if (!trace_event_try_get_ref(tp_event)) {
 877                         return NULL;
 878                         break;
 879                 }
 880                 return tp_event;
 881                 break;
 882         }
 883         return NULL;
 884 }
 885
 886 static int trace_eprobe_tp_update_arg(struct trace_eprobe *ep, const char *argv[], int i)
 887 {
 888         unsigned int flags = TPARG_FL_KERNEL | TPARG_FL_TPOINT;
 889         int ret;
 890
 891         ret = traceprobe_parse_probe_arg(&ep->tp, i, argv[i], flags);
 892         if (ret)
 893                 return ret;
 894
 895         if (ep->tp.args[i].code->op == FETCH_OP_TP_ARG) {
 896                 ret = trace_eprobe_tp_arg_update(ep, i);
 897                 if (ret)
 898                         trace_probe_log_err(0, BAD_ATTACH_ARG);
 899         }
 900
 901         /* Handle symbols "@" */
 902         if (!ret)
 903                 ret = traceprobe_update_arg(&ep->tp.args[i]);
 904
 905         return ret;
 906 }
 907
 908 static int trace_eprobe_parse_filter(struct trace_eprobe *ep, int argc, const char *argv[])
 909 {
 910         struct event_filter *dummy = NULL;
 911         int i, ret, len = 0;
 912         char *p;
 913
 914         if (argc == 0) {
 915                 trace_probe_log_err(0, NO_EP_FILTER);
 916                 return -EINVAL;
 917         }
 918
 919         /* Recover the filter string */
 920         for (i = 0; i < argc; i++)
 921                 len += strlen(argv[i]) + 1;
 922
 923         ep->filter_str = kzalloc(len, GFP_KERNEL);
 924         if (!ep->filter_str)
 925                 return -ENOMEM;
 926
 927         p = ep->filter_str;
 928         for (i = 0; i < argc; i++) {
 929                 ret = snprintf(p, len, "%s ", argv[i]);
 930                 if (ret < 0)
 931                         goto error;
 932                 if (ret > len) {
 933                         ret = -E2BIG;
 934                         goto error;
 935                 }
 936                 p += ret;
 937                 len -= ret;
 938         }
 939         p[-1] = '\0';
 940
 941         /*
 942          * Ensure the filter string can be parsed correctly. Note, this
 943          * filter string is for the original event, not for the eprobe.
 944          */
 945         ret = create_event_filter(top_trace_array(), ep->event, ep->filter_str,
 946                                   true, &dummy);
 947         free_event_filter(dummy);
 948         if (ret)
 949                 goto error;
 950
 951         return 0;
 952 error:
 953         kfree(ep->filter_str);
 954         ep->filter_str = NULL;
 955         return ret;
 956 }
 957
 958 static int __trace_eprobe_create(int argc, const char *argv[])
 959 {
 960         /*
 961          * Argument syntax:
 962          *      e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS] [if FILTER]
 963          * Fetch args (no space):
 964          *  <name>=$<field>[:TYPE]
 965          */
 966         const char *event = NULL, *group = EPROBE_EVENT_SYSTEM;
 967         const char *sys_event = NULL, *sys_name = NULL;
 968         struct trace_event_call *event_call;
 969         struct trace_eprobe *ep = NULL;
 970         char buf1[MAX_EVENT_NAME_LEN];
 971         char buf2[MAX_EVENT_NAME_LEN];
 972         char gbuf[MAX_EVENT_NAME_LEN];
 973         int ret = 0, filter_idx = 0;
 974         int i, filter_cnt;
 975
 976         if (argc < 2 || argv[0][0] != 'e')
 977                 return -ECANCELED;
 978
 979         trace_probe_log_init("event_probe", argc, argv);
 980
 981         event = strchr(&argv[0][1], ':');
 982         if (event) {
 983                 event++;
 984                 ret = traceprobe_parse_event_name(&event, &group, gbuf,
 985                                                   event - argv[0]);
 986                 if (ret)
 987                         goto parse_error;
 988         }
 989
 990         trace_probe_log_set_index(1);
 991         sys_event = argv[1];
 992         ret = traceprobe_parse_event_name(&sys_event, &sys_name, buf2, 0);
 993         if (ret || !sys_event || !sys_name) {
 994                 trace_probe_log_err(0, NO_EVENT_INFO);
 995                 goto parse_error;
 996         }
 997
 998         if (!event) {
 999                 strscpy(buf1, sys_event, MAX_EVENT_NAME_LEN);
1000                 event = buf1;
1001         }
1002
1003         for (i = 2; i < argc; i++) {
1004                 if (!strcmp(argv[i], "if")) {
1005                         filter_idx = i + 1;
1006                         filter_cnt = argc - filter_idx;
1007                         argc = i;
1008                         break;
1009                 }
1010         }
1011
1012         mutex_lock(&event_mutex);
1013         event_call = find_and_get_event(sys_name, sys_event);
1014         ep = alloc_event_probe(group, event, event_call, argc - 2);
1015         mutex_unlock(&event_mutex);
1016
1017         if (IS_ERR(ep)) {
1018                 ret = PTR_ERR(ep);
1019                 if (ret == -ENODEV)
1020                         trace_probe_log_err(0, BAD_ATTACH_EVENT);
1021                 /* This must return -ENOMEM or missing event, else there is a bug */
1022                 WARN_ON_ONCE(ret != -ENOMEM && ret != -ENODEV);
1023                 ep = NULL;
1024                 goto error;
1025         }
1026
1027         if (filter_idx) {
1028                 trace_probe_log_set_index(filter_idx);
1029                 ret = trace_eprobe_parse_filter(ep, filter_cnt, argv + filter_idx);
1030                 if (ret)
1031                         goto parse_error;
1032         } else
1033                 ep->filter_str = NULL;
1034
1035         argc -= 2; argv += 2;
1036         /* parse arguments */
1037         for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
1038                 trace_probe_log_set_index(i + 2);
1039                 ret = trace_eprobe_tp_update_arg(ep, argv, i);
1040                 if (ret)
1041                         goto error;
1042         }
1043         ret = traceprobe_set_print_fmt(&ep->tp, PROBE_PRINT_EVENT);
1044         if (ret < 0)
1045                 goto error;
1046         init_trace_eprobe_call(ep);
1047         mutex_lock(&event_mutex);
1048         ret = trace_probe_register_event_call(&ep->tp);
1049         if (ret) {
1050                 if (ret == -EEXIST) {
1051                         trace_probe_log_set_index(0);
1052                         trace_probe_log_err(0, EVENT_EXIST);
1053                 }
1054                 mutex_unlock(&event_mutex);
1055                 goto error;
1056         }
1057         ret = dyn_event_add(&ep->devent, &ep->tp.event->call);
1058         mutex_unlock(&event_mutex);
1059         return ret;
1060 parse_error:
1061         ret = -EINVAL;
1062 error:
1063         trace_event_probe_cleanup(ep);
1064         return ret;
1065 }
1066
1067 /*
1068  * Register dynevent at core_initcall. This allows kernel to setup eprobe
1069  * events in postcore_initcall without tracefs.
1070  */
1071 static __init int trace_events_eprobe_init_early(void)
1072 {
1073         int err = 0;
1074
1075         err = dyn_event_register(&eprobe_dyn_event_ops);
1076         if (err)
1077                 pr_warn("Could not register eprobe_dyn_event_ops\n");
1078
1079         return err;
1080 }
1081 core_initcall(trace_events_eprobe_init_early);