Merge tag 'icc-6.1-rc6' of git://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc...
[platform/kernel/linux-starfive.git] / kernel / trace / trace_eprobe.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * event probes
4  *
5  * Part of this code was copied from kernel/trace/trace_kprobe.c written by
6  * Masami Hiramatsu <mhiramat@kernel.org>
7  *
8  * Copyright (C) 2021, VMware Inc, Steven Rostedt <rostedt@goodmis.org>
9  * Copyright (C) 2021, VMware Inc, Tzvetomir Stoyanov tz.stoyanov@gmail.com>
10  *
11  */
12 #include <linux/module.h>
13 #include <linux/mutex.h>
14 #include <linux/ftrace.h>
15
16 #include "trace_dynevent.h"
17 #include "trace_probe.h"
18 #include "trace_probe_tmpl.h"
19 #include "trace_probe_kernel.h"
20
21 #define EPROBE_EVENT_SYSTEM "eprobes"
22
23 struct trace_eprobe {
24         /* tracepoint system */
25         const char *event_system;
26
27         /* tracepoint event */
28         const char *event_name;
29
30         /* filter string for the tracepoint */
31         char *filter_str;
32
33         struct trace_event_call *event;
34
35         struct dyn_event        devent;
36         struct trace_probe      tp;
37 };
38
39 struct eprobe_data {
40         struct trace_event_file *file;
41         struct trace_eprobe     *ep;
42 };
43
44 static int __trace_eprobe_create(int argc, const char *argv[]);
45
46 static void trace_event_probe_cleanup(struct trace_eprobe *ep)
47 {
48         if (!ep)
49                 return;
50         trace_probe_cleanup(&ep->tp);
51         kfree(ep->event_name);
52         kfree(ep->event_system);
53         if (ep->event)
54                 trace_event_put_ref(ep->event);
55         kfree(ep->filter_str);
56         kfree(ep);
57 }
58
59 static struct trace_eprobe *to_trace_eprobe(struct dyn_event *ev)
60 {
61         return container_of(ev, struct trace_eprobe, devent);
62 }
63
64 static int eprobe_dyn_event_create(const char *raw_command)
65 {
66         return trace_probe_create(raw_command, __trace_eprobe_create);
67 }
68
69 static int eprobe_dyn_event_show(struct seq_file *m, struct dyn_event *ev)
70 {
71         struct trace_eprobe *ep = to_trace_eprobe(ev);
72         int i;
73
74         seq_printf(m, "e:%s/%s", trace_probe_group_name(&ep->tp),
75                                 trace_probe_name(&ep->tp));
76         seq_printf(m, " %s.%s", ep->event_system, ep->event_name);
77
78         for (i = 0; i < ep->tp.nr_args; i++)
79                 seq_printf(m, " %s=%s", ep->tp.args[i].name, ep->tp.args[i].comm);
80         seq_putc(m, '\n');
81
82         return 0;
83 }
84
85 static int unregister_trace_eprobe(struct trace_eprobe *ep)
86 {
87         /* If other probes are on the event, just unregister eprobe */
88         if (trace_probe_has_sibling(&ep->tp))
89                 goto unreg;
90
91         /* Enabled event can not be unregistered */
92         if (trace_probe_is_enabled(&ep->tp))
93                 return -EBUSY;
94
95         /* Will fail if probe is being used by ftrace or perf */
96         if (trace_probe_unregister_event_call(&ep->tp))
97                 return -EBUSY;
98
99 unreg:
100         dyn_event_remove(&ep->devent);
101         trace_probe_unlink(&ep->tp);
102
103         return 0;
104 }
105
106 static int eprobe_dyn_event_release(struct dyn_event *ev)
107 {
108         struct trace_eprobe *ep = to_trace_eprobe(ev);
109         int ret = unregister_trace_eprobe(ep);
110
111         if (!ret)
112                 trace_event_probe_cleanup(ep);
113         return ret;
114 }
115
116 static bool eprobe_dyn_event_is_busy(struct dyn_event *ev)
117 {
118         struct trace_eprobe *ep = to_trace_eprobe(ev);
119
120         return trace_probe_is_enabled(&ep->tp);
121 }
122
123 static bool eprobe_dyn_event_match(const char *system, const char *event,
124                         int argc, const char **argv, struct dyn_event *ev)
125 {
126         struct trace_eprobe *ep = to_trace_eprobe(ev);
127         const char *slash;
128
129         /*
130          * We match the following:
131          *  event only                  - match all eprobes with event name
132          *  system and event only       - match all system/event probes
133          *  system only                 - match all system probes
134          *
135          * The below has the above satisfied with more arguments:
136          *
137          *  attached system/event       - If the arg has the system and event
138          *                                the probe is attached to, match
139          *                                probes with the attachment.
140          *
141          *  If any more args are given, then it requires a full match.
142          */
143
144         /*
145          * If system exists, but this probe is not part of that system
146          * do not match.
147          */
148         if (system && strcmp(trace_probe_group_name(&ep->tp), system) != 0)
149                 return false;
150
151         /* Must match the event name */
152         if (event[0] != '\0' && strcmp(trace_probe_name(&ep->tp), event) != 0)
153                 return false;
154
155         /* No arguments match all */
156         if (argc < 1)
157                 return true;
158
159         /* First argument is the system/event the probe is attached to */
160
161         slash = strchr(argv[0], '/');
162         if (!slash)
163                 slash = strchr(argv[0], '.');
164         if (!slash)
165                 return false;
166
167         if (strncmp(ep->event_system, argv[0], slash - argv[0]))
168                 return false;
169         if (strcmp(ep->event_name, slash + 1))
170                 return false;
171
172         argc--;
173         argv++;
174
175         /* If there are no other args, then match */
176         if (argc < 1)
177                 return true;
178
179         return trace_probe_match_command_args(&ep->tp, argc, argv);
180 }
181
182 static struct dyn_event_operations eprobe_dyn_event_ops = {
183         .create = eprobe_dyn_event_create,
184         .show = eprobe_dyn_event_show,
185         .is_busy = eprobe_dyn_event_is_busy,
186         .free = eprobe_dyn_event_release,
187         .match = eprobe_dyn_event_match,
188 };
189
190 static struct trace_eprobe *alloc_event_probe(const char *group,
191                                               const char *this_event,
192                                               struct trace_event_call *event,
193                                               int nargs)
194 {
195         struct trace_eprobe *ep;
196         const char *event_name;
197         const char *sys_name;
198         int ret = -ENOMEM;
199
200         if (!event)
201                 return ERR_PTR(-ENODEV);
202
203         sys_name = event->class->system;
204         event_name = trace_event_name(event);
205
206         ep = kzalloc(struct_size(ep, tp.args, nargs), GFP_KERNEL);
207         if (!ep) {
208                 trace_event_put_ref(event);
209                 goto error;
210         }
211         ep->event = event;
212         ep->event_name = kstrdup(event_name, GFP_KERNEL);
213         if (!ep->event_name)
214                 goto error;
215         ep->event_system = kstrdup(sys_name, GFP_KERNEL);
216         if (!ep->event_system)
217                 goto error;
218
219         ret = trace_probe_init(&ep->tp, this_event, group, false);
220         if (ret < 0)
221                 goto error;
222
223         dyn_event_init(&ep->devent, &eprobe_dyn_event_ops);
224         return ep;
225 error:
226         trace_event_probe_cleanup(ep);
227         return ERR_PTR(ret);
228 }
229
230 static int trace_eprobe_tp_arg_update(struct trace_eprobe *ep, int i)
231 {
232         struct probe_arg *parg = &ep->tp.args[i];
233         struct ftrace_event_field *field;
234         struct list_head *head;
235         int ret = -ENOENT;
236
237         head = trace_get_fields(ep->event);
238         list_for_each_entry(field, head, link) {
239                 if (!strcmp(parg->code->data, field->name)) {
240                         kfree(parg->code->data);
241                         parg->code->data = field;
242                         return 0;
243                 }
244         }
245
246         /*
247          * Argument not found on event. But allow for comm and COMM
248          * to be used to get the current->comm.
249          */
250         if (strcmp(parg->code->data, "COMM") == 0 ||
251             strcmp(parg->code->data, "comm") == 0) {
252                 parg->code->op = FETCH_OP_COMM;
253                 ret = 0;
254         }
255
256         kfree(parg->code->data);
257         parg->code->data = NULL;
258         return ret;
259 }
260
261 static int eprobe_event_define_fields(struct trace_event_call *event_call)
262 {
263         struct eprobe_trace_entry_head field;
264         struct trace_probe *tp;
265
266         tp = trace_probe_primary_from_call(event_call);
267         if (WARN_ON_ONCE(!tp))
268                 return -ENOENT;
269
270         return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
271 }
272
273 static struct trace_event_fields eprobe_fields_array[] = {
274         { .type = TRACE_FUNCTION_TYPE,
275           .define_fields = eprobe_event_define_fields },
276         {}
277 };
278
279 /* Event entry printers */
280 static enum print_line_t
281 print_eprobe_event(struct trace_iterator *iter, int flags,
282                    struct trace_event *event)
283 {
284         struct eprobe_trace_entry_head *field;
285         struct trace_event_call *pevent;
286         struct trace_event *probed_event;
287         struct trace_seq *s = &iter->seq;
288         struct trace_eprobe *ep;
289         struct trace_probe *tp;
290         unsigned int type;
291
292         field = (struct eprobe_trace_entry_head *)iter->ent;
293         tp = trace_probe_primary_from_call(
294                 container_of(event, struct trace_event_call, event));
295         if (WARN_ON_ONCE(!tp))
296                 goto out;
297
298         ep = container_of(tp, struct trace_eprobe, tp);
299         type = ep->event->event.type;
300
301         trace_seq_printf(s, "%s: (", trace_probe_name(tp));
302
303         probed_event = ftrace_find_event(type);
304         if (probed_event) {
305                 pevent = container_of(probed_event, struct trace_event_call, event);
306                 trace_seq_printf(s, "%s.%s", pevent->class->system,
307                                  trace_event_name(pevent));
308         } else {
309                 trace_seq_printf(s, "%u", type);
310         }
311
312         trace_seq_putc(s, ')');
313
314         if (print_probe_args(s, tp->args, tp->nr_args,
315                              (u8 *)&field[1], field) < 0)
316                 goto out;
317
318         trace_seq_putc(s, '\n');
319  out:
320         return trace_handle_return(s);
321 }
322
323 static unsigned long get_event_field(struct fetch_insn *code, void *rec)
324 {
325         struct ftrace_event_field *field = code->data;
326         unsigned long val;
327         void *addr;
328
329         addr = rec + field->offset;
330
331         if (is_string_field(field)) {
332                 switch (field->filter_type) {
333                 case FILTER_DYN_STRING:
334                         val = (unsigned long)(rec + (*(unsigned int *)addr & 0xffff));
335                         break;
336                 case FILTER_RDYN_STRING:
337                         val = (unsigned long)(addr + (*(unsigned int *)addr & 0xffff));
338                         break;
339                 case FILTER_STATIC_STRING:
340                         val = (unsigned long)addr;
341                         break;
342                 case FILTER_PTR_STRING:
343                         val = (unsigned long)(*(char *)addr);
344                         break;
345                 default:
346                         WARN_ON_ONCE(1);
347                         return 0;
348                 }
349                 return val;
350         }
351
352         switch (field->size) {
353         case 1:
354                 if (field->is_signed)
355                         val = *(char *)addr;
356                 else
357                         val = *(unsigned char *)addr;
358                 break;
359         case 2:
360                 if (field->is_signed)
361                         val = *(short *)addr;
362                 else
363                         val = *(unsigned short *)addr;
364                 break;
365         case 4:
366                 if (field->is_signed)
367                         val = *(int *)addr;
368                 else
369                         val = *(unsigned int *)addr;
370                 break;
371         default:
372                 if (field->is_signed)
373                         val = *(long *)addr;
374                 else
375                         val = *(unsigned long *)addr;
376                 break;
377         }
378         return val;
379 }
380
381 static int get_eprobe_size(struct trace_probe *tp, void *rec)
382 {
383         struct fetch_insn *code;
384         struct probe_arg *arg;
385         int i, len, ret = 0;
386
387         for (i = 0; i < tp->nr_args; i++) {
388                 arg = tp->args + i;
389                 if (arg->dynamic) {
390                         unsigned long val;
391
392                         code = arg->code;
393  retry:
394                         switch (code->op) {
395                         case FETCH_OP_TP_ARG:
396                                 val = get_event_field(code, rec);
397                                 break;
398                         case FETCH_OP_IMM:
399                                 val = code->immediate;
400                                 break;
401                         case FETCH_OP_COMM:
402                                 val = (unsigned long)current->comm;
403                                 break;
404                         case FETCH_OP_DATA:
405                                 val = (unsigned long)code->data;
406                                 break;
407                         case FETCH_NOP_SYMBOL:  /* Ignore a place holder */
408                                 code++;
409                                 goto retry;
410                         default:
411                                 continue;
412                         }
413                         code++;
414                         len = process_fetch_insn_bottom(code, val, NULL, NULL);
415                         if (len > 0)
416                                 ret += len;
417                 }
418         }
419
420         return ret;
421 }
422
423 /* Kprobe specific fetch functions */
424
425 /* Note that we don't verify it, since the code does not come from user space */
426 static int
427 process_fetch_insn(struct fetch_insn *code, void *rec, void *dest,
428                    void *base)
429 {
430         unsigned long val;
431
432  retry:
433         switch (code->op) {
434         case FETCH_OP_TP_ARG:
435                 val = get_event_field(code, rec);
436                 break;
437         case FETCH_OP_IMM:
438                 val = code->immediate;
439                 break;
440         case FETCH_OP_COMM:
441                 val = (unsigned long)current->comm;
442                 break;
443         case FETCH_OP_DATA:
444                 val = (unsigned long)code->data;
445                 break;
446         case FETCH_NOP_SYMBOL:  /* Ignore a place holder */
447                 code++;
448                 goto retry;
449         default:
450                 return -EILSEQ;
451         }
452         code++;
453         return process_fetch_insn_bottom(code, val, dest, base);
454 }
455 NOKPROBE_SYMBOL(process_fetch_insn)
456
457 /* Return the length of string -- including null terminal byte */
458 static nokprobe_inline int
459 fetch_store_strlen_user(unsigned long addr)
460 {
461         return kern_fetch_store_strlen_user(addr);
462 }
463
464 /* Return the length of string -- including null terminal byte */
465 static nokprobe_inline int
466 fetch_store_strlen(unsigned long addr)
467 {
468         return kern_fetch_store_strlen(addr);
469 }
470
471 /*
472  * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf
473  * with max length and relative data location.
474  */
475 static nokprobe_inline int
476 fetch_store_string_user(unsigned long addr, void *dest, void *base)
477 {
478         return kern_fetch_store_string_user(addr, dest, base);
479 }
480
481 /*
482  * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max
483  * length and relative data location.
484  */
485 static nokprobe_inline int
486 fetch_store_string(unsigned long addr, void *dest, void *base)
487 {
488         return kern_fetch_store_string(addr, dest, base);
489 }
490
491 static nokprobe_inline int
492 probe_mem_read_user(void *dest, void *src, size_t size)
493 {
494         const void __user *uaddr =  (__force const void __user *)src;
495
496         return copy_from_user_nofault(dest, uaddr, size);
497 }
498
499 static nokprobe_inline int
500 probe_mem_read(void *dest, void *src, size_t size)
501 {
502 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
503         if ((unsigned long)src < TASK_SIZE)
504                 return probe_mem_read_user(dest, src, size);
505 #endif
506         return copy_from_kernel_nofault(dest, src, size);
507 }
508
509 /* eprobe handler */
510 static inline void
511 __eprobe_trace_func(struct eprobe_data *edata, void *rec)
512 {
513         struct eprobe_trace_entry_head *entry;
514         struct trace_event_call *call = trace_probe_event_call(&edata->ep->tp);
515         struct trace_event_buffer fbuffer;
516         int dsize;
517
518         if (WARN_ON_ONCE(call != edata->file->event_call))
519                 return;
520
521         if (trace_trigger_soft_disabled(edata->file))
522                 return;
523
524         dsize = get_eprobe_size(&edata->ep->tp, rec);
525
526         entry = trace_event_buffer_reserve(&fbuffer, edata->file,
527                                            sizeof(*entry) + edata->ep->tp.size + dsize);
528
529         if (!entry)
530                 return;
531
532         entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
533         store_trace_args(&entry[1], &edata->ep->tp, rec, sizeof(*entry), dsize);
534
535         trace_event_buffer_commit(&fbuffer);
536 }
537
538 /*
539  * The event probe implementation uses event triggers to get access to
540  * the event it is attached to, but is not an actual trigger. The below
541  * functions are just stubs to fulfill what is needed to use the trigger
542  * infrastructure.
543  */
544 static int eprobe_trigger_init(struct event_trigger_data *data)
545 {
546         return 0;
547 }
548
549 static void eprobe_trigger_free(struct event_trigger_data *data)
550 {
551
552 }
553
554 static int eprobe_trigger_print(struct seq_file *m,
555                                 struct event_trigger_data *data)
556 {
557         /* Do not print eprobe event triggers */
558         return 0;
559 }
560
561 static void eprobe_trigger_func(struct event_trigger_data *data,
562                                 struct trace_buffer *buffer, void *rec,
563                                 struct ring_buffer_event *rbe)
564 {
565         struct eprobe_data *edata = data->private_data;
566
567         if (unlikely(!rec))
568                 return;
569
570         __eprobe_trace_func(edata, rec);
571 }
572
573 static struct event_trigger_ops eprobe_trigger_ops = {
574         .trigger                = eprobe_trigger_func,
575         .print                  = eprobe_trigger_print,
576         .init                   = eprobe_trigger_init,
577         .free                   = eprobe_trigger_free,
578 };
579
580 static int eprobe_trigger_cmd_parse(struct event_command *cmd_ops,
581                                     struct trace_event_file *file,
582                                     char *glob, char *cmd,
583                                     char *param_and_filter)
584 {
585         return -1;
586 }
587
588 static int eprobe_trigger_reg_func(char *glob,
589                                    struct event_trigger_data *data,
590                                    struct trace_event_file *file)
591 {
592         return -1;
593 }
594
595 static void eprobe_trigger_unreg_func(char *glob,
596                                       struct event_trigger_data *data,
597                                       struct trace_event_file *file)
598 {
599
600 }
601
602 static struct event_trigger_ops *eprobe_trigger_get_ops(char *cmd,
603                                                         char *param)
604 {
605         return &eprobe_trigger_ops;
606 }
607
608 static struct event_command event_trigger_cmd = {
609         .name                   = "eprobe",
610         .trigger_type           = ETT_EVENT_EPROBE,
611         .flags                  = EVENT_CMD_FL_NEEDS_REC,
612         .parse                  = eprobe_trigger_cmd_parse,
613         .reg                    = eprobe_trigger_reg_func,
614         .unreg                  = eprobe_trigger_unreg_func,
615         .unreg_all              = NULL,
616         .get_trigger_ops        = eprobe_trigger_get_ops,
617         .set_filter             = NULL,
618 };
619
620 static struct event_trigger_data *
621 new_eprobe_trigger(struct trace_eprobe *ep, struct trace_event_file *file)
622 {
623         struct event_trigger_data *trigger;
624         struct event_filter *filter = NULL;
625         struct eprobe_data *edata;
626         int ret;
627
628         edata = kzalloc(sizeof(*edata), GFP_KERNEL);
629         trigger = kzalloc(sizeof(*trigger), GFP_KERNEL);
630         if (!trigger || !edata) {
631                 ret = -ENOMEM;
632                 goto error;
633         }
634
635         trigger->flags = EVENT_TRIGGER_FL_PROBE;
636         trigger->count = -1;
637         trigger->ops = &eprobe_trigger_ops;
638
639         /*
640          * EVENT PROBE triggers are not registered as commands with
641          * register_event_command(), as they are not controlled by the user
642          * from the trigger file
643          */
644         trigger->cmd_ops = &event_trigger_cmd;
645
646         INIT_LIST_HEAD(&trigger->list);
647
648         if (ep->filter_str) {
649                 ret = create_event_filter(file->tr, ep->event,
650                                         ep->filter_str, false, &filter);
651                 if (ret)
652                         goto error;
653         }
654         RCU_INIT_POINTER(trigger->filter, filter);
655
656         edata->file = file;
657         edata->ep = ep;
658         trigger->private_data = edata;
659
660         return trigger;
661 error:
662         free_event_filter(filter);
663         kfree(edata);
664         kfree(trigger);
665         return ERR_PTR(ret);
666 }
667
668 static int enable_eprobe(struct trace_eprobe *ep,
669                          struct trace_event_file *eprobe_file)
670 {
671         struct event_trigger_data *trigger;
672         struct trace_event_file *file;
673         struct trace_array *tr = eprobe_file->tr;
674
675         file = find_event_file(tr, ep->event_system, ep->event_name);
676         if (!file)
677                 return -ENOENT;
678         trigger = new_eprobe_trigger(ep, eprobe_file);
679         if (IS_ERR(trigger))
680                 return PTR_ERR(trigger);
681
682         list_add_tail_rcu(&trigger->list, &file->triggers);
683
684         trace_event_trigger_enable_disable(file, 1);
685         update_cond_flag(file);
686
687         return 0;
688 }
689
690 static struct trace_event_functions eprobe_funcs = {
691         .trace          = print_eprobe_event
692 };
693
694 static int disable_eprobe(struct trace_eprobe *ep,
695                           struct trace_array *tr)
696 {
697         struct event_trigger_data *trigger = NULL, *iter;
698         struct trace_event_file *file;
699         struct event_filter *filter;
700         struct eprobe_data *edata;
701
702         file = find_event_file(tr, ep->event_system, ep->event_name);
703         if (!file)
704                 return -ENOENT;
705
706         list_for_each_entry(iter, &file->triggers, list) {
707                 if (!(iter->flags & EVENT_TRIGGER_FL_PROBE))
708                         continue;
709                 edata = iter->private_data;
710                 if (edata->ep == ep) {
711                         trigger = iter;
712                         break;
713                 }
714         }
715         if (!trigger)
716                 return -ENODEV;
717
718         list_del_rcu(&trigger->list);
719
720         trace_event_trigger_enable_disable(file, 0);
721         update_cond_flag(file);
722
723         /* Make sure nothing is using the edata or trigger */
724         tracepoint_synchronize_unregister();
725
726         filter = rcu_access_pointer(trigger->filter);
727
728         if (filter)
729                 free_event_filter(filter);
730         kfree(edata);
731         kfree(trigger);
732
733         return 0;
734 }
735
736 static int enable_trace_eprobe(struct trace_event_call *call,
737                                struct trace_event_file *file)
738 {
739         struct trace_probe *pos, *tp;
740         struct trace_eprobe *ep;
741         bool enabled;
742         int ret = 0;
743
744         tp = trace_probe_primary_from_call(call);
745         if (WARN_ON_ONCE(!tp))
746                 return -ENODEV;
747         enabled = trace_probe_is_enabled(tp);
748
749         /* This also changes "enabled" state */
750         if (file) {
751                 ret = trace_probe_add_file(tp, file);
752                 if (ret)
753                         return ret;
754         } else
755                 trace_probe_set_flag(tp, TP_FLAG_PROFILE);
756
757         if (enabled)
758                 return 0;
759
760         list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
761                 ep = container_of(pos, struct trace_eprobe, tp);
762                 ret = enable_eprobe(ep, file);
763                 if (ret)
764                         break;
765                 enabled = true;
766         }
767
768         if (ret) {
769                 /* Failed to enable one of them. Roll back all */
770                 if (enabled)
771                         disable_eprobe(ep, file->tr);
772                 if (file)
773                         trace_probe_remove_file(tp, file);
774                 else
775                         trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
776         }
777
778         return ret;
779 }
780
781 static int disable_trace_eprobe(struct trace_event_call *call,
782                                 struct trace_event_file *file)
783 {
784         struct trace_probe *pos, *tp;
785         struct trace_eprobe *ep;
786
787         tp = trace_probe_primary_from_call(call);
788         if (WARN_ON_ONCE(!tp))
789                 return -ENODEV;
790
791         if (file) {
792                 if (!trace_probe_get_file_link(tp, file))
793                         return -ENOENT;
794                 if (!trace_probe_has_single_file(tp))
795                         goto out;
796                 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
797         } else
798                 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
799
800         if (!trace_probe_is_enabled(tp)) {
801                 list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
802                         ep = container_of(pos, struct trace_eprobe, tp);
803                         disable_eprobe(ep, file->tr);
804                 }
805         }
806
807  out:
808         if (file)
809                 /*
810                  * Synchronization is done in below function. For perf event,
811                  * file == NULL and perf_trace_event_unreg() calls
812                  * tracepoint_synchronize_unregister() to ensure synchronize
813                  * event. We don't need to care about it.
814                  */
815                 trace_probe_remove_file(tp, file);
816
817         return 0;
818 }
819
820 static int eprobe_register(struct trace_event_call *event,
821                            enum trace_reg type, void *data)
822 {
823         struct trace_event_file *file = data;
824
825         switch (type) {
826         case TRACE_REG_REGISTER:
827                 return enable_trace_eprobe(event, file);
828         case TRACE_REG_UNREGISTER:
829                 return disable_trace_eprobe(event, file);
830 #ifdef CONFIG_PERF_EVENTS
831         case TRACE_REG_PERF_REGISTER:
832         case TRACE_REG_PERF_UNREGISTER:
833         case TRACE_REG_PERF_OPEN:
834         case TRACE_REG_PERF_CLOSE:
835         case TRACE_REG_PERF_ADD:
836         case TRACE_REG_PERF_DEL:
837                 return 0;
838 #endif
839         }
840         return 0;
841 }
842
843 static inline void init_trace_eprobe_call(struct trace_eprobe *ep)
844 {
845         struct trace_event_call *call = trace_probe_event_call(&ep->tp);
846
847         call->flags = TRACE_EVENT_FL_EPROBE;
848         call->event.funcs = &eprobe_funcs;
849         call->class->fields_array = eprobe_fields_array;
850         call->class->reg = eprobe_register;
851 }
852
853 static struct trace_event_call *
854 find_and_get_event(const char *system, const char *event_name)
855 {
856         struct trace_event_call *tp_event;
857         const char *name;
858
859         list_for_each_entry(tp_event, &ftrace_events, list) {
860                 /* Skip other probes and ftrace events */
861                 if (tp_event->flags &
862                     (TRACE_EVENT_FL_IGNORE_ENABLE |
863                      TRACE_EVENT_FL_KPROBE |
864                      TRACE_EVENT_FL_UPROBE |
865                      TRACE_EVENT_FL_EPROBE))
866                         continue;
867                 if (!tp_event->class->system ||
868                     strcmp(system, tp_event->class->system))
869                         continue;
870                 name = trace_event_name(tp_event);
871                 if (!name || strcmp(event_name, name))
872                         continue;
873                 if (!trace_event_try_get_ref(tp_event)) {
874                         return NULL;
875                         break;
876                 }
877                 return tp_event;
878                 break;
879         }
880         return NULL;
881 }
882
883 static int trace_eprobe_tp_update_arg(struct trace_eprobe *ep, const char *argv[], int i)
884 {
885         unsigned int flags = TPARG_FL_KERNEL | TPARG_FL_TPOINT;
886         int ret;
887
888         ret = traceprobe_parse_probe_arg(&ep->tp, i, argv[i], flags);
889         if (ret)
890                 return ret;
891
892         if (ep->tp.args[i].code->op == FETCH_OP_TP_ARG) {
893                 ret = trace_eprobe_tp_arg_update(ep, i);
894                 if (ret)
895                         trace_probe_log_err(0, BAD_ATTACH_ARG);
896         }
897
898         /* Handle symbols "@" */
899         if (!ret)
900                 ret = traceprobe_update_arg(&ep->tp.args[i]);
901
902         return ret;
903 }
904
905 static int trace_eprobe_parse_filter(struct trace_eprobe *ep, int argc, const char *argv[])
906 {
907         struct event_filter *dummy = NULL;
908         int i, ret, len = 0;
909         char *p;
910
911         if (argc == 0) {
912                 trace_probe_log_err(0, NO_EP_FILTER);
913                 return -EINVAL;
914         }
915
916         /* Recover the filter string */
917         for (i = 0; i < argc; i++)
918                 len += strlen(argv[i]) + 1;
919
920         ep->filter_str = kzalloc(len, GFP_KERNEL);
921         if (!ep->filter_str)
922                 return -ENOMEM;
923
924         p = ep->filter_str;
925         for (i = 0; i < argc; i++) {
926                 ret = snprintf(p, len, "%s ", argv[i]);
927                 if (ret < 0)
928                         goto error;
929                 if (ret > len) {
930                         ret = -E2BIG;
931                         goto error;
932                 }
933                 p += ret;
934                 len -= ret;
935         }
936         p[-1] = '\0';
937
938         /*
939          * Ensure the filter string can be parsed correctly. Note, this
940          * filter string is for the original event, not for the eprobe.
941          */
942         ret = create_event_filter(top_trace_array(), ep->event, ep->filter_str,
943                                   true, &dummy);
944         free_event_filter(dummy);
945         if (ret)
946                 goto error;
947
948         return 0;
949 error:
950         kfree(ep->filter_str);
951         ep->filter_str = NULL;
952         return ret;
953 }
954
955 static int __trace_eprobe_create(int argc, const char *argv[])
956 {
957         /*
958          * Argument syntax:
959          *      e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS] [if FILTER]
960          * Fetch args (no space):
961          *  <name>=$<field>[:TYPE]
962          */
963         const char *event = NULL, *group = EPROBE_EVENT_SYSTEM;
964         const char *sys_event = NULL, *sys_name = NULL;
965         struct trace_event_call *event_call;
966         struct trace_eprobe *ep = NULL;
967         char buf1[MAX_EVENT_NAME_LEN];
968         char buf2[MAX_EVENT_NAME_LEN];
969         char gbuf[MAX_EVENT_NAME_LEN];
970         int ret = 0, filter_idx = 0;
971         int i, filter_cnt;
972
973         if (argc < 2 || argv[0][0] != 'e')
974                 return -ECANCELED;
975
976         trace_probe_log_init("event_probe", argc, argv);
977
978         event = strchr(&argv[0][1], ':');
979         if (event) {
980                 event++;
981                 ret = traceprobe_parse_event_name(&event, &group, gbuf,
982                                                   event - argv[0]);
983                 if (ret)
984                         goto parse_error;
985         }
986
987         trace_probe_log_set_index(1);
988         sys_event = argv[1];
989         ret = traceprobe_parse_event_name(&sys_event, &sys_name, buf2, 0);
990         if (ret || !sys_event || !sys_name) {
991                 trace_probe_log_err(0, NO_EVENT_INFO);
992                 goto parse_error;
993         }
994
995         if (!event) {
996                 strscpy(buf1, sys_event, MAX_EVENT_NAME_LEN);
997                 event = buf1;
998         }
999
1000         for (i = 2; i < argc; i++) {
1001                 if (!strcmp(argv[i], "if")) {
1002                         filter_idx = i + 1;
1003                         filter_cnt = argc - filter_idx;
1004                         argc = i;
1005                         break;
1006                 }
1007         }
1008
1009         mutex_lock(&event_mutex);
1010         event_call = find_and_get_event(sys_name, sys_event);
1011         ep = alloc_event_probe(group, event, event_call, argc - 2);
1012         mutex_unlock(&event_mutex);
1013
1014         if (IS_ERR(ep)) {
1015                 ret = PTR_ERR(ep);
1016                 if (ret == -ENODEV)
1017                         trace_probe_log_err(0, BAD_ATTACH_EVENT);
1018                 /* This must return -ENOMEM or missing event, else there is a bug */
1019                 WARN_ON_ONCE(ret != -ENOMEM && ret != -ENODEV);
1020                 ep = NULL;
1021                 goto error;
1022         }
1023
1024         if (filter_idx) {
1025                 trace_probe_log_set_index(filter_idx);
1026                 ret = trace_eprobe_parse_filter(ep, filter_cnt, argv + filter_idx);
1027                 if (ret)
1028                         goto parse_error;
1029         } else
1030                 ep->filter_str = NULL;
1031
1032         argc -= 2; argv += 2;
1033         /* parse arguments */
1034         for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
1035                 trace_probe_log_set_index(i + 2);
1036                 ret = trace_eprobe_tp_update_arg(ep, argv, i);
1037                 if (ret)
1038                         goto error;
1039         }
1040         ret = traceprobe_set_print_fmt(&ep->tp, PROBE_PRINT_EVENT);
1041         if (ret < 0)
1042                 goto error;
1043         init_trace_eprobe_call(ep);
1044         mutex_lock(&event_mutex);
1045         ret = trace_probe_register_event_call(&ep->tp);
1046         if (ret) {
1047                 if (ret == -EEXIST) {
1048                         trace_probe_log_set_index(0);
1049                         trace_probe_log_err(0, EVENT_EXIST);
1050                 }
1051                 mutex_unlock(&event_mutex);
1052                 goto error;
1053         }
1054         ret = dyn_event_add(&ep->devent, &ep->tp.event->call);
1055         mutex_unlock(&event_mutex);
1056         return ret;
1057 parse_error:
1058         ret = -EINVAL;
1059 error:
1060         trace_event_probe_cleanup(ep);
1061         return ret;
1062 }
1063
1064 /*
1065  * Register dynevent at core_initcall. This allows kernel to setup eprobe
1066  * events in postcore_initcall without tracefs.
1067  */
1068 static __init int trace_events_eprobe_init_early(void)
1069 {
1070         int err = 0;
1071
1072         err = dyn_event_register(&eprobe_dyn_event_ops);
1073         if (err)
1074                 pr_warn("Could not register eprobe_dyn_event_ops\n");
1075
1076         return err;
1077 }
1078 core_initcall(trace_events_eprobe_init_early);