kernel/trace/trace_probe.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Common code for probe-based Dynamic events.
   4  *
   5  * This code was copied from kernel/trace/trace_kprobe.c written by
   6  * Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
   7  *
   8  * Updates to make this generic:
   9  * Copyright (C) IBM Corporation, 2010-2011
  10  * Author:     Srikar Dronamraju
  11  */
  12 #define pr_fmt(fmt)     "trace_probe: " fmt
  13
  14 #include <linux/bpf.h>
  15
  16 #include "trace_probe.h"
  17
  18 #undef C
  19 #define C(a, b)         b
  20
  21 static const char *trace_probe_err_text[] = { ERRORS };
  22
  23 static const char *reserved_field_names[] = {
  24         "common_type",
  25         "common_flags",
  26         "common_preempt_count",
  27         "common_pid",
  28         "common_tgid",
  29         FIELD_STRING_IP,
  30         FIELD_STRING_RETIP,
  31         FIELD_STRING_FUNC,
  32 };
  33
  34 /* Printing  in basic type function template */
  35 #define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt)                  \
  36 int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
  37 {                                                                       \
  38         trace_seq_printf(s, fmt, *(type *)data);                        \
  39         return !trace_seq_has_overflowed(s);                            \
  40 }                                                                       \
  41 const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
  42
  43 DEFINE_BASIC_PRINT_TYPE_FUNC(u8,  u8,  "%u")
  44 DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
  45 DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
  46 DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
  47 DEFINE_BASIC_PRINT_TYPE_FUNC(s8,  s8,  "%d")
  48 DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
  49 DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
  50 DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
  51 DEFINE_BASIC_PRINT_TYPE_FUNC(x8,  u8,  "0x%x")
  52 DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
  53 DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
  54 DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
  55 DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")
  56
  57 int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
  58 {
  59         trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
  60         return !trace_seq_has_overflowed(s);
  61 }
  62 const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
  63
  64 /* Print type function for string type */
  65 int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
  66 {
  67         int len = *(u32 *)data >> 16;
  68
  69         if (!len)
  70                 trace_seq_puts(s, "(fault)");
  71         else
  72                 trace_seq_printf(s, "\"%s\"",
  73                                  (const char *)get_loc_data(data, ent));
  74         return !trace_seq_has_overflowed(s);
  75 }
  76
  77 const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
  78
  79 /* Fetch type information table */
  80 static const struct fetch_type probe_fetch_types[] = {
  81         /* Special types */
  82         __ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1,
  83                             "__data_loc char[]"),
  84         __ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1,
  85                             "__data_loc char[]"),
  86         __ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1,
  87                             "__data_loc char[]"),
  88         /* Basic types */
  89         ASSIGN_FETCH_TYPE(u8,  u8,  0),
  90         ASSIGN_FETCH_TYPE(u16, u16, 0),
  91         ASSIGN_FETCH_TYPE(u32, u32, 0),
  92         ASSIGN_FETCH_TYPE(u64, u64, 0),
  93         ASSIGN_FETCH_TYPE(s8,  u8,  1),
  94         ASSIGN_FETCH_TYPE(s16, u16, 1),
  95         ASSIGN_FETCH_TYPE(s32, u32, 1),
  96         ASSIGN_FETCH_TYPE(s64, u64, 1),
  97         ASSIGN_FETCH_TYPE_ALIAS(x8,  u8,  u8,  0),
  98         ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
  99         ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
 100         ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
 101         ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8,  0),
 102         ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
 103
 104         ASSIGN_FETCH_TYPE_END
 105 };
 106
 107 static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
 108 {
 109         int i;
 110
 111         /* Reject the symbol/symstr for uprobes */
 112         if (type && (flags & TPARG_FL_USER) &&
 113             (!strcmp(type, "symbol") || !strcmp(type, "symstr")))
 114                 return NULL;
 115
 116         if (!type)
 117                 type = DEFAULT_FETCH_TYPE_STR;
 118
 119         /* Special case: bitfield */
 120         if (*type == 'b') {
 121                 unsigned long bs;
 122
 123                 type = strchr(type, '/');
 124                 if (!type)
 125                         goto fail;
 126
 127                 type++;
 128                 if (kstrtoul(type, 0, &bs))
 129                         goto fail;
 130
 131                 switch (bs) {
 132                 case 8:
 133                         return find_fetch_type("u8", flags);
 134                 case 16:
 135                         return find_fetch_type("u16", flags);
 136                 case 32:
 137                         return find_fetch_type("u32", flags);
 138                 case 64:
 139                         return find_fetch_type("u64", flags);
 140                 default:
 141                         goto fail;
 142                 }
 143         }
 144
 145         for (i = 0; probe_fetch_types[i].name; i++) {
 146                 if (strcmp(type, probe_fetch_types[i].name) == 0)
 147                         return &probe_fetch_types[i];
 148         }
 149
 150 fail:
 151         return NULL;
 152 }
 153
 154 static struct trace_probe_log trace_probe_log;
 155
 156 void trace_probe_log_init(const char *subsystem, int argc, const char **argv)
 157 {
 158         trace_probe_log.subsystem = subsystem;
 159         trace_probe_log.argc = argc;
 160         trace_probe_log.argv = argv;
 161         trace_probe_log.index = 0;
 162 }
 163
 164 void trace_probe_log_clear(void)
 165 {
 166         memset(&trace_probe_log, 0, sizeof(trace_probe_log));
 167 }
 168
 169 void trace_probe_log_set_index(int index)
 170 {
 171         trace_probe_log.index = index;
 172 }
 173
 174 void __trace_probe_log_err(int offset, int err_type)
 175 {
 176         char *command, *p;
 177         int i, len = 0, pos = 0;
 178
 179         if (!trace_probe_log.argv)
 180                 return;
 181
 182         /* Recalculate the length and allocate buffer */
 183         for (i = 0; i < trace_probe_log.argc; i++) {
 184                 if (i == trace_probe_log.index)
 185                         pos = len;
 186                 len += strlen(trace_probe_log.argv[i]) + 1;
 187         }
 188         command = kzalloc(len, GFP_KERNEL);
 189         if (!command)
 190                 return;
 191
 192         if (trace_probe_log.index >= trace_probe_log.argc) {
 193                 /**
 194                  * Set the error position is next to the last arg + space.
 195                  * Note that len includes the terminal null and the cursor
 196                  * appears at pos + 1.
 197                  */
 198                 pos = len;
 199                 offset = 0;
 200         }
 201
 202         /* And make a command string from argv array */
 203         p = command;
 204         for (i = 0; i < trace_probe_log.argc; i++) {
 205                 len = strlen(trace_probe_log.argv[i]);
 206                 strcpy(p, trace_probe_log.argv[i]);
 207                 p[len] = ' ';
 208                 p += len + 1;
 209         }
 210         *(p - 1) = '\0';
 211
 212         tracing_log_err(NULL, trace_probe_log.subsystem, command,
 213                         trace_probe_err_text, err_type, pos + offset);
 214
 215         kfree(command);
 216 }
 217
 218 /* Split symbol and offset. */
 219 int traceprobe_split_symbol_offset(char *symbol, long *offset)
 220 {
 221         char *tmp;
 222         int ret;
 223
 224         if (!offset)
 225                 return -EINVAL;
 226
 227         tmp = strpbrk(symbol, "+-");
 228         if (tmp) {
 229                 ret = kstrtol(tmp, 0, offset);
 230                 if (ret)
 231                         return ret;
 232                 *tmp = '\0';
 233         } else
 234                 *offset = 0;
 235
 236         return 0;
 237 }
 238
 239 /* @buf must has MAX_EVENT_NAME_LEN size */
 240 int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
 241                                 char *buf, int offset)
 242 {
 243         const char *slash, *event = *pevent;
 244         int len;
 245
 246         slash = strchr(event, '/');
 247         if (!slash)
 248                 slash = strchr(event, '.');
 249
 250         if (slash) {
 251                 if (slash == event) {
 252                         trace_probe_log_err(offset, NO_GROUP_NAME);
 253                         return -EINVAL;
 254                 }
 255                 if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
 256                         trace_probe_log_err(offset, GROUP_TOO_LONG);
 257                         return -EINVAL;
 258                 }
 259                 strscpy(buf, event, slash - event + 1);
 260                 if (!is_good_system_name(buf)) {
 261                         trace_probe_log_err(offset, BAD_GROUP_NAME);
 262                         return -EINVAL;
 263                 }
 264                 *pgroup = buf;
 265                 *pevent = slash + 1;
 266                 offset += slash - event + 1;
 267                 event = *pevent;
 268         }
 269         len = strlen(event);
 270         if (len == 0) {
 271                 if (slash) {
 272                         *pevent = NULL;
 273                         return 0;
 274                 }
 275                 trace_probe_log_err(offset, NO_EVENT_NAME);
 276                 return -EINVAL;
 277         } else if (len > MAX_EVENT_NAME_LEN) {
 278                 trace_probe_log_err(offset, EVENT_TOO_LONG);
 279                 return -EINVAL;
 280         }
 281         if (!is_good_name(event)) {
 282                 trace_probe_log_err(offset, BAD_EVENT_NAME);
 283                 return -EINVAL;
 284         }
 285         return 0;
 286 }
 287
 288 static int parse_trace_event_arg(char *arg, struct fetch_insn *code,
 289                                  struct traceprobe_parse_context *ctx)
 290 {
 291         struct ftrace_event_field *field;
 292         struct list_head *head;
 293
 294         head = trace_get_fields(ctx->event);
 295         list_for_each_entry(field, head, link) {
 296                 if (!strcmp(arg, field->name)) {
 297                         code->op = FETCH_OP_TP_ARG;
 298                         code->data = field;
 299                         return 0;
 300                 }
 301         }
 302         return -ENOENT;
 303 }
 304
 305 #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
 306
 307 static struct btf *traceprobe_get_btf(void)
 308 {
 309         struct btf *btf = bpf_get_btf_vmlinux();
 310
 311         if (IS_ERR_OR_NULL(btf))
 312                 return NULL;
 313
 314         return btf;
 315 }
 316
 317 static u32 btf_type_int(const struct btf_type *t)
 318 {
 319         return *(u32 *)(t + 1);
 320 }
 321
 322 static const char *type_from_btf_id(struct btf *btf, s32 id)
 323 {
 324         const struct btf_type *t;
 325         u32 intdata;
 326         s32 tid;
 327
 328         /* TODO: const char * could be converted as a string */
 329         t = btf_type_skip_modifiers(btf, id, &tid);
 330
 331         switch (BTF_INFO_KIND(t->info)) {
 332         case BTF_KIND_ENUM:
 333                 /* enum is "int", so convert to "s32" */
 334                 return "s32";
 335         case BTF_KIND_ENUM64:
 336                 return "s64";
 337         case BTF_KIND_PTR:
 338                 /* pointer will be converted to "x??" */
 339                 if (IS_ENABLED(CONFIG_64BIT))
 340                         return "x64";
 341                 else
 342                         return "x32";
 343         case BTF_KIND_INT:
 344                 intdata = btf_type_int(t);
 345                 if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
 346                         switch (BTF_INT_BITS(intdata)) {
 347                         case 8:
 348                                 return "s8";
 349                         case 16:
 350                                 return "s16";
 351                         case 32:
 352                                 return "s32";
 353                         case 64:
 354                                 return "s64";
 355                         }
 356                 } else {        /* unsigned */
 357                         switch (BTF_INT_BITS(intdata)) {
 358                         case 8:
 359                                 return "u8";
 360                         case 16:
 361                                 return "u16";
 362                         case 32:
 363                                 return "u32";
 364                         case 64:
 365                                 return "u64";
 366                         }
 367                 }
 368         }
 369         /* TODO: support other types */
 370
 371         return NULL;
 372 }
 373
 374 static const struct btf_type *find_btf_func_proto(const char *funcname)
 375 {
 376         struct btf *btf = traceprobe_get_btf();
 377         const struct btf_type *t;
 378         s32 id;
 379
 380         if (!btf || !funcname)
 381                 return ERR_PTR(-EINVAL);
 382
 383         id = btf_find_by_name_kind(btf, funcname, BTF_KIND_FUNC);
 384         if (id <= 0)
 385                 return ERR_PTR(-ENOENT);
 386
 387         /* Get BTF_KIND_FUNC type */
 388         t = btf_type_by_id(btf, id);
 389         if (!btf_type_is_func(t))
 390                 return ERR_PTR(-ENOENT);
 391
 392         /* The type of BTF_KIND_FUNC is BTF_KIND_FUNC_PROTO */
 393         t = btf_type_by_id(btf, t->type);
 394         if (!btf_type_is_func_proto(t))
 395                 return ERR_PTR(-ENOENT);
 396
 397         return t;
 398 }
 399
 400 static const struct btf_param *find_btf_func_param(const char *funcname, s32 *nr,
 401                                                    bool tracepoint)
 402 {
 403         const struct btf_param *param;
 404         const struct btf_type *t;
 405
 406         if (!funcname || !nr)
 407                 return ERR_PTR(-EINVAL);
 408
 409         t = find_btf_func_proto(funcname);
 410         if (IS_ERR(t))
 411                 return (const struct btf_param *)t;
 412
 413         *nr = btf_type_vlen(t);
 414         param = (const struct btf_param *)(t + 1);
 415
 416         /* Hide the first 'data' argument of tracepoint */
 417         if (tracepoint) {
 418                 (*nr)--;
 419                 param++;
 420         }
 421
 422         if (*nr > 0)
 423                 return param;
 424         else
 425                 return NULL;
 426 }
 427
 428 static int parse_btf_arg(const char *varname, struct fetch_insn *code,
 429                          struct traceprobe_parse_context *ctx)
 430 {
 431         struct btf *btf = traceprobe_get_btf();
 432         const struct btf_param *params;
 433         int i;
 434
 435         if (!btf) {
 436                 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
 437                 return -EOPNOTSUPP;
 438         }
 439
 440         if (WARN_ON_ONCE(!ctx->funcname))
 441                 return -EINVAL;
 442
 443         if (!ctx->params) {
 444                 params = find_btf_func_param(ctx->funcname, &ctx->nr_params,
 445                                              ctx->flags & TPARG_FL_TPOINT);
 446                 if (IS_ERR(params)) {
 447                         trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
 448                         return PTR_ERR(params);
 449                 }
 450                 ctx->params = params;
 451         } else
 452                 params = ctx->params;
 453
 454         for (i = 0; i < ctx->nr_params; i++) {
 455                 const char *name = btf_name_by_offset(btf, params[i].name_off);
 456
 457                 if (name && !strcmp(name, varname)) {
 458                         code->op = FETCH_OP_ARG;
 459                         if (ctx->flags & TPARG_FL_TPOINT)
 460                                 code->param = i + 1;
 461                         else
 462                                 code->param = i;
 463                         return 0;
 464                 }
 465         }
 466         trace_probe_log_err(ctx->offset, NO_BTFARG);
 467         return -ENOENT;
 468 }
 469
 470 static const struct fetch_type *parse_btf_arg_type(int arg_idx,
 471                                         struct traceprobe_parse_context *ctx)
 472 {
 473         struct btf *btf = traceprobe_get_btf();
 474         const char *typestr = NULL;
 475
 476         if (btf && ctx->params) {
 477                 if (ctx->flags & TPARG_FL_TPOINT)
 478                         arg_idx--;
 479                 typestr = type_from_btf_id(btf, ctx->params[arg_idx].type);
 480         }
 481
 482         return find_fetch_type(typestr, ctx->flags);
 483 }
 484
 485 static const struct fetch_type *parse_btf_retval_type(
 486                                         struct traceprobe_parse_context *ctx)
 487 {
 488         struct btf *btf = traceprobe_get_btf();
 489         const char *typestr = NULL;
 490         const struct btf_type *t;
 491
 492         if (btf && ctx->funcname) {
 493                 t = find_btf_func_proto(ctx->funcname);
 494                 if (!IS_ERR(t))
 495                         typestr = type_from_btf_id(btf, t->type);
 496         }
 497
 498         return find_fetch_type(typestr, ctx->flags);
 499 }
 500
 501 static bool is_btf_retval_void(const char *funcname)
 502 {
 503         const struct btf_type *t;
 504
 505         t = find_btf_func_proto(funcname);
 506         if (IS_ERR(t))
 507                 return false;
 508
 509         return t->type == 0;
 510 }
 511 #else
 512 static struct btf *traceprobe_get_btf(void)
 513 {
 514         return NULL;
 515 }
 516
 517 static const struct btf_param *find_btf_func_param(const char *funcname, s32 *nr,
 518                                                    bool tracepoint)
 519 {
 520         return ERR_PTR(-EOPNOTSUPP);
 521 }
 522
 523 static int parse_btf_arg(const char *varname, struct fetch_insn *code,
 524                          struct traceprobe_parse_context *ctx)
 525 {
 526         trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
 527         return -EOPNOTSUPP;
 528 }
 529
 530 #define parse_btf_arg_type(idx, ctx)            \
 531         find_fetch_type(NULL, ctx->flags)
 532
 533 #define parse_btf_retval_type(ctx)              \
 534         find_fetch_type(NULL, ctx->flags)
 535
 536 #define is_btf_retval_void(funcname)    (false)
 537
 538 #endif
 539
 540 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
 541
 542 static int parse_probe_vars(char *arg, const struct fetch_type *t,
 543                             struct fetch_insn *code,
 544                             struct traceprobe_parse_context *ctx)
 545 {
 546         unsigned long param;
 547         int err = TP_ERR_BAD_VAR;
 548         int ret = 0;
 549         int len;
 550
 551         if (ctx->flags & TPARG_FL_TEVENT) {
 552                 if (code->data)
 553                         return -EFAULT;
 554                 ret = parse_trace_event_arg(arg, code, ctx);
 555                 if (!ret)
 556                         return 0;
 557                 if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
 558                         code->op = FETCH_OP_COMM;
 559                         return 0;
 560                 }
 561                 /* backward compatibility */
 562                 ctx->offset = 0;
 563                 goto inval;
 564         }
 565
 566         if (strcmp(arg, "retval") == 0) {
 567                 if (ctx->flags & TPARG_FL_RETURN) {
 568                         if ((ctx->flags & TPARG_FL_KERNEL) &&
 569                             is_btf_retval_void(ctx->funcname)) {
 570                                 err = TP_ERR_NO_RETVAL;
 571                                 goto inval;
 572                         }
 573                         code->op = FETCH_OP_RETVAL;
 574                         return 0;
 575                 }
 576                 err = TP_ERR_RETVAL_ON_PROBE;
 577                 goto inval;
 578         }
 579
 580         len = str_has_prefix(arg, "stack");
 581         if (len) {
 582
 583                 if (arg[len] == '\0') {
 584                         code->op = FETCH_OP_STACKP;
 585                         return 0;
 586                 }
 587
 588                 if (isdigit(arg[len])) {
 589                         ret = kstrtoul(arg + len, 10, &param);
 590                         if (ret)
 591                                 goto inval;
 592
 593                         if ((ctx->flags & TPARG_FL_KERNEL) &&
 594                             param > PARAM_MAX_STACK) {
 595                                 err = TP_ERR_BAD_STACK_NUM;
 596                                 goto inval;
 597                         }
 598                         code->op = FETCH_OP_STACK;
 599                         code->param = (unsigned int)param;
 600                         return 0;
 601                 }
 602                 goto inval;
 603         }
 604
 605         if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
 606                 code->op = FETCH_OP_COMM;
 607                 return 0;
 608         }
 609
 610 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
 611         len = str_has_prefix(arg, "arg");
 612         if (len && tparg_is_function_entry(ctx->flags)) {
 613                 ret = kstrtoul(arg + len, 10, &param);
 614                 if (ret)
 615                         goto inval;
 616
 617                 if (!param || param > PARAM_MAX_STACK) {
 618                         err = TP_ERR_BAD_ARG_NUM;
 619                         goto inval;
 620                 }
 621
 622                 code->op = FETCH_OP_ARG;
 623                 code->param = (unsigned int)param - 1;
 624                 /*
 625                  * The tracepoint probe will probe a stub function, and the
 626                  * first parameter of the stub is a dummy and should be ignored.
 627                  */
 628                 if (ctx->flags & TPARG_FL_TPOINT)
 629                         code->param++;
 630                 return 0;
 631         }
 632 #endif
 633
 634 inval:
 635         __trace_probe_log_err(ctx->offset, err);
 636         return -EINVAL;
 637 }
 638
 639 static int str_to_immediate(char *str, unsigned long *imm)
 640 {
 641         if (isdigit(str[0]))
 642                 return kstrtoul(str, 0, imm);
 643         else if (str[0] == '-')
 644                 return kstrtol(str, 0, (long *)imm);
 645         else if (str[0] == '+')
 646                 return kstrtol(str + 1, 0, (long *)imm);
 647         return -EINVAL;
 648 }
 649
 650 static int __parse_imm_string(char *str, char **pbuf, int offs)
 651 {
 652         size_t len = strlen(str);
 653
 654         if (str[len - 1] != '"') {
 655                 trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE);
 656                 return -EINVAL;
 657         }
 658         *pbuf = kstrndup(str, len - 1, GFP_KERNEL);
 659         if (!*pbuf)
 660                 return -ENOMEM;
 661         return 0;
 662 }
 663
 664 /* Recursive argument parser */
 665 static int
 666 parse_probe_arg(char *arg, const struct fetch_type *type,
 667                 struct fetch_insn **pcode, struct fetch_insn *end,
 668                 struct traceprobe_parse_context *ctx)
 669 {
 670         struct fetch_insn *code = *pcode;
 671         unsigned long param;
 672         int deref = FETCH_OP_DEREF;
 673         long offset = 0;
 674         char *tmp;
 675         int ret = 0;
 676
 677         switch (arg[0]) {
 678         case '$':
 679                 ret = parse_probe_vars(arg + 1, type, code, ctx);
 680                 break;
 681
 682         case '%':       /* named register */
 683                 if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
 684                         /* eprobe and fprobe do not handle registers */
 685                         trace_probe_log_err(ctx->offset, BAD_VAR);
 686                         break;
 687                 }
 688                 ret = regs_query_register_offset(arg + 1);
 689                 if (ret >= 0) {
 690                         code->op = FETCH_OP_REG;
 691                         code->param = (unsigned int)ret;
 692                         ret = 0;
 693                 } else
 694                         trace_probe_log_err(ctx->offset, BAD_REG_NAME);
 695                 break;
 696
 697         case '@':       /* memory, file-offset or symbol */
 698                 if (isdigit(arg[1])) {
 699                         ret = kstrtoul(arg + 1, 0, &param);
 700                         if (ret) {
 701                                 trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
 702                                 break;
 703                         }
 704                         /* load address */
 705                         code->op = FETCH_OP_IMM;
 706                         code->immediate = param;
 707                 } else if (arg[1] == '+') {
 708                         /* kprobes don't support file offsets */
 709                         if (ctx->flags & TPARG_FL_KERNEL) {
 710                                 trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
 711                                 return -EINVAL;
 712                         }
 713                         ret = kstrtol(arg + 2, 0, &offset);
 714                         if (ret) {
 715                                 trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
 716                                 break;
 717                         }
 718
 719                         code->op = FETCH_OP_FOFFS;
 720                         code->immediate = (unsigned long)offset;  // imm64?
 721                 } else {
 722                         /* uprobes don't support symbols */
 723                         if (!(ctx->flags & TPARG_FL_KERNEL)) {
 724                                 trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
 725                                 return -EINVAL;
 726                         }
 727                         /* Preserve symbol for updating */
 728                         code->op = FETCH_NOP_SYMBOL;
 729                         code->data = kstrdup(arg + 1, GFP_KERNEL);
 730                         if (!code->data)
 731                                 return -ENOMEM;
 732                         if (++code == end) {
 733                                 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 734                                 return -EINVAL;
 735                         }
 736                         code->op = FETCH_OP_IMM;
 737                         code->immediate = 0;
 738                 }
 739                 /* These are fetching from memory */
 740                 if (++code == end) {
 741                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 742                         return -EINVAL;
 743                 }
 744                 *pcode = code;
 745                 code->op = FETCH_OP_DEREF;
 746                 code->offset = offset;
 747                 break;
 748
 749         case '+':       /* deref memory */
 750         case '-':
 751                 if (arg[1] == 'u') {
 752                         deref = FETCH_OP_UDEREF;
 753                         arg[1] = arg[0];
 754                         arg++;
 755                 }
 756                 if (arg[0] == '+')
 757                         arg++;  /* Skip '+', because kstrtol() rejects it. */
 758                 tmp = strchr(arg, '(');
 759                 if (!tmp) {
 760                         trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
 761                         return -EINVAL;
 762                 }
 763                 *tmp = '\0';
 764                 ret = kstrtol(arg, 0, &offset);
 765                 if (ret) {
 766                         trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
 767                         break;
 768                 }
 769                 ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
 770                 arg = tmp + 1;
 771                 tmp = strrchr(arg, ')');
 772                 if (!tmp) {
 773                         trace_probe_log_err(ctx->offset + strlen(arg),
 774                                             DEREF_OPEN_BRACE);
 775                         return -EINVAL;
 776                 } else {
 777                         const struct fetch_type *t2 = find_fetch_type(NULL, ctx->flags);
 778                         int cur_offs = ctx->offset;
 779
 780                         *tmp = '\0';
 781                         ret = parse_probe_arg(arg, t2, &code, end, ctx);
 782                         if (ret)
 783                                 break;
 784                         ctx->offset = cur_offs;
 785                         if (code->op == FETCH_OP_COMM ||
 786                             code->op == FETCH_OP_DATA) {
 787                                 trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
 788                                 return -EINVAL;
 789                         }
 790                         if (++code == end) {
 791                                 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
 792                                 return -EINVAL;
 793                         }
 794                         *pcode = code;
 795
 796                         code->op = deref;
 797                         code->offset = offset;
 798                 }
 799                 break;
 800         case '\\':      /* Immediate value */
 801                 if (arg[1] == '"') {    /* Immediate string */
 802                         ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
 803                         if (ret)
 804                                 break;
 805                         code->op = FETCH_OP_DATA;
 806                         code->data = tmp;
 807                 } else {
 808                         ret = str_to_immediate(arg + 1, &code->immediate);
 809                         if (ret)
 810                                 trace_probe_log_err(ctx->offset + 1, BAD_IMM);
 811                         else
 812                                 code->op = FETCH_OP_IMM;
 813                 }
 814                 break;
 815         default:
 816                 if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */
 817                         if (!tparg_is_function_entry(ctx->flags)) {
 818                                 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
 819                                 return -EINVAL;
 820                         }
 821                         ret = parse_btf_arg(arg, code, ctx);
 822                         break;
 823                 }
 824         }
 825         if (!ret && code->op == FETCH_OP_NOP) {
 826                 /* Parsed, but do not find fetch method */
 827                 trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
 828                 ret = -EINVAL;
 829         }
 830         return ret;
 831 }
 832
 833 #define BYTES_TO_BITS(nb)       ((BITS_PER_LONG * (nb)) / sizeof(long))
 834
 835 /* Bitfield type needs to be parsed into a fetch function */
 836 static int __parse_bitfield_probe_arg(const char *bf,
 837                                       const struct fetch_type *t,
 838                                       struct fetch_insn **pcode)
 839 {
 840         struct fetch_insn *code = *pcode;
 841         unsigned long bw, bo;
 842         char *tail;
 843
 844         if (*bf != 'b')
 845                 return 0;
 846
 847         bw = simple_strtoul(bf + 1, &tail, 0);  /* Use simple one */
 848
 849         if (bw == 0 || *tail != '@')
 850                 return -EINVAL;
 851
 852         bf = tail + 1;
 853         bo = simple_strtoul(bf, &tail, 0);
 854
 855         if (tail == bf || *tail != '/')
 856                 return -EINVAL;
 857         code++;
 858         if (code->op != FETCH_OP_NOP)
 859                 return -EINVAL;
 860         *pcode = code;
 861
 862         code->op = FETCH_OP_MOD_BF;
 863         code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
 864         code->rshift = BYTES_TO_BITS(t->size) - bw;
 865         code->basesize = t->size;
 866
 867         return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
 868 }
 869
 870 /* String length checking wrapper */
 871 static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
 872                                            struct probe_arg *parg,
 873                                            struct traceprobe_parse_context *ctx)
 874 {
 875         struct fetch_insn *code, *scode, *tmp = NULL;
 876         char *t, *t2, *t3;
 877         int ret, len;
 878         char *arg;
 879
 880         arg = kstrdup(argv, GFP_KERNEL);
 881         if (!arg)
 882                 return -ENOMEM;
 883
 884         ret = -EINVAL;
 885         len = strlen(arg);
 886         if (len > MAX_ARGSTR_LEN) {
 887                 trace_probe_log_err(ctx->offset, ARG_TOO_LONG);
 888                 goto out;
 889         } else if (len == 0) {
 890                 trace_probe_log_err(ctx->offset, NO_ARG_BODY);
 891                 goto out;
 892         }
 893
 894         ret = -ENOMEM;
 895         parg->comm = kstrdup(arg, GFP_KERNEL);
 896         if (!parg->comm)
 897                 goto out;
 898
 899         ret = -EINVAL;
 900         t = strchr(arg, ':');
 901         if (t) {
 902                 *t = '\0';
 903                 t2 = strchr(++t, '[');
 904                 if (t2) {
 905                         *t2++ = '\0';
 906                         t3 = strchr(t2, ']');
 907                         if (!t3) {
 908                                 int offs = t2 + strlen(t2) - arg;
 909
 910                                 trace_probe_log_err(ctx->offset + offs,
 911                                                     ARRAY_NO_CLOSE);
 912                                 goto out;
 913                         } else if (t3[1] != '\0') {
 914                                 trace_probe_log_err(ctx->offset + t3 + 1 - arg,
 915                                                     BAD_ARRAY_SUFFIX);
 916                                 goto out;
 917                         }
 918                         *t3 = '\0';
 919                         if (kstrtouint(t2, 0, &parg->count) || !parg->count) {
 920                                 trace_probe_log_err(ctx->offset + t2 - arg,
 921                                                     BAD_ARRAY_NUM);
 922                                 goto out;
 923                         }
 924                         if (parg->count > MAX_ARRAY_LEN) {
 925                                 trace_probe_log_err(ctx->offset + t2 - arg,
 926                                                     ARRAY_TOO_BIG);
 927                                 goto out;
 928                         }
 929                 }
 930         }
 931
 932         /*
 933          * Since $comm and immediate string can not be dereferenced,
 934          * we can find those by strcmp. But ignore for eprobes.
 935          */
 936         if (!(ctx->flags & TPARG_FL_TEVENT) &&
 937             (strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 ||
 938              strncmp(arg, "\\\"", 2) == 0)) {
 939                 /* The type of $comm must be "string", and not an array. */
 940                 if (parg->count || (t && strcmp(t, "string")))
 941                         goto out;
 942                 parg->type = find_fetch_type("string", ctx->flags);
 943         } else
 944                 parg->type = find_fetch_type(t, ctx->flags);
 945         if (!parg->type) {
 946                 trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0), BAD_TYPE);
 947                 goto out;
 948         }
 949         parg->offset = *size;
 950         *size += parg->type->size * (parg->count ?: 1);
 951
 952         ret = -ENOMEM;
 953         if (parg->count) {
 954                 len = strlen(parg->type->fmttype) + 6;
 955                 parg->fmt = kmalloc(len, GFP_KERNEL);
 956                 if (!parg->fmt)
 957                         goto out;
 958                 snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
 959                          parg->count);
 960         }
 961
 962         code = tmp = kcalloc(FETCH_INSN_MAX, sizeof(*code), GFP_KERNEL);
 963         if (!code)
 964                 goto out;
 965         code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
 966
 967         ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
 968                               ctx);
 969         if (ret)
 970                 goto fail;
 971
 972         /* Update storing type if BTF is available */
 973         if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) && !t) {
 974                 if (code->op == FETCH_OP_ARG)
 975                         parg->type = parse_btf_arg_type(code->param, ctx);
 976                 else if (code->op == FETCH_OP_RETVAL)
 977                         parg->type = parse_btf_retval_type(ctx);
 978         }
 979
 980         ret = -EINVAL;
 981         /* Store operation */
 982         if (parg->type->is_string) {
 983                 if (!strcmp(parg->type->name, "symstr")) {
 984                         if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK &&
 985                             code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG &&
 986                             code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) {
 987                                 trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
 988                                                     BAD_SYMSTRING);
 989                                 goto fail;
 990                         }
 991                 } else {
 992                         if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
 993                             code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
 994                             code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) {
 995                                 trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
 996                                                     BAD_STRING);
 997                                 goto fail;
 998                         }
 999                 }
1000                 if (!strcmp(parg->type->name, "symstr") ||
1001                     (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
1002                      code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG ||
1003                      parg->count) {
1004                         /*
1005                          * IMM, DATA and COMM is pointing actual address, those
1006                          * must be kept, and if parg->count != 0, this is an
1007                          * array of string pointers instead of string address
1008                          * itself.
1009                          * For the symstr, it doesn't need to dereference, thus
1010                          * it just get the value.
1011                          */
1012                         code++;
1013                         if (code->op != FETCH_OP_NOP) {
1014                                 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1015                                 goto fail;
1016                         }
1017                 }
1018                 /* If op == DEREF, replace it with STRING */
1019                 if (!strcmp(parg->type->name, "ustring") ||
1020                     code->op == FETCH_OP_UDEREF)
1021                         code->op = FETCH_OP_ST_USTRING;
1022                 else if (!strcmp(parg->type->name, "symstr"))
1023                         code->op = FETCH_OP_ST_SYMSTR;
1024                 else
1025                         code->op = FETCH_OP_ST_STRING;
1026                 code->size = parg->type->size;
1027                 parg->dynamic = true;
1028         } else if (code->op == FETCH_OP_DEREF) {
1029                 code->op = FETCH_OP_ST_MEM;
1030                 code->size = parg->type->size;
1031         } else if (code->op == FETCH_OP_UDEREF) {
1032                 code->op = FETCH_OP_ST_UMEM;
1033                 code->size = parg->type->size;
1034         } else {
1035                 code++;
1036                 if (code->op != FETCH_OP_NOP) {
1037                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1038                         goto fail;
1039                 }
1040                 code->op = FETCH_OP_ST_RAW;
1041                 code->size = parg->type->size;
1042         }
1043         scode = code;
1044         /* Modify operation */
1045         if (t != NULL) {
1046                 ret = __parse_bitfield_probe_arg(t, parg->type, &code);
1047                 if (ret) {
1048                         trace_probe_log_err(ctx->offset + t - arg, BAD_BITFIELD);
1049                         goto fail;
1050                 }
1051         }
1052         ret = -EINVAL;
1053         /* Loop(Array) operation */
1054         if (parg->count) {
1055                 if (scode->op != FETCH_OP_ST_MEM &&
1056                     scode->op != FETCH_OP_ST_STRING &&
1057                     scode->op != FETCH_OP_ST_USTRING) {
1058                         trace_probe_log_err(ctx->offset + (t ? (t - arg) : 0),
1059                                             BAD_STRING);
1060                         goto fail;
1061                 }
1062                 code++;
1063                 if (code->op != FETCH_OP_NOP) {
1064                         trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1065                         goto fail;
1066                 }
1067                 code->op = FETCH_OP_LP_ARRAY;
1068                 code->param = parg->count;
1069         }
1070         code++;
1071         code->op = FETCH_OP_END;
1072
1073         ret = 0;
1074         /* Shrink down the code buffer */
1075         parg->code = kcalloc(code - tmp + 1, sizeof(*code), GFP_KERNEL);
1076         if (!parg->code)
1077                 ret = -ENOMEM;
1078         else
1079                 memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
1080
1081 fail:
1082         if (ret) {
1083                 for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
1084                         if (code->op == FETCH_NOP_SYMBOL ||
1085                             code->op == FETCH_OP_DATA)
1086                                 kfree(code->data);
1087         }
1088         kfree(tmp);
1089 out:
1090         kfree(arg);
1091
1092         return ret;
1093 }
1094
1095 /* Return 1 if name is reserved or already used by another argument */
1096 static int traceprobe_conflict_field_name(const char *name,
1097                                           struct probe_arg *args, int narg)
1098 {
1099         int i;
1100
1101         for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
1102                 if (strcmp(reserved_field_names[i], name) == 0)
1103                         return 1;
1104
1105         for (i = 0; i < narg; i++)
1106                 if (strcmp(args[i].name, name) == 0)
1107                         return 1;
1108
1109         return 0;
1110 }
1111
1112 static char *generate_probe_arg_name(const char *arg, int idx)
1113 {
1114         char *name = NULL;
1115         const char *end;
1116
1117         /*
1118          * If argument name is omitted, try arg as a name (BTF variable)
1119          * or "argN".
1120          */
1121         if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
1122                 end = strchr(arg, ':');
1123                 if (!end)
1124                         end = arg + strlen(arg);
1125
1126                 name = kmemdup_nul(arg, end - arg, GFP_KERNEL);
1127                 if (!name || !is_good_name(name)) {
1128                         kfree(name);
1129                         name = NULL;
1130                 }
1131         }
1132
1133         if (!name)
1134                 name = kasprintf(GFP_KERNEL, "arg%d", idx + 1);
1135
1136         return name;
1137 }
1138
1139 int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
1140                                struct traceprobe_parse_context *ctx)
1141 {
1142         struct probe_arg *parg = &tp->args[i];
1143         const char *body;
1144
1145         /* Increment count for freeing args in error case */
1146         tp->nr_args++;
1147
1148         body = strchr(arg, '=');
1149         if (body) {
1150                 if (body - arg > MAX_ARG_NAME_LEN) {
1151                         trace_probe_log_err(0, ARG_NAME_TOO_LONG);
1152                         return -EINVAL;
1153                 } else if (body == arg) {
1154                         trace_probe_log_err(0, NO_ARG_NAME);
1155                         return -EINVAL;
1156                 }
1157                 parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL);
1158                 body++;
1159         } else {
1160                 parg->name = generate_probe_arg_name(arg, i);
1161                 body = arg;
1162         }
1163         if (!parg->name)
1164                 return -ENOMEM;
1165
1166         if (!is_good_name(parg->name)) {
1167                 trace_probe_log_err(0, BAD_ARG_NAME);
1168                 return -EINVAL;
1169         }
1170         if (traceprobe_conflict_field_name(parg->name, tp->args, i)) {
1171                 trace_probe_log_err(0, USED_ARG_NAME);
1172                 return -EINVAL;
1173         }
1174         ctx->offset = body - arg;
1175         /* Parse fetch argument */
1176         return traceprobe_parse_probe_arg_body(body, &tp->size, parg, ctx);
1177 }
1178
1179 void traceprobe_free_probe_arg(struct probe_arg *arg)
1180 {
1181         struct fetch_insn *code = arg->code;
1182
1183         while (code && code->op != FETCH_OP_END) {
1184                 if (code->op == FETCH_NOP_SYMBOL ||
1185                     code->op == FETCH_OP_DATA)
1186                         kfree(code->data);
1187                 code++;
1188         }
1189         kfree(arg->code);
1190         kfree(arg->name);
1191         kfree(arg->comm);
1192         kfree(arg->fmt);
1193 }
1194
1195 static int argv_has_var_arg(int argc, const char *argv[], int *args_idx,
1196                             struct traceprobe_parse_context *ctx)
1197 {
1198         int i, found = 0;
1199
1200         for (i = 0; i < argc; i++)
1201                 if (str_has_prefix(argv[i], "$arg")) {
1202                         trace_probe_log_set_index(i + 2);
1203
1204                         if (!tparg_is_function_entry(ctx->flags)) {
1205                                 trace_probe_log_err(0, NOFENTRY_ARGS);
1206                                 return -EINVAL;
1207                         }
1208
1209                         if (isdigit(argv[i][4])) {
1210                                 found = 1;
1211                                 continue;
1212                         }
1213
1214                         if (argv[i][4] != '*') {
1215                                 trace_probe_log_err(0, BAD_VAR);
1216                                 return -EINVAL;
1217                         }
1218
1219                         if (*args_idx >= 0 && *args_idx < argc) {
1220                                 trace_probe_log_err(0, DOUBLE_ARGS);
1221                                 return -EINVAL;
1222                         }
1223                         found = 1;
1224                         *args_idx = i;
1225                 }
1226
1227         return found;
1228 }
1229
1230 static int sprint_nth_btf_arg(int idx, const char *type,
1231                               char *buf, int bufsize,
1232                               struct traceprobe_parse_context *ctx)
1233 {
1234         struct btf *btf = traceprobe_get_btf();
1235         const char *name;
1236         int ret;
1237
1238         if (idx >= ctx->nr_params) {
1239                 trace_probe_log_err(0, NO_BTFARG);
1240                 return -ENOENT;
1241         }
1242         name = btf_name_by_offset(btf, ctx->params[idx].name_off);
1243         if (!name) {
1244                 trace_probe_log_err(0, NO_BTF_ENTRY);
1245                 return -ENOENT;
1246         }
1247         ret = snprintf(buf, bufsize, "%s%s", name, type);
1248         if (ret >= bufsize) {
1249                 trace_probe_log_err(0, ARGS_2LONG);
1250                 return -E2BIG;
1251         }
1252         return ret;
1253 }
1254
1255 /* Return new_argv which must be freed after use */
1256 const char **traceprobe_expand_meta_args(int argc, const char *argv[],
1257                                          int *new_argc, char *buf, int bufsize,
1258                                          struct traceprobe_parse_context *ctx)
1259 {
1260         const struct btf_param *params = NULL;
1261         int i, j, n, used, ret, args_idx = -1;
1262         const char **new_argv = NULL;
1263         int nr_params;
1264
1265         ret = argv_has_var_arg(argc, argv, &args_idx, ctx);
1266         if (ret < 0)
1267                 return ERR_PTR(ret);
1268
1269         if (!ret) {
1270                 *new_argc = argc;
1271                 return NULL;
1272         }
1273
1274         params = find_btf_func_param(ctx->funcname, &nr_params,
1275                                      ctx->flags & TPARG_FL_TPOINT);
1276         if (IS_ERR(params)) {
1277                 if (args_idx != -1) {
1278                         /* $arg* requires BTF info */
1279                         trace_probe_log_err(0, NOSUP_BTFARG);
1280                         return (const char **)params;
1281                 }
1282                 *new_argc = argc;
1283                 return NULL;
1284         }
1285         ctx->params = params;
1286         ctx->nr_params = nr_params;
1287
1288         if (args_idx >= 0)
1289                 *new_argc = argc + ctx->nr_params - 1;
1290         else
1291                 *new_argc = argc;
1292
1293         new_argv = kcalloc(*new_argc, sizeof(char *), GFP_KERNEL);
1294         if (!new_argv)
1295                 return ERR_PTR(-ENOMEM);
1296
1297         used = 0;
1298         for (i = 0, j = 0; i < argc; i++) {
1299                 trace_probe_log_set_index(i + 2);
1300                 if (i == args_idx) {
1301                         for (n = 0; n < nr_params; n++) {
1302                                 ret = sprint_nth_btf_arg(n, "", buf + used,
1303                                                          bufsize - used, ctx);
1304                                 if (ret < 0)
1305                                         goto error;
1306
1307                                 new_argv[j++] = buf + used;
1308                                 used += ret + 1;
1309                         }
1310                         continue;
1311                 }
1312
1313                 if (str_has_prefix(argv[i], "$arg")) {
1314                         char *type = NULL;
1315
1316                         n = simple_strtoul(argv[i] + 4, &type, 10);
1317                         if (type && !(*type == ':' || *type == '\0')) {
1318                                 trace_probe_log_err(0, BAD_VAR);
1319                                 ret = -ENOENT;
1320                                 goto error;
1321                         }
1322                         /* Note: $argN starts from $arg1 */
1323                         ret = sprint_nth_btf_arg(n - 1, type, buf + used,
1324                                                  bufsize - used, ctx);
1325                         if (ret < 0)
1326                                 goto error;
1327                         new_argv[j++] = buf + used;
1328                         used += ret + 1;
1329                 } else
1330                         new_argv[j++] = argv[i];
1331         }
1332
1333         return new_argv;
1334
1335 error:
1336         kfree(new_argv);
1337         return ERR_PTR(ret);
1338 }
1339
1340 int traceprobe_update_arg(struct probe_arg *arg)
1341 {
1342         struct fetch_insn *code = arg->code;
1343         long offset;
1344         char *tmp;
1345         char c;
1346         int ret = 0;
1347
1348         while (code && code->op != FETCH_OP_END) {
1349                 if (code->op == FETCH_NOP_SYMBOL) {
1350                         if (code[1].op != FETCH_OP_IMM)
1351                                 return -EINVAL;
1352
1353                         tmp = strpbrk(code->data, "+-");
1354                         if (tmp)
1355                                 c = *tmp;
1356                         ret = traceprobe_split_symbol_offset(code->data,
1357                                                              &offset);
1358                         if (ret)
1359                                 return ret;
1360
1361                         code[1].immediate =
1362                                 (unsigned long)kallsyms_lookup_name(code->data);
1363                         if (tmp)
1364                                 *tmp = c;
1365                         if (!code[1].immediate)
1366                                 return -ENOENT;
1367                         code[1].immediate += offset;
1368                 }
1369                 code++;
1370         }
1371         return 0;
1372 }
1373
1374 /* When len=0, we just calculate the needed length */
1375 #define LEN_OR_ZERO (len ? len - pos : 0)
1376 static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
1377                            enum probe_print_type ptype)
1378 {
1379         struct probe_arg *parg;
1380         int i, j;
1381         int pos = 0;
1382         const char *fmt, *arg;
1383
1384         switch (ptype) {
1385         case PROBE_PRINT_NORMAL:
1386                 fmt = "(%lx)";
1387                 arg = ", REC->" FIELD_STRING_IP;
1388                 break;
1389         case PROBE_PRINT_RETURN:
1390                 fmt = "(%lx <- %lx)";
1391                 arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
1392                 break;
1393         case PROBE_PRINT_EVENT:
1394                 fmt = "";
1395                 arg = "";
1396                 break;
1397         default:
1398                 WARN_ON_ONCE(1);
1399                 return 0;
1400         }
1401
1402         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
1403
1404         for (i = 0; i < tp->nr_args; i++) {
1405                 parg = tp->args + i;
1406                 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
1407                 if (parg->count) {
1408                         pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
1409                                         parg->type->fmt);
1410                         for (j = 1; j < parg->count; j++)
1411                                 pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
1412                                                 parg->type->fmt);
1413                         pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
1414                 } else
1415                         pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
1416                                         parg->type->fmt);
1417         }
1418
1419         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", arg);
1420
1421         for (i = 0; i < tp->nr_args; i++) {
1422                 parg = tp->args + i;
1423                 if (parg->count) {
1424                         if (parg->type->is_string)
1425                                 fmt = ", __get_str(%s[%d])";
1426                         else
1427                                 fmt = ", REC->%s[%d]";
1428                         for (j = 0; j < parg->count; j++)
1429                                 pos += snprintf(buf + pos, LEN_OR_ZERO,
1430                                                 fmt, parg->name, j);
1431                 } else {
1432                         if (parg->type->is_string)
1433                                 fmt = ", __get_str(%s)";
1434                         else
1435                                 fmt = ", REC->%s";
1436                         pos += snprintf(buf + pos, LEN_OR_ZERO,
1437                                         fmt, parg->name);
1438                 }
1439         }
1440
1441         /* return the length of print_fmt */
1442         return pos;
1443 }
1444 #undef LEN_OR_ZERO
1445
1446 int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
1447 {
1448         struct trace_event_call *call = trace_probe_event_call(tp);
1449         int len;
1450         char *print_fmt;
1451
1452         /* First: called with 0 length to calculate the needed length */
1453         len = __set_print_fmt(tp, NULL, 0, ptype);
1454         print_fmt = kmalloc(len + 1, GFP_KERNEL);
1455         if (!print_fmt)
1456                 return -ENOMEM;
1457
1458         /* Second: actually write the @print_fmt */
1459         __set_print_fmt(tp, print_fmt, len + 1, ptype);
1460         call->print_fmt = print_fmt;
1461
1462         return 0;
1463 }
1464
1465 int traceprobe_define_arg_fields(struct trace_event_call *event_call,
1466                                  size_t offset, struct trace_probe *tp)
1467 {
1468         int ret, i;
1469
1470         /* Set argument names as fields */
1471         for (i = 0; i < tp->nr_args; i++) {
1472                 struct probe_arg *parg = &tp->args[i];
1473                 const char *fmt = parg->type->fmttype;
1474                 int size = parg->type->size;
1475
1476                 if (parg->fmt)
1477                         fmt = parg->fmt;
1478                 if (parg->count)
1479                         size *= parg->count;
1480                 ret = trace_define_field(event_call, fmt, parg->name,
1481                                          offset + parg->offset, size,
1482                                          parg->type->is_signed,
1483                                          FILTER_OTHER);
1484                 if (ret)
1485                         return ret;
1486         }
1487         return 0;
1488 }
1489
1490 static void trace_probe_event_free(struct trace_probe_event *tpe)
1491 {
1492         kfree(tpe->class.system);
1493         kfree(tpe->call.name);
1494         kfree(tpe->call.print_fmt);
1495         kfree(tpe);
1496 }
1497
1498 int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
1499 {
1500         if (trace_probe_has_sibling(tp))
1501                 return -EBUSY;
1502
1503         list_del_init(&tp->list);
1504         trace_probe_event_free(tp->event);
1505
1506         tp->event = to->event;
1507         list_add_tail(&tp->list, trace_probe_probe_list(to));
1508
1509         return 0;
1510 }
1511
1512 void trace_probe_unlink(struct trace_probe *tp)
1513 {
1514         list_del_init(&tp->list);
1515         if (list_empty(trace_probe_probe_list(tp)))
1516                 trace_probe_event_free(tp->event);
1517         tp->event = NULL;
1518 }
1519
1520 void trace_probe_cleanup(struct trace_probe *tp)
1521 {
1522         int i;
1523
1524         for (i = 0; i < tp->nr_args; i++)
1525                 traceprobe_free_probe_arg(&tp->args[i]);
1526
1527         if (tp->event)
1528                 trace_probe_unlink(tp);
1529 }
1530
1531 int trace_probe_init(struct trace_probe *tp, const char *event,
1532                      const char *group, bool alloc_filter)
1533 {
1534         struct trace_event_call *call;
1535         size_t size = sizeof(struct trace_probe_event);
1536         int ret = 0;
1537
1538         if (!event || !group)
1539                 return -EINVAL;
1540
1541         if (alloc_filter)
1542                 size += sizeof(struct trace_uprobe_filter);
1543
1544         tp->event = kzalloc(size, GFP_KERNEL);
1545         if (!tp->event)
1546                 return -ENOMEM;
1547
1548         INIT_LIST_HEAD(&tp->event->files);
1549         INIT_LIST_HEAD(&tp->event->class.fields);
1550         INIT_LIST_HEAD(&tp->event->probes);
1551         INIT_LIST_HEAD(&tp->list);
1552         list_add(&tp->list, &tp->event->probes);
1553
1554         call = trace_probe_event_call(tp);
1555         call->class = &tp->event->class;
1556         call->name = kstrdup(event, GFP_KERNEL);
1557         if (!call->name) {
1558                 ret = -ENOMEM;
1559                 goto error;
1560         }
1561
1562         tp->event->class.system = kstrdup(group, GFP_KERNEL);
1563         if (!tp->event->class.system) {
1564                 ret = -ENOMEM;
1565                 goto error;
1566         }
1567
1568         return 0;
1569
1570 error:
1571         trace_probe_cleanup(tp);
1572         return ret;
1573 }
1574
1575 static struct trace_event_call *
1576 find_trace_event_call(const char *system, const char *event_name)
1577 {
1578         struct trace_event_call *tp_event;
1579         const char *name;
1580
1581         list_for_each_entry(tp_event, &ftrace_events, list) {
1582                 if (!tp_event->class->system ||
1583                     strcmp(system, tp_event->class->system))
1584                         continue;
1585                 name = trace_event_name(tp_event);
1586                 if (!name || strcmp(event_name, name))
1587                         continue;
1588                 return tp_event;
1589         }
1590
1591         return NULL;
1592 }
1593
1594 int trace_probe_register_event_call(struct trace_probe *tp)
1595 {
1596         struct trace_event_call *call = trace_probe_event_call(tp);
1597         int ret;
1598
1599         lockdep_assert_held(&event_mutex);
1600
1601         if (find_trace_event_call(trace_probe_group_name(tp),
1602                                   trace_probe_name(tp)))
1603                 return -EEXIST;
1604
1605         ret = register_trace_event(&call->event);
1606         if (!ret)
1607                 return -ENODEV;
1608
1609         ret = trace_add_event_call(call);
1610         if (ret)
1611                 unregister_trace_event(&call->event);
1612
1613         return ret;
1614 }
1615
1616 int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
1617 {
1618         struct event_file_link *link;
1619
1620         link = kmalloc(sizeof(*link), GFP_KERNEL);
1621         if (!link)
1622                 return -ENOMEM;
1623
1624         link->file = file;
1625         INIT_LIST_HEAD(&link->list);
1626         list_add_tail_rcu(&link->list, &tp->event->files);
1627         trace_probe_set_flag(tp, TP_FLAG_TRACE);
1628         return 0;
1629 }
1630
1631 struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
1632                                                   struct trace_event_file *file)
1633 {
1634         struct event_file_link *link;
1635
1636         trace_probe_for_each_link(link, tp) {
1637                 if (link->file == file)
1638                         return link;
1639         }
1640
1641         return NULL;
1642 }
1643
1644 int trace_probe_remove_file(struct trace_probe *tp,
1645                             struct trace_event_file *file)
1646 {
1647         struct event_file_link *link;
1648
1649         link = trace_probe_get_file_link(tp, file);
1650         if (!link)
1651                 return -ENOENT;
1652
1653         list_del_rcu(&link->list);
1654         kvfree_rcu_mightsleep(link);
1655
1656         if (list_empty(&tp->event->files))
1657                 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
1658
1659         return 0;
1660 }
1661
1662 /*
1663  * Return the smallest index of different type argument (start from 1).
1664  * If all argument types and name are same, return 0.
1665  */
1666 int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
1667 {
1668         int i;
1669
1670         /* In case of more arguments */
1671         if (a->nr_args < b->nr_args)
1672                 return a->nr_args + 1;
1673         if (a->nr_args > b->nr_args)
1674                 return b->nr_args + 1;
1675
1676         for (i = 0; i < a->nr_args; i++) {
1677                 if ((b->nr_args <= i) ||
1678                     ((a->args[i].type != b->args[i].type) ||
1679                      (a->args[i].count != b->args[i].count) ||
1680                      strcmp(a->args[i].name, b->args[i].name)))
1681                         return i + 1;
1682         }
1683
1684         return 0;
1685 }
1686
1687 bool trace_probe_match_command_args(struct trace_probe *tp,
1688                                     int argc, const char **argv)
1689 {
1690         char buf[MAX_ARGSTR_LEN + 1];
1691         int i;
1692
1693         if (tp->nr_args < argc)
1694                 return false;
1695
1696         for (i = 0; i < argc; i++) {
1697                 snprintf(buf, sizeof(buf), "%s=%s",
1698                          tp->args[i].name, tp->args[i].comm);
1699                 if (strcmp(buf, argv[i]))
1700                         return false;
1701         }
1702         return true;
1703 }
1704
1705 int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
1706 {
1707         int argc = 0, ret = 0;
1708         char **argv;
1709
1710         argv = argv_split(GFP_KERNEL, raw_command, &argc);
1711         if (!argv)
1712                 return -ENOMEM;
1713
1714         if (argc)
1715                 ret = createfn(argc, (const char **)argv);
1716
1717         argv_free(argv);
1718
1719         return ret;
1720 }
1721
1722 int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
1723                  u8 *data, void *field)
1724 {
1725         void *p;
1726         int i, j;
1727
1728         for (i = 0; i < nr_args; i++) {
1729                 struct probe_arg *a = args + i;
1730
1731                 trace_seq_printf(s, " %s=", a->name);
1732                 if (likely(!a->count)) {
1733                         if (!a->type->print(s, data + a->offset, field))
1734                                 return -ENOMEM;
1735                         continue;
1736                 }
1737                 trace_seq_putc(s, '{');
1738                 p = data + a->offset;
1739                 for (j = 0; j < a->count; j++) {
1740                         if (!a->type->print(s, p, field))
1741                                 return -ENOMEM;
1742                         trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
1743                         p += a->type->size;
1744                 }
1745         }
1746         return 0;
1747 }