kernel/bpf/bpf_iter.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2020 Facebook */
   3
   4 #include <linux/fs.h>
   5 #include <linux/anon_inodes.h>
   6 #include <linux/filter.h>
   7 #include <linux/bpf.h>
   8 #include <linux/rcupdate_trace.h>
   9
  10 struct bpf_iter_target_info {
  11         struct list_head list;
  12         const struct bpf_iter_reg *reg_info;
  13         u32 btf_id;     /* cached value */
  14 };
  15
  16 struct bpf_iter_link {
  17         struct bpf_link link;
  18         struct bpf_iter_aux_info aux;
  19         struct bpf_iter_target_info *tinfo;
  20 };
  21
  22 struct bpf_iter_priv_data {
  23         struct bpf_iter_target_info *tinfo;
  24         const struct bpf_iter_seq_info *seq_info;
  25         struct bpf_prog *prog;
  26         u64 session_id;
  27         u64 seq_num;
  28         bool done_stop;
  29         u8 target_private[] __aligned(8);
  30 };
  31
  32 static struct list_head targets = LIST_HEAD_INIT(targets);
  33 static DEFINE_MUTEX(targets_mutex);
  34
  35 /* protect bpf_iter_link changes */
  36 static DEFINE_MUTEX(link_mutex);
  37
  38 /* incremented on every opened seq_file */
  39 static atomic64_t session_id;
  40
  41 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
  42                             const struct bpf_iter_seq_info *seq_info);
  43
  44 static void bpf_iter_inc_seq_num(struct seq_file *seq)
  45 {
  46         struct bpf_iter_priv_data *iter_priv;
  47
  48         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  49                                  target_private);
  50         iter_priv->seq_num++;
  51 }
  52
  53 static void bpf_iter_dec_seq_num(struct seq_file *seq)
  54 {
  55         struct bpf_iter_priv_data *iter_priv;
  56
  57         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  58                                  target_private);
  59         iter_priv->seq_num--;
  60 }
  61
  62 static void bpf_iter_done_stop(struct seq_file *seq)
  63 {
  64         struct bpf_iter_priv_data *iter_priv;
  65
  66         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  67                                  target_private);
  68         iter_priv->done_stop = true;
  69 }
  70
  71 static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_info *tinfo)
  72 {
  73         return tinfo->reg_info->feature & BPF_ITER_RESCHED;
  74 }
  75
  76 static bool bpf_iter_support_resched(struct seq_file *seq)
  77 {
  78         struct bpf_iter_priv_data *iter_priv;
  79
  80         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
  81                                  target_private);
  82         return bpf_iter_target_support_resched(iter_priv->tinfo);
  83 }
  84
  85 /* maximum visited objects before bailing out */
  86 #define MAX_ITER_OBJECTS        1000000
  87
  88 /* bpf_seq_read, a customized and simpler version for bpf iterator.
  89  * The following are differences from seq_read():
  90  *  . fixed buffer size (PAGE_SIZE)
  91  *  . assuming NULL ->llseek()
  92  *  . stop() may call bpf program, handling potential overflow there
  93  */
  94 static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
  95                             loff_t *ppos)
  96 {
  97         struct seq_file *seq = file->private_data;
  98         size_t n, offs, copied = 0;
  99         int err = 0, num_objs = 0;
 100         bool can_resched;
 101         void *p;
 102
 103         mutex_lock(&seq->lock);
 104
 105         if (!seq->buf) {
 106                 seq->size = PAGE_SIZE << 3;
 107                 seq->buf = kvmalloc(seq->size, GFP_KERNEL);
 108                 if (!seq->buf) {
 109                         err = -ENOMEM;
 110                         goto done;
 111                 }
 112         }
 113
 114         if (seq->count) {
 115                 n = min(seq->count, size);
 116                 err = copy_to_user(buf, seq->buf + seq->from, n);
 117                 if (err) {
 118                         err = -EFAULT;
 119                         goto done;
 120                 }
 121                 seq->count -= n;
 122                 seq->from += n;
 123                 copied = n;
 124                 goto done;
 125         }
 126
 127         seq->from = 0;
 128         p = seq->op->start(seq, &seq->index);
 129         if (!p)
 130                 goto stop;
 131         if (IS_ERR(p)) {
 132                 err = PTR_ERR(p);
 133                 seq->op->stop(seq, p);
 134                 seq->count = 0;
 135                 goto done;
 136         }
 137
 138         err = seq->op->show(seq, p);
 139         if (err > 0) {
 140                 /* object is skipped, decrease seq_num, so next
 141                  * valid object can reuse the same seq_num.
 142                  */
 143                 bpf_iter_dec_seq_num(seq);
 144                 seq->count = 0;
 145         } else if (err < 0 || seq_has_overflowed(seq)) {
 146                 if (!err)
 147                         err = -E2BIG;
 148                 seq->op->stop(seq, p);
 149                 seq->count = 0;
 150                 goto done;
 151         }
 152
 153         can_resched = bpf_iter_support_resched(seq);
 154         while (1) {
 155                 loff_t pos = seq->index;
 156
 157                 num_objs++;
 158                 offs = seq->count;
 159                 p = seq->op->next(seq, p, &seq->index);
 160                 if (pos == seq->index) {
 161                         pr_info_ratelimited("buggy seq_file .next function %ps "
 162                                 "did not updated position index\n",
 163                                 seq->op->next);
 164                         seq->index++;
 165                 }
 166
 167                 if (IS_ERR_OR_NULL(p))
 168                         break;
 169
 170                 /* got a valid next object, increase seq_num */
 171                 bpf_iter_inc_seq_num(seq);
 172
 173                 if (seq->count >= size)
 174                         break;
 175
 176                 if (num_objs >= MAX_ITER_OBJECTS) {
 177                         if (offs == 0) {
 178                                 err = -EAGAIN;
 179                                 seq->op->stop(seq, p);
 180                                 goto done;
 181                         }
 182                         break;
 183                 }
 184
 185                 err = seq->op->show(seq, p);
 186                 if (err > 0) {
 187                         bpf_iter_dec_seq_num(seq);
 188                         seq->count = offs;
 189                 } else if (err < 0 || seq_has_overflowed(seq)) {
 190                         seq->count = offs;
 191                         if (offs == 0) {
 192                                 if (!err)
 193                                         err = -E2BIG;
 194                                 seq->op->stop(seq, p);
 195                                 goto done;
 196                         }
 197                         break;
 198                 }
 199
 200                 if (can_resched)
 201                         cond_resched();
 202         }
 203 stop:
 204         offs = seq->count;
 205         if (IS_ERR(p)) {
 206                 seq->op->stop(seq, NULL);
 207                 err = PTR_ERR(p);
 208                 goto done;
 209         }
 210         /* bpf program called if !p */
 211         seq->op->stop(seq, p);
 212         if (!p) {
 213                 if (!seq_has_overflowed(seq)) {
 214                         bpf_iter_done_stop(seq);
 215                 } else {
 216                         seq->count = offs;
 217                         if (offs == 0) {
 218                                 err = -E2BIG;
 219                                 goto done;
 220                         }
 221                 }
 222         }
 223
 224         n = min(seq->count, size);
 225         err = copy_to_user(buf, seq->buf, n);
 226         if (err) {
 227                 err = -EFAULT;
 228                 goto done;
 229         }
 230         copied = n;
 231         seq->count -= n;
 232         seq->from = n;
 233 done:
 234         if (!copied)
 235                 copied = err;
 236         else
 237                 *ppos += copied;
 238         mutex_unlock(&seq->lock);
 239         return copied;
 240 }
 241
 242 static const struct bpf_iter_seq_info *
 243 __get_seq_info(struct bpf_iter_link *link)
 244 {
 245         const struct bpf_iter_seq_info *seq_info;
 246
 247         if (link->aux.map) {
 248                 seq_info = link->aux.map->ops->iter_seq_info;
 249                 if (seq_info)
 250                         return seq_info;
 251         }
 252
 253         return link->tinfo->reg_info->seq_info;
 254 }
 255
 256 static int iter_open(struct inode *inode, struct file *file)
 257 {
 258         struct bpf_iter_link *link = inode->i_private;
 259
 260         return prepare_seq_file(file, link, __get_seq_info(link));
 261 }
 262
 263 static int iter_release(struct inode *inode, struct file *file)
 264 {
 265         struct bpf_iter_priv_data *iter_priv;
 266         struct seq_file *seq;
 267
 268         seq = file->private_data;
 269         if (!seq)
 270                 return 0;
 271
 272         iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
 273                                  target_private);
 274
 275         if (iter_priv->seq_info->fini_seq_private)
 276                 iter_priv->seq_info->fini_seq_private(seq->private);
 277
 278         bpf_prog_put(iter_priv->prog);
 279         seq->private = iter_priv;
 280
 281         return seq_release_private(inode, file);
 282 }
 283
 284 const struct file_operations bpf_iter_fops = {
 285         .open           = iter_open,
 286         .llseek         = no_llseek,
 287         .read           = bpf_seq_read,
 288         .release        = iter_release,
 289 };
 290
 291 /* The argument reg_info will be cached in bpf_iter_target_info.
 292  * The common practice is to declare target reg_info as
 293  * a const static variable and passed as an argument to
 294  * bpf_iter_reg_target().
 295  */
 296 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
 297 {
 298         struct bpf_iter_target_info *tinfo;
 299
 300         tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
 301         if (!tinfo)
 302                 return -ENOMEM;
 303
 304         tinfo->reg_info = reg_info;
 305         INIT_LIST_HEAD(&tinfo->list);
 306
 307         mutex_lock(&targets_mutex);
 308         list_add(&tinfo->list, &targets);
 309         mutex_unlock(&targets_mutex);
 310
 311         return 0;
 312 }
 313
 314 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
 315 {
 316         struct bpf_iter_target_info *tinfo;
 317         bool found = false;
 318
 319         mutex_lock(&targets_mutex);
 320         list_for_each_entry(tinfo, &targets, list) {
 321                 if (reg_info == tinfo->reg_info) {
 322                         list_del(&tinfo->list);
 323                         kfree(tinfo);
 324                         found = true;
 325                         break;
 326                 }
 327         }
 328         mutex_unlock(&targets_mutex);
 329
 330         WARN_ON(found == false);
 331 }
 332
 333 static void cache_btf_id(struct bpf_iter_target_info *tinfo,
 334                          struct bpf_prog *prog)
 335 {
 336         tinfo->btf_id = prog->aux->attach_btf_id;
 337 }
 338
 339 bool bpf_iter_prog_supported(struct bpf_prog *prog)
 340 {
 341         const char *attach_fname = prog->aux->attach_func_name;
 342         struct bpf_iter_target_info *tinfo = NULL, *iter;
 343         u32 prog_btf_id = prog->aux->attach_btf_id;
 344         const char *prefix = BPF_ITER_FUNC_PREFIX;
 345         int prefix_len = strlen(prefix);
 346
 347         if (strncmp(attach_fname, prefix, prefix_len))
 348                 return false;
 349
 350         mutex_lock(&targets_mutex);
 351         list_for_each_entry(iter, &targets, list) {
 352                 if (iter->btf_id && iter->btf_id == prog_btf_id) {
 353                         tinfo = iter;
 354                         break;
 355                 }
 356                 if (!strcmp(attach_fname + prefix_len, iter->reg_info->target)) {
 357                         cache_btf_id(iter, prog);
 358                         tinfo = iter;
 359                         break;
 360                 }
 361         }
 362         mutex_unlock(&targets_mutex);
 363
 364         if (tinfo) {
 365                 prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
 366                 prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
 367         }
 368
 369         return tinfo != NULL;
 370 }
 371
 372 const struct bpf_func_proto *
 373 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 374 {
 375         const struct bpf_iter_target_info *tinfo;
 376         const struct bpf_func_proto *fn = NULL;
 377
 378         mutex_lock(&targets_mutex);
 379         list_for_each_entry(tinfo, &targets, list) {
 380                 if (tinfo->btf_id == prog->aux->attach_btf_id) {
 381                         const struct bpf_iter_reg *reg_info;
 382
 383                         reg_info = tinfo->reg_info;
 384                         if (reg_info->get_func_proto)
 385                                 fn = reg_info->get_func_proto(func_id, prog);
 386                         break;
 387                 }
 388         }
 389         mutex_unlock(&targets_mutex);
 390
 391         return fn;
 392 }
 393
 394 static void bpf_iter_link_release(struct bpf_link *link)
 395 {
 396         struct bpf_iter_link *iter_link =
 397                 container_of(link, struct bpf_iter_link, link);
 398
 399         if (iter_link->tinfo->reg_info->detach_target)
 400                 iter_link->tinfo->reg_info->detach_target(&iter_link->aux);
 401 }
 402
 403 static void bpf_iter_link_dealloc(struct bpf_link *link)
 404 {
 405         struct bpf_iter_link *iter_link =
 406                 container_of(link, struct bpf_iter_link, link);
 407
 408         kfree(iter_link);
 409 }
 410
 411 static int bpf_iter_link_replace(struct bpf_link *link,
 412                                  struct bpf_prog *new_prog,
 413                                  struct bpf_prog *old_prog)
 414 {
 415         int ret = 0;
 416
 417         mutex_lock(&link_mutex);
 418         if (old_prog && link->prog != old_prog) {
 419                 ret = -EPERM;
 420                 goto out_unlock;
 421         }
 422
 423         if (link->prog->type != new_prog->type ||
 424             link->prog->expected_attach_type != new_prog->expected_attach_type ||
 425             link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) {
 426                 ret = -EINVAL;
 427                 goto out_unlock;
 428         }
 429
 430         old_prog = xchg(&link->prog, new_prog);
 431         bpf_prog_put(old_prog);
 432
 433 out_unlock:
 434         mutex_unlock(&link_mutex);
 435         return ret;
 436 }
 437
 438 static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
 439                                       struct seq_file *seq)
 440 {
 441         struct bpf_iter_link *iter_link =
 442                 container_of(link, struct bpf_iter_link, link);
 443         bpf_iter_show_fdinfo_t show_fdinfo;
 444
 445         seq_printf(seq,
 446                    "target_name:\t%s\n",
 447                    iter_link->tinfo->reg_info->target);
 448
 449         show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
 450         if (show_fdinfo)
 451                 show_fdinfo(&iter_link->aux, seq);
 452 }
 453
 454 static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
 455                                         struct bpf_link_info *info)
 456 {
 457         struct bpf_iter_link *iter_link =
 458                 container_of(link, struct bpf_iter_link, link);
 459         char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
 460         bpf_iter_fill_link_info_t fill_link_info;
 461         u32 ulen = info->iter.target_name_len;
 462         const char *target_name;
 463         u32 target_len;
 464
 465         if (!ulen ^ !ubuf)
 466                 return -EINVAL;
 467
 468         target_name = iter_link->tinfo->reg_info->target;
 469         target_len =  strlen(target_name);
 470         info->iter.target_name_len = target_len + 1;
 471
 472         if (ubuf) {
 473                 if (ulen >= target_len + 1) {
 474                         if (copy_to_user(ubuf, target_name, target_len + 1))
 475                                 return -EFAULT;
 476                 } else {
 477                         char zero = '\0';
 478
 479                         if (copy_to_user(ubuf, target_name, ulen - 1))
 480                                 return -EFAULT;
 481                         if (put_user(zero, ubuf + ulen - 1))
 482                                 return -EFAULT;
 483                         return -ENOSPC;
 484                 }
 485         }
 486
 487         fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
 488         if (fill_link_info)
 489                 return fill_link_info(&iter_link->aux, info);
 490
 491         return 0;
 492 }
 493
 494 static const struct bpf_link_ops bpf_iter_link_lops = {
 495         .release = bpf_iter_link_release,
 496         .dealloc = bpf_iter_link_dealloc,
 497         .update_prog = bpf_iter_link_replace,
 498         .show_fdinfo = bpf_iter_link_show_fdinfo,
 499         .fill_link_info = bpf_iter_link_fill_link_info,
 500 };
 501
 502 bool bpf_link_is_iter(struct bpf_link *link)
 503 {
 504         return link->ops == &bpf_iter_link_lops;
 505 }
 506
 507 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
 508                          struct bpf_prog *prog)
 509 {
 510         struct bpf_iter_target_info *tinfo = NULL, *iter;
 511         struct bpf_link_primer link_primer;
 512         union bpf_iter_link_info linfo;
 513         struct bpf_iter_link *link;
 514         u32 prog_btf_id, linfo_len;
 515         bpfptr_t ulinfo;
 516         int err;
 517
 518         if (attr->link_create.target_fd || attr->link_create.flags)
 519                 return -EINVAL;
 520
 521         memset(&linfo, 0, sizeof(union bpf_iter_link_info));
 522
 523         ulinfo = make_bpfptr(attr->link_create.iter_info, uattr.is_kernel);
 524         linfo_len = attr->link_create.iter_info_len;
 525         if (bpfptr_is_null(ulinfo) ^ !linfo_len)
 526                 return -EINVAL;
 527
 528         if (!bpfptr_is_null(ulinfo)) {
 529                 err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo),
 530                                                linfo_len);
 531                 if (err)
 532                         return err;
 533                 linfo_len = min_t(u32, linfo_len, sizeof(linfo));
 534                 if (copy_from_bpfptr(&linfo, ulinfo, linfo_len))
 535                         return -EFAULT;
 536         }
 537
 538         prog_btf_id = prog->aux->attach_btf_id;
 539         mutex_lock(&targets_mutex);
 540         list_for_each_entry(iter, &targets, list) {
 541                 if (iter->btf_id == prog_btf_id) {
 542                         tinfo = iter;
 543                         break;
 544                 }
 545         }
 546         mutex_unlock(&targets_mutex);
 547         if (!tinfo)
 548                 return -ENOENT;
 549
 550         /* Only allow sleepable program for resched-able iterator */
 551         if (prog->aux->sleepable && !bpf_iter_target_support_resched(tinfo))
 552                 return -EINVAL;
 553
 554         link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
 555         if (!link)
 556                 return -ENOMEM;
 557
 558         bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
 559         link->tinfo = tinfo;
 560
 561         err = bpf_link_prime(&link->link, &link_primer);
 562         if (err) {
 563                 kfree(link);
 564                 return err;
 565         }
 566
 567         if (tinfo->reg_info->attach_target) {
 568                 err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux);
 569                 if (err) {
 570                         bpf_link_cleanup(&link_primer);
 571                         return err;
 572                 }
 573         }
 574
 575         return bpf_link_settle(&link_primer);
 576 }
 577
 578 static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
 579                           struct bpf_iter_target_info *tinfo,
 580                           const struct bpf_iter_seq_info *seq_info,
 581                           struct bpf_prog *prog)
 582 {
 583         priv_data->tinfo = tinfo;
 584         priv_data->seq_info = seq_info;
 585         priv_data->prog = prog;
 586         priv_data->session_id = atomic64_inc_return(&session_id);
 587         priv_data->seq_num = 0;
 588         priv_data->done_stop = false;
 589 }
 590
 591 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
 592                             const struct bpf_iter_seq_info *seq_info)
 593 {
 594         struct bpf_iter_priv_data *priv_data;
 595         struct bpf_iter_target_info *tinfo;
 596         struct bpf_prog *prog;
 597         u32 total_priv_dsize;
 598         struct seq_file *seq;
 599         int err = 0;
 600
 601         mutex_lock(&link_mutex);
 602         prog = link->link.prog;
 603         bpf_prog_inc(prog);
 604         mutex_unlock(&link_mutex);
 605
 606         tinfo = link->tinfo;
 607         total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
 608                            seq_info->seq_priv_size;
 609         priv_data = __seq_open_private(file, seq_info->seq_ops,
 610                                        total_priv_dsize);
 611         if (!priv_data) {
 612                 err = -ENOMEM;
 613                 goto release_prog;
 614         }
 615
 616         if (seq_info->init_seq_private) {
 617                 err = seq_info->init_seq_private(priv_data->target_private, &link->aux);
 618                 if (err)
 619                         goto release_seq_file;
 620         }
 621
 622         init_seq_meta(priv_data, tinfo, seq_info, prog);
 623         seq = file->private_data;
 624         seq->private = priv_data->target_private;
 625
 626         return 0;
 627
 628 release_seq_file:
 629         seq_release_private(file->f_inode, file);
 630         file->private_data = NULL;
 631 release_prog:
 632         bpf_prog_put(prog);
 633         return err;
 634 }
 635
 636 int bpf_iter_new_fd(struct bpf_link *link)
 637 {
 638         struct bpf_iter_link *iter_link;
 639         struct file *file;
 640         unsigned int flags;
 641         int err, fd;
 642
 643         if (link->ops != &bpf_iter_link_lops)
 644                 return -EINVAL;
 645
 646         flags = O_RDONLY | O_CLOEXEC;
 647         fd = get_unused_fd_flags(flags);
 648         if (fd < 0)
 649                 return fd;
 650
 651         file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags);
 652         if (IS_ERR(file)) {
 653                 err = PTR_ERR(file);
 654                 goto free_fd;
 655         }
 656
 657         iter_link = container_of(link, struct bpf_iter_link, link);
 658         err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link));
 659         if (err)
 660                 goto free_file;
 661
 662         fd_install(fd, file);
 663         return fd;
 664
 665 free_file:
 666         fput(file);
 667 free_fd:
 668         put_unused_fd(fd);
 669         return err;
 670 }
 671
 672 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
 673 {
 674         struct bpf_iter_priv_data *iter_priv;
 675         struct seq_file *seq;
 676         void *seq_priv;
 677
 678         seq = meta->seq;
 679         if (seq->file->f_op != &bpf_iter_fops)
 680                 return NULL;
 681
 682         seq_priv = seq->private;
 683         iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
 684                                  target_private);
 685
 686         if (in_stop && iter_priv->done_stop)
 687                 return NULL;
 688
 689         meta->session_id = iter_priv->session_id;
 690         meta->seq_num = iter_priv->seq_num;
 691
 692         return iter_priv->prog;
 693 }
 694
 695 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 696 {
 697         struct bpf_run_ctx run_ctx, *old_run_ctx;
 698         int ret;
 699
 700         if (prog->aux->sleepable) {
 701                 rcu_read_lock_trace();
 702                 migrate_disable();
 703                 might_fault();
 704                 old_run_ctx = bpf_set_run_ctx(&run_ctx);
 705                 ret = bpf_prog_run(prog, ctx);
 706                 bpf_reset_run_ctx(old_run_ctx);
 707                 migrate_enable();
 708                 rcu_read_unlock_trace();
 709         } else {
 710                 rcu_read_lock();
 711                 migrate_disable();
 712                 old_run_ctx = bpf_set_run_ctx(&run_ctx);
 713                 ret = bpf_prog_run(prog, ctx);
 714                 bpf_reset_run_ctx(old_run_ctx);
 715                 migrate_enable();
 716                 rcu_read_unlock();
 717         }
 718
 719         /* bpf program can only return 0 or 1:
 720          *  0 : okay
 721          *  1 : retry the same object
 722          * The bpf_iter_run_prog() return value
 723          * will be seq_ops->show() return value.
 724          */
 725         return ret == 0 ? 0 : -EAGAIN;
 726 }
 727
 728 BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn,
 729            void *, callback_ctx, u64, flags)
 730 {
 731         return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags);
 732 }
 733
 734 const struct bpf_func_proto bpf_for_each_map_elem_proto = {
 735         .func           = bpf_for_each_map_elem,
 736         .gpl_only       = false,
 737         .ret_type       = RET_INTEGER,
 738         .arg1_type      = ARG_CONST_MAP_PTR,
 739         .arg2_type      = ARG_PTR_TO_FUNC,
 740         .arg3_type      = ARG_PTR_TO_STACK_OR_NULL,
 741         .arg4_type      = ARG_ANYTHING,
 742 };
 743
 744 BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx,
 745            u64, flags)
 746 {
 747         bpf_callback_t callback = (bpf_callback_t)callback_fn;
 748         u64 ret;
 749         u32 i;
 750
 751         /* Note: these safety checks are also verified when bpf_loop
 752          * is inlined, be careful to modify this code in sync. See
 753          * function verifier.c:inline_bpf_loop.
 754          */
 755         if (flags)
 756                 return -EINVAL;
 757         if (nr_loops > BPF_MAX_LOOPS)
 758                 return -E2BIG;
 759
 760         for (i = 0; i < nr_loops; i++) {
 761                 ret = callback((u64)i, (u64)(long)callback_ctx, 0, 0, 0);
 762                 /* return value: 0 - continue, 1 - stop and return */
 763                 if (ret)
 764                         return i + 1;
 765         }
 766
 767         return i;
 768 }
 769
 770 const struct bpf_func_proto bpf_loop_proto = {
 771         .func           = bpf_loop,
 772         .gpl_only       = false,
 773         .ret_type       = RET_INTEGER,
 774         .arg1_type      = ARG_ANYTHING,
 775         .arg2_type      = ARG_PTR_TO_FUNC,
 776         .arg3_type      = ARG_PTR_TO_STACK_OR_NULL,
 777         .arg4_type      = ARG_ANYTHING,
 778 };
 779
 780 struct bpf_iter_num_kern {
 781         int cur; /* current value, inclusive */
 782         int end; /* final value, exclusive */
 783 } __aligned(8);
 784
 785 __diag_push();
 786 __diag_ignore_all("-Wmissing-prototypes",
 787                   "Global functions as their definitions will be in vmlinux BTF");
 788
 789 __bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end)
 790 {
 791         struct bpf_iter_num_kern *s = (void *)it;
 792
 793         BUILD_BUG_ON(sizeof(struct bpf_iter_num_kern) != sizeof(struct bpf_iter_num));
 794         BUILD_BUG_ON(__alignof__(struct bpf_iter_num_kern) != __alignof__(struct bpf_iter_num));
 795
 796         BTF_TYPE_EMIT(struct btf_iter_num);
 797
 798         /* start == end is legit, it's an empty range and we'll just get NULL
 799          * on first (and any subsequent) bpf_iter_num_next() call
 800          */
 801         if (start > end) {
 802                 s->cur = s->end = 0;
 803                 return -EINVAL;
 804         }
 805
 806         /* avoid overflows, e.g., if start == INT_MIN and end == INT_MAX */
 807         if ((s64)end - (s64)start > BPF_MAX_LOOPS) {
 808                 s->cur = s->end = 0;
 809                 return -E2BIG;
 810         }
 811
 812         /* user will call bpf_iter_num_next() first,
 813          * which will set s->cur to exactly start value;
 814          * underflow shouldn't matter
 815          */
 816         s->cur = start - 1;
 817         s->end = end;
 818
 819         return 0;
 820 }
 821
 822 __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it)
 823 {
 824         struct bpf_iter_num_kern *s = (void *)it;
 825
 826         /* check failed initialization or if we are done (same behavior);
 827          * need to be careful about overflow, so convert to s64 for checks,
 828          * e.g., if s->cur == s->end == INT_MAX, we can't just do
 829          * s->cur + 1 >= s->end
 830          */
 831         if ((s64)(s->cur + 1) >= s->end) {
 832                 s->cur = s->end = 0;
 833                 return NULL;
 834         }
 835
 836         s->cur++;
 837
 838         return &s->cur;
 839 }
 840
 841 __bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it)
 842 {
 843         struct bpf_iter_num_kern *s = (void *)it;
 844
 845         s->cur = s->end = 0;
 846 }
 847
 848 __diag_pop();