preload/preload_module.c

   1 #include <linux/module.h>
   2 #include <linux/dcache.h>
   3 #include <linux/namei.h>
   4 #include <linux/mman.h>
   5 #include <linux/err.h>
   6 #include <linux/types.h>
   7 #include <kprobe/swap_kprobes.h>
   8 #include <kprobe/swap_kprobes_deps.h>
   9 #include <us_manager/sspt/sspt_proc.h>
  10 #include <us_manager/sspt/sspt_ip.h>
  11 #include <us_manager/callbacks.h>
  12 #include <writer/kernel_operations.h>
  13 #include <master/swap_initializer.h>
  14 #include "preload.h"
  15 #include "preload_probe.h"
  16 #include "preload_debugfs.h"
  17 #include "preload_module.h"
  18 #include "preload_storage.h"
  19 #include "preload_control.h"
  20 #include "preload_threads.h"
  21 #include "preload_pd.h"
  22
  23
  24 struct us_priv {
  25         struct pt_regs regs;
  26         unsigned long arg0;
  27         unsigned long arg1;
  28         unsigned long raddr;
  29         unsigned long origin;
  30 };
  31
  32 static atomic_t dentry_balance = ATOMIC_INIT(0);
  33
  34 enum preload_status_t {
  35         SWAP_PRELOAD_NOT_READY = 0,
  36         SWAP_PRELOAD_READY = 1,
  37         SWAP_PRELOAD_RUNNING = 2
  38 };
  39
  40 static enum preload_status_t __preload_status = SWAP_PRELOAD_NOT_READY;
  41
  42 static int __preload_cbs_start_h = -1;
  43 static int __preload_cbs_stop_h = -1;
  44
  45
  46 static struct dentry *__get_dentry(struct dentry *dentry)
  47 {
  48         atomic_inc(&dentry_balance);
  49         return dget(dentry);
  50 }
  51
  52
  53
  54 bool preload_module_is_running(void)
  55 {
  56         if (__preload_status == SWAP_PRELOAD_RUNNING)
  57                 return true;
  58
  59         return false;
  60 }
  61
  62 bool preload_module_is_ready(void)
  63 {
  64         if (__preload_status == SWAP_PRELOAD_READY)
  65                 return true;
  66
  67         return false;
  68 }
  69
  70 bool preload_module_is_not_ready(void)
  71 {
  72         if (__preload_status == SWAP_PRELOAD_NOT_READY)
  73                 return true;
  74
  75         return false;
  76 }
  77
  78 void preload_module_set_ready(void)
  79 {
  80         __preload_status = SWAP_PRELOAD_READY;
  81 }
  82
  83 void preload_module_set_running(void)
  84 {
  85         __preload_status = SWAP_PRELOAD_RUNNING;
  86 }
  87
  88 void preload_module_set_not_ready(void)
  89 {
  90         __preload_status = SWAP_PRELOAD_NOT_READY;
  91 }
  92
  93 struct dentry *get_dentry(const char *filepath)
  94 {
  95         struct path path;
  96         struct dentry *dentry = NULL;
  97
  98         if (kern_path(filepath, LOOKUP_FOLLOW, &path) == 0) {
  99                 dentry = __get_dentry(path.dentry);
 100                 path_put(&path);
 101         }
 102
 103         return dentry;
 104 }
 105
 106 void put_dentry(struct dentry *dentry)
 107 {
 108         atomic_dec(&dentry_balance);
 109         dput(dentry);
 110 }
 111
 112 static inline void __prepare_ujump(struct uretprobe_instance *ri,
 113                                    struct pt_regs *regs,
 114                                    unsigned long vaddr)
 115 {
 116 #ifdef CONFIG_ARM
 117         ri->preload.use = true;
 118         ri->preload.thumb = !!thumb_mode(regs);
 119 #endif /* CONFIG_ARM */
 120
 121         swap_set_instr_ptr(regs, vaddr);
 122 }
 123
 124 static inline int __push(struct pt_regs *regs, void *buf, size_t len)
 125 {
 126         unsigned long sp = swap_get_stack_ptr(regs) - len;
 127
 128         sp = PTR_ALIGN(sp, sizeof(unsigned long));
 129         if (copy_to_user((void __user *)sp, buf, len))
 130                 return -EIO;
 131         swap_set_stack_ptr(regs, sp);
 132
 133         return 0;
 134 }
 135
 136 static inline void __save_uregs(struct uretprobe_instance *ri,
 137                                 struct pt_regs *regs)
 138 {
 139         struct us_priv *priv = (struct us_priv *)ri->data;
 140
 141         memcpy(ri->data, regs, sizeof(*regs));
 142         priv->arg0 = swap_get_arg(regs, 0);
 143         priv->arg1 = swap_get_arg(regs, 1);
 144         priv->raddr = swap_get_ret_addr(regs);
 145 }
 146
 147 static inline void __restore_uregs(struct uretprobe_instance *ri,
 148                                    struct pt_regs *regs)
 149 {
 150         struct us_priv *priv = (struct us_priv *)ri->data;
 151
 152         memcpy(regs, ri->data, sizeof(*regs));
 153         swap_set_arg(regs, 0, priv->arg0);
 154         swap_set_arg(regs, 1, priv->arg1);
 155         swap_set_ret_addr(regs, priv->raddr);
 156 #ifndef CONFIG_ARM
 157         /* need to do it only on x86 */
 158         regs->EREG(ip) -= 1;
 159 #endif /* !CONFIG_ARM */
 160         /* we have just restored the registers => no need to do it in
 161          * trampoline_uprobe_handler */
 162         ri->ret_addr = NULL;
 163 }
 164
 165 static inline void print_regs(const char *prefix, struct pt_regs *regs,
 166                               struct uretprobe_instance *ri, struct hd_t *hd)
 167 {
 168         struct dentry *dentry = preload_pd_get_dentry(hd);
 169
 170 #ifdef CONFIG_ARM
 171         printk(PRELOAD_PREFIX "%s[%d/%d] %s (%d) %s addr(%08lx), "
 172                "r0(%08lx), r1(%08lx), r2(%08lx), r3(%08lx), "
 173                "r4(%08lx), r5(%08lx), r6(%08lx), r7(%08lx), "
 174                "sp(%08lx), lr(%08lx), pc(%08lx)\n",
 175                current->comm, current->tgid, current->pid,
 176                dentry != NULL ? (char *)(dentry->d_name.name) :
 177                                 (char *)("NULL"),
 178                (int)preload_pd_get_state(hd),
 179                prefix, (unsigned long)ri->rp->up.addr,
 180                regs->ARM_r0, regs->ARM_r1, regs->ARM_r2, regs->ARM_r3,
 181                regs->ARM_r4, regs->ARM_r5, regs->ARM_r6, regs->ARM_r7,
 182                regs->ARM_sp, regs->ARM_lr, regs->ARM_pc);
 183 #else /* !CONFIG_ARM */
 184         printk(PRELOAD_PREFIX "%s[%d/%d] %s (%d) %s addr(%08lx), "
 185                "ip(%08lx), arg0(%08lx), arg1(%08lx), raddr(%08lx)\n",
 186                current->comm, current->tgid, current->pid,
 187                dentry != NULL ? (char *)(dentry->d_name.name) :
 188                                 (char *)("NULL"),
 189                (int)preload_pd_get_state(hd),
 190                prefix, (unsigned long)ri->rp->up.addr,
 191                regs->EREG(ip), swap_get_arg(regs, 0), swap_get_arg(regs, 1),
 192                swap_get_ret_addr(regs));
 193 #endif /* CONFIG_ARM */
 194 }
 195
 196 static inline unsigned long __get_r_debug_off(struct vm_area_struct *linker_vma)
 197 {
 198         unsigned long start_addr;
 199         unsigned long offset = preload_debugfs_r_debug_offset();
 200
 201         if (linker_vma == NULL)
 202                 return 0;
 203
 204         start_addr = linker_vma->vm_start;
 205
 206         return (offset ? start_addr + offset : 0);
 207 }
 208
 209 static struct vm_area_struct *__get_linker_vma(struct task_struct *task)
 210 {
 211         struct vm_area_struct *vma = NULL;
 212         struct bin_info *ld_info;
 213
 214         ld_info = preload_storage_get_linker_info();
 215         if (ld_info == NULL) {
 216                 printk(PRELOAD_PREFIX "Cannot get linker info [%u %u %s]!\n",
 217                        task->tgid, task->pid, task->comm);
 218                 return NULL;
 219         }
 220
 221         for (vma = task->mm->mmap; vma; vma = vma->vm_next) {
 222                 if (vma->vm_file && vma->vm_flags & VM_EXEC
 223                     && vma->vm_file->f_dentry == ld_info->dentry) {
 224                                 preload_storage_put_linker_info(ld_info);
 225                                 return vma;
 226                 }
 227         }
 228
 229         preload_storage_put_linker_info(ld_info);
 230         return NULL;
 231 }
 232
 233 static inline struct vm_area_struct *__get_vma_by_addr(struct task_struct *task,
 234                                                         unsigned long caller_addr)
 235 {
 236         struct vm_area_struct *vma = NULL;
 237
 238         if (task->mm == NULL)
 239                 return NULL;
 240         vma = find_vma_intersection(task->mm, caller_addr, caller_addr + 1);
 241
 242         return vma;
 243 }
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253 static bool __is_proc_mmap_mappable(struct task_struct *task)
 254 {
 255         struct vm_area_struct *linker_vma = __get_linker_vma(task);
 256         struct sspt_proc *proc;
 257         unsigned long r_debug_addr;
 258         unsigned int state;
 259         enum { r_state_offset = sizeof(int) + sizeof(void *) + sizeof(long) };
 260
 261         if (linker_vma == NULL)
 262                 return false;
 263
 264         r_debug_addr = __get_r_debug_off(linker_vma);
 265         if (r_debug_addr == 0)
 266                 return false;
 267
 268         r_debug_addr += r_state_offset;
 269         proc = sspt_proc_get_by_task(task);
 270         if (proc) {
 271                 proc->r_state_addr = r_debug_addr;
 272                 sspt_proc_put(proc);
 273         }
 274
 275         if (get_user(state, (unsigned long *)r_debug_addr))
 276                 return false;
 277
 278         return !state;
 279 }
 280
 281 static bool __should_we_preload_handlers(struct task_struct *task,
 282                                          struct pt_regs *regs)
 283 {
 284         unsigned long caller_addr = get_regs_ret_func(regs);
 285         struct vm_area_struct *cvma = __get_vma_by_addr(current, caller_addr);
 286
 287         if (!__is_proc_mmap_mappable(task) ||
 288             ((cvma != NULL) && (cvma->vm_file != NULL) &&
 289             (cvma->vm_file->f_path.dentry != NULL) &&
 290             preload_control_check_dentry_is_ignored(cvma->vm_file->f_path.dentry)))
 291                 return false;
 292
 293         return true;
 294 }
 295
 296
 297
 298
 299
 300 struct mmap_priv {
 301         struct dentry *dentry;
 302 };
 303
 304 static inline bool check_prot(unsigned long prot)
 305 {
 306         return !!((prot & PROT_READ) && (prot & PROT_EXEC));
 307 }
 308
 309 static int mmap_entry_handler(struct kretprobe_instance *ri,
 310                               struct pt_regs *regs)
 311 {
 312         struct file *file = (struct file *)swap_get_karg(regs, 0);
 313         unsigned long prot = swap_get_karg(regs, 3);
 314         struct mmap_priv *priv = (struct mmap_priv *)ri->data;
 315         struct task_struct *task = current->group_leader;
 316         struct dentry *dentry, *loader_dentry;
 317         struct pd_t *pd;
 318         struct hd_t *hd;
 319         struct sspt_proc *proc;
 320
 321         priv->dentry = NULL;
 322         if (!check_prot(prot))
 323                 return 0;
 324
 325         if (!file)
 326                 return 0;
 327
 328         dentry = file->f_dentry;
 329         if (dentry == NULL)
 330                 return 0;
 331
 332         loader_dentry = preload_debugfs_get_loader_dentry();
 333         if (dentry == loader_dentry) {
 334                 priv->dentry = loader_dentry;
 335                 return 0;
 336         }
 337
 338         proc = sspt_proc_by_task(task);
 339         if (!proc)
 340                 return 0;
 341
 342         pd = preload_pd_get(proc);
 343         if (pd == NULL) {
 344                 printk(PRELOAD_PREFIX "%d: No process data! Current %d %s\n",
 345                        __LINE__, current->tgid, current->comm);
 346                 return 0;
 347         }
 348
 349         hd = preload_pd_get_hd(pd, dentry);
 350         if (hd != NULL)
 351                 priv->dentry = preload_pd_get_dentry(hd);
 352
 353         return 0;
 354 }
 355
 356 static int mmap_ret_handler(struct kretprobe_instance *ri,
 357                             struct pt_regs *regs)
 358 {
 359         struct mmap_priv *priv = (struct mmap_priv *)ri->data;
 360         struct task_struct *task = current->group_leader;
 361         struct pd_t *pd;
 362         struct hd_t *hd;
 363         struct sspt_proc *proc;
 364         struct dentry *loader_dentry;
 365         unsigned long vaddr;
 366
 367         if (!task->mm)
 368                 return 0;
 369
 370         if (priv->dentry == NULL)
 371                 return 0;
 372
 373         vaddr = (unsigned long)regs_return_value(regs);
 374         if (IS_ERR_VALUE(vaddr))
 375                 return 0;
 376
 377         proc = sspt_proc_by_task(task);
 378         if (!proc)
 379                 return 0;
 380
 381         pd = preload_pd_get(proc);
 382         if (pd == NULL) {
 383                 printk(PRELOAD_PREFIX "%d: No process data! Current %d %s\n",
 384                        __LINE__, current->tgid, current->comm);
 385                 return 0;
 386         }
 387
 388         loader_dentry = preload_debugfs_get_loader_dentry();
 389         if (priv->dentry == loader_dentry)
 390                 preload_pd_set_loader_base(pd, vaddr);
 391
 392
 393         hd = preload_pd_get_hd(pd, priv->dentry);
 394         if (hd != NULL)
 395                 preload_pd_set_handlers_base(hd, vaddr);
 396
 397         return 0;
 398 }
 399
 400 static struct kretprobe mmap_rp = {
 401         .kp.symbol_name = "do_mmap_pgoff",
 402         .data_size = sizeof(struct mmap_priv),
 403         .entry_handler = mmap_entry_handler,
 404         .handler = mmap_ret_handler
 405 };
 406
 407 static void preload_start_cb(void)
 408 {
 409         int res;
 410
 411         res = swap_register_kretprobe(&mmap_rp);
 412         if (res != 0)
 413                 printk(KERN_ERR PRELOAD_PREFIX "Registering do_mmap_pgoff probe failed\n");
 414 }
 415
 416 static void preload_stop_cb(void)
 417 {
 418         swap_unregister_kretprobe(&mmap_rp);
 419 }
 420
 421 static unsigned long __not_loaded_entry(struct uretprobe_instance *ri,
 422                                         struct pt_regs *regs,
 423                                         struct pd_t *pd, struct hd_t *hd)
 424 {
 425         char __user *path = NULL;
 426         unsigned long vaddr = 0;
 427         unsigned long base;
 428
 429         /* if linker is still doing its work, we do nothing */
 430         if (!__should_we_preload_handlers(current, regs))
 431                 return 0;
 432
 433         base = preload_pd_get_loader_base(pd);
 434         if (base == 0)
 435                 return 0;   /* loader isn't mapped */
 436
 437         /* jump to loader code if ready */
 438         vaddr = base + preload_debugfs_get_loader_offset();
 439         if (vaddr) {
 440                 /* save original regs state */
 441                 __save_uregs(ri, regs);
 442                 print_regs("ORIG", regs, ri, hd);
 443
 444                 path = preload_pd_get_path(pd, hd);
 445
 446                 /* set dlopen args: filename, flags */
 447                 swap_set_arg(regs, 0, (unsigned long)path/*swap_get_stack_ptr(regs)*/);
 448                 swap_set_arg(regs, 1, 2 /* RTLD_NOW */);
 449
 450                 /* do the jump to dlopen */
 451                 __prepare_ujump(ri, regs, vaddr);
 452                 /* set new state */
 453                 preload_pd_set_state(hd, LOADING);
 454         }
 455
 456         return vaddr;
 457 }
 458
 459 static void __loading_ret(struct uretprobe_instance *ri, struct pt_regs *regs,
 460                           struct pd_t *pd, struct hd_t *hd)
 461 {
 462         struct us_priv *priv = (struct us_priv *)ri->data;
 463         unsigned long vaddr = 0;
 464
 465         /* check if preloading has been completed */
 466         vaddr = preload_pd_get_loader_base(pd) +
 467                 preload_debugfs_get_loader_offset();
 468         if (vaddr && (priv->origin == vaddr)) {
 469                 preload_pd_set_handle(hd,
 470                                       (void __user *)regs_return_value(regs));
 471
 472                 /* restore original regs state */
 473                 __restore_uregs(ri, regs);
 474                 print_regs("REST", regs, ri, hd);
 475                 /* check if preloading done */
 476
 477                 if (preload_pd_get_handle(hd)) {
 478                         preload_pd_set_state(hd, LOADED);
 479                 } else {
 480                         preload_pd_dec_attempts(hd);
 481                         preload_pd_set_state(hd, FAILED);
 482                 }
 483         }
 484 }
 485
 486 static void __failed_ret(struct uretprobe_instance *ri, struct pt_regs *regs,
 487                          struct pd_t *pd, struct hd_t *hd)
 488 {
 489         if (preload_pd_get_attempts(hd))
 490                 preload_pd_set_state(hd, NOT_LOADED);
 491 }
 492
 493
 494
 495 unsigned long preload_not_loaded_entry(struct uretprobe_instance *ri,
 496                                        struct pt_regs *regs, struct pd_t *pd,
 497                                        struct hd_t *hd)
 498 {
 499         return __not_loaded_entry(ri, regs, pd, hd);
 500 }
 501
 502 void preload_loading_ret(struct uretprobe_instance *ri, struct pt_regs *regs,
 503                          struct pd_t *pd, struct hd_t *hd)
 504 {
 505         __loading_ret(ri, regs, pd, hd);
 506 }
 507
 508 void preload_failed_ret(struct uretprobe_instance *ri, struct pt_regs *regs,
 509                         struct pd_t *pd, struct hd_t *hd)
 510 {
 511         __failed_ret(ri, regs, pd, hd);
 512 }
 513
 514 void preload_module_prepare_ujump(struct uretprobe_instance *ri,
 515                                   struct pt_regs *regs, unsigned long addr)
 516 {
 517         __prepare_ujump(ri, regs, addr);
 518 }
 519
 520 void preload_set_rp_data_size(struct uretprobe *rp)
 521 {
 522         rp->data_size = sizeof(struct us_priv);
 523 }
 524
 525 void preload_set_priv_origin(struct uretprobe_instance *ri, unsigned long addr)
 526 {
 527         struct us_priv *priv = (struct us_priv *)ri->data;
 528
 529         priv->origin = addr;
 530 }
 531
 532 unsigned long preload_get_priv_origin(struct uretprobe_instance *ri)
 533 {
 534         struct us_priv *priv = (struct us_priv *)ri->data;
 535
 536     return priv->origin;
 537 }
 538
 539
 540 int preload_set(void)
 541 {
 542         if (preload_module_is_running())
 543                 return -EBUSY;
 544
 545         return 0;
 546 }
 547
 548 void preload_unset(void)
 549 {
 550         swap_unregister_kretprobe(&mmap_rp);
 551         /*module_put(THIS_MODULE);*/
 552         preload_module_set_not_ready();
 553
 554 }
 555
 556
 557 static int preload_module_init(void)
 558 {
 559         int ret;
 560
 561         ret = preload_debugfs_init();
 562         if (ret)
 563                 goto out_err;
 564
 565         ret = preload_storage_init();
 566         if (ret)
 567                 goto exit_debugfs;
 568
 569         ret = preload_pd_init();
 570         if (ret)
 571                 goto exit_storage;
 572
 573         /* TODO do not forget to remove set (it is just for debugging) */
 574         ret = preload_set();
 575         if (ret)
 576                 goto exit_pd;
 577
 578         ret = preload_control_init();
 579         if (ret)
 580                 goto exit_set;
 581
 582         ret = preload_threads_init();
 583         if (ret)
 584                 goto exit_control;
 585
 586         ret = register_preload_probes();
 587         if (ret)
 588                 goto exit_threads;
 589
 590         __preload_cbs_start_h = us_manager_reg_cb(START_CB, preload_start_cb);
 591         if (__preload_cbs_start_h < 0)
 592                 goto exit_start_cb;
 593
 594         __preload_cbs_stop_h = us_manager_reg_cb(STOP_CB, preload_stop_cb);
 595         if (__preload_cbs_stop_h < 0)
 596                 goto exit_stop_cb;
 597
 598         return 0;
 599
 600 exit_stop_cb:
 601         us_manager_unreg_cb(__preload_cbs_start_h);
 602
 603 exit_start_cb:
 604         unregister_preload_probes();
 605
 606 exit_threads:
 607         preload_threads_exit();
 608
 609 exit_control:
 610         preload_control_exit();
 611
 612 exit_set:
 613         preload_unset();
 614
 615 exit_pd:
 616         preload_pd_uninit();
 617
 618 exit_storage:
 619         preload_storage_exit();
 620
 621 exit_debugfs:
 622         preload_debugfs_exit();
 623
 624 out_err:
 625         return ret;
 626 }
 627
 628 static void preload_module_exit(void)
 629 {
 630         int balance;
 631
 632         us_manager_unreg_cb(__preload_cbs_start_h);
 633         us_manager_unreg_cb(__preload_cbs_stop_h);
 634         unregister_preload_probes();
 635         preload_threads_exit();
 636         preload_control_exit();
 637         preload_unset();
 638         preload_pd_uninit();
 639         preload_storage_exit();
 640         preload_debugfs_exit();
 641
 642         balance = atomic_read(&dentry_balance);
 643         atomic_set(&dentry_balance, 0);
 644
 645         WARN(balance, "Bad GET/PUT dentry balance: %d\n", balance);
 646 }
 647
 648 SWAP_LIGHT_INIT_MODULE(NULL, preload_module_init, preload_module_exit,
 649                        NULL, NULL);
 650
 651 MODULE_LICENSE("GPL");
 652 MODULE_DESCRIPTION("SWAP Preload Module");
 653 MODULE_AUTHOR("Vasiliy Ulyanov <v.ulyanov@samsung.com>"
 654               "Alexander Aksenov <a.aksenov@samsung.com>");