driver/us_proc_inst.c

   1 ////////////////////////////////////////////////////////////////////////////////////
   2 //
   3 //      FILE:           us_proc_inst.c
   4 //
   5 //      DESCRIPTION:
   6 //      This file is C source for SWAP driver.
   7 //
   8 //      SEE ALSO:       us_proc_inst.h
   9 //      AUTHOR:         A.Gerenkov, E. Gorelkina
  10 //      COMPANY NAME:   Samsung Research Center in Moscow
  11 //      DEPT NAME:      Advanced Software Group
  12 //      CREATED:        2008.06.02
  13 //      VERSION:        1.0
  14 //      REVISION DATE:  2008.12.02
  15 //
  16 ////////////////////////////////////////////////////////////////////////////////////
  17
  18 #include "module.h"
  19 #include "us_proc_inst.h"
  20
  21 #include "../kprobe/dbi_kprobes_deps.h"
  22 #include "../uprobe/swap_uprobes.h"
  23
  24 #include "sspt/sspt.h"
  25
  26 #include <dbi_insn_slots.h>
  27
  28 #define mm_read_lock(task, mm, atomic, lock)                    \
  29         mm = atomic ? task->active_mm : get_task_mm(task);      \
  30         if (mm == NULL) {                                       \
  31                 /* FIXME: */                                    \
  32                 panic("ERRR mm_read_lock: mm == NULL\n");       \
  33         }                                                       \
  34                                                                 \
  35         if (atomic) {                                           \
  36                 lock = down_read_trylock(&mm->mmap_sem);        \
  37         } else {                                                \
  38                 lock = 1;                                       \
  39                 down_read(&mm->mmap_sem);                       \
  40         }
  41
  42 #define mm_read_unlock(mm, atomic, lock)                        \
  43         if (lock) {                                             \
  44                 up_read(&mm->mmap_sem);                         \
  45         }                                                       \
  46                                                                 \
  47         if (!atomic) {                                          \
  48                 mmput(mm);                                      \
  49         }
  50
  51 unsigned long ujprobe_event_pre_handler (struct us_ip *ip, struct pt_regs *regs);
  52 void ujprobe_event_handler (unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5, unsigned long arg6);
  53 int uretprobe_event_handler(struct uretprobe_instance *probe, struct pt_regs *regs, struct us_ip *ip);
  54
  55
  56 LIST_HEAD(proc_probes_list);
  57
  58 #define print_event(fmt, args...)                                               \
  59 {                                                                               \
  60         char *buf[1024];                                                        \
  61         sprintf(buf, fmt, ##args);                                              \
  62         pack_event_info(US_PROBE_ID, RECORD_ENTRY, "ds", 0x0badc0de, buf);      \
  63 }
  64
  65 static inline int is_libonly(void)
  66 {
  67         return !strcmp(us_proc_info.path,"*");
  68 }
  69
  70 // is user-space instrumentation
  71 static inline int is_us_instrumentation(void)
  72 {
  73         return !!us_proc_info.path;
  74 }
  75
  76 static unsigned long alloc_user_pages(struct task_struct *task, unsigned long len, unsigned long prot, unsigned long flags)
  77 {
  78         unsigned long ret = 0;
  79         struct task_struct *otask = current;
  80         struct mm_struct *mm;
  81         int atomic = in_atomic();
  82
  83         mm = atomic ? task->active_mm : get_task_mm (task);
  84         if (mm) {
  85                 if (!atomic) {
  86                         if (!down_write_trylock(&mm->mmap_sem)) {
  87                                 rcu_read_lock();
  88
  89                                 up_read(&mm->mmap_sem);
  90                                 down_write(&mm->mmap_sem);
  91
  92                                 rcu_read_unlock();
  93                         }
  94                 }
  95                 // FIXME: its seems to be bad decision to replace 'current' pointer temporarily
  96                 current_thread_info()->task = task;
  97                 ret = do_mmap_pgoff(NULL, 0, len, prot, flags, 0);
  98                 current_thread_info()->task = otask;
  99                 if (!atomic) {
 100                         downgrade_write (&mm->mmap_sem);
 101                         mmput(mm);
 102                 }
 103         } else {
 104                 printk("proc %d has no mm", task->tgid);
 105         }
 106
 107         return ret;
 108 }
 109
 110 static void *sm_alloc_us(struct slot_manager *sm)
 111 {
 112         struct task_struct *task = sm->data;
 113
 114         return (void *)alloc_user_pages(task, PAGE_SIZE,
 115                                         PROT_EXEC|PROT_READ|PROT_WRITE,
 116                                         MAP_ANONYMOUS|MAP_PRIVATE);
 117 }
 118
 119 static void sm_free_us(struct slot_manager *sm, void *ptr)
 120 {
 121         struct task_struct *task = sm->data;
 122
 123         /*
 124          * E. G.: This code provides kernel dump because of rescheduling while atomic.
 125          * As workaround, this code was commented. In this case we will have memory leaks
 126          * for instrumented process, but instrumentation process should functionate correctly.
 127          * Planned that good solution for this problem will be done during redesigning KProbe
 128          * for improving supportability and performance.
 129          */
 130 #if 0
 131         mm = get_task_mm(task);
 132         if (mm) {
 133                 down_write(&mm->mmap_sem);
 134                 do_munmap(mm, (unsigned long)(ptr), PAGE_SIZE);
 135                 up_write(&mm->mmap_sem);
 136                 mmput(mm);
 137         }
 138 #endif
 139         /* FIXME: implement the removal of memory for task */
 140 }
 141
 142 struct slot_manager *create_sm_us(struct task_struct *task)
 143 {
 144         struct slot_manager *sm = kmalloc(sizeof(*sm), GFP_ATOMIC);
 145         sm->slot_size = UPROBES_TRAMP_LEN;
 146         sm->alloc = sm_alloc_us;
 147         sm->free = sm_free_us;
 148         INIT_HLIST_NODE(&sm->page_list);
 149         sm->data = task;
 150 }
 151
 152 void free_sm_us(struct slot_manager *sm)
 153 {
 154         /* FIXME: free */
 155 }
 156
 157 struct pf_data {
 158         unsigned long addr;
 159 };
 160
 161 static int entry_handler_pf(struct kretprobe_instance *ri, struct pt_regs *regs);
 162 static int ret_handler_pf(struct kretprobe_instance *ri, struct pt_regs *regs);
 163
 164 static struct kretprobe pf_kretprobe = {
 165         .entry_handler = entry_handler_pf,
 166         .handler = ret_handler_pf,
 167         .data_size = sizeof(struct pf_data)
 168 };
 169
 170 static void copy_process_ret_pre_code(struct task_struct *p);
 171
 172 static int ret_handler_cp(struct kretprobe_instance *ri, struct pt_regs *regs)
 173 {
 174         struct task_struct* task = (struct task_struct *)regs_return_value(regs);
 175
 176         copy_process_ret_pre_code(task);
 177
 178         return 0;
 179 }
 180
 181 static struct kretprobe cp_kretprobe = {
 182         .handler = ret_handler_cp,
 183 };
 184
 185 static void jmm_release(struct task_struct *tsk, struct mm_struct *mm);
 186
 187 static struct jprobe mr_jprobe = {
 188         .entry = jmm_release
 189 };
 190
 191 static int jdo_munmap(struct mm_struct *mm, unsigned long start, size_t len);
 192
 193 static struct jprobe unmap_jprobe = {
 194         .entry = jdo_munmap
 195 };
 196
 197 static struct sspt_procs *get_proc_probes_by_task(struct task_struct *task)
 198 {
 199         struct sspt_procs *procs, *tmp;
 200
 201         if (!is_libonly()) {
 202                 if (task != current) {
 203                         printk("ERROR get_proc_probes_by_task: \'task != current\'\n");
 204                         return NULL;
 205                 }
 206
 207                 return us_proc_info.pp;
 208         }
 209
 210         list_for_each_entry_safe(procs, tmp, &proc_probes_list, list) {
 211                 if (procs->tgid == task->tgid) {
 212                         return procs;
 213                 }
 214         }
 215
 216         return NULL;
 217 }
 218
 219 static void add_proc_probes(struct task_struct *task, struct sspt_procs *procs)
 220 {
 221         list_add_tail(&procs->list, &proc_probes_list);
 222 }
 223
 224 static struct sspt_procs *get_proc_probes_by_task_or_new(struct task_struct *task)
 225 {
 226         struct sspt_procs *procs = get_proc_probes_by_task(task);
 227         if (procs == NULL) {
 228                 procs = sspt_procs_copy(us_proc_info.pp, task);
 229                 procs->sm = create_sm_us(task);
 230                 add_proc_probes(task, procs);
 231         }
 232
 233         return procs;
 234 }
 235
 236 struct dentry *dentry_by_path(const char *path)
 237 {
 238         struct dentry *dentry;
 239 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 240         struct path st_path;
 241         if (kern_path(path, LOOKUP_FOLLOW, &st_path) != 0) {
 242 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 243         struct nameidata nd;
 244         if (path_lookup(path, LOOKUP_FOLLOW, &nd) != 0) {
 245 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 246                 EPRINTF("failed to lookup dentry for path %s!", path);
 247                 return NULL;
 248         }
 249
 250 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)
 251         dentry = nd.dentry;
 252         path_release(&nd);
 253 #elif LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 38)
 254         dentry = nd.path.dentry;
 255         path_put(&nd.path);
 256 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 257         dentry = st_path.dentry;
 258         path_put(&st_path);
 259 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25) */
 260         return dentry;
 261 }
 262
 263 static int check_vma(struct vm_area_struct *vma)
 264 {
 265         return vma->vm_file && !(vma->vm_pgoff != 0 || !(vma->vm_flags & VM_EXEC) || (vma->vm_flags & VM_ACCOUNT) ||
 266                         !(vma->vm_flags & (VM_WRITE | VM_MAYWRITE)) ||
 267                         !(vma->vm_flags & (VM_READ | VM_MAYREAD)));
 268 }
 269
 270 static int find_task_by_path (const char *path, struct task_struct **p_task, struct list_head *tids)
 271 {
 272         int found = 0;
 273         struct task_struct *task;
 274         struct vm_area_struct *vma;
 275         struct mm_struct *mm;
 276         struct dentry *dentry = dentry_by_path(path);
 277
 278         *p_task = NULL;
 279
 280         /* find corresponding dir entry, this is also check for valid path */
 281         // TODO: test - try to instrument process with non-existing path
 282         // TODO: test - try to instrument process  with existing path and delete file just after start
 283         if (dentry == NULL) {
 284                 return -EINVAL;
 285         }
 286
 287         rcu_read_lock();
 288         for_each_process (task) {
 289
 290                 if  ( 0 != inst_pid && ( inst_pid != task->pid ) )
 291                         continue;
 292
 293                 mm = get_task_mm(task);
 294                 if (!mm)
 295                         continue;
 296                 vma = mm->mmap;
 297                 while (vma) {
 298                         if (check_vma(vma)) {
 299                                 if (vma->vm_file->f_dentry == dentry) {
 300                                         if (!*p_task) {
 301                                                 *p_task = task;
 302                                                 get_task_struct (task);
 303                                         }
 304                                                 //break;
 305                                 }
 306                         }
 307                         vma = vma->vm_next;
 308                 }
 309                 // only decrement usage count on mm since we cannot sleep here
 310                 atomic_dec(&mm->mm_users);
 311                 if (found)
 312                         break;
 313         }
 314         rcu_read_unlock();
 315
 316         if (*p_task) {
 317                 DPRINTF ("found pid %d for %s.", (*p_task)->pid, path);
 318                 *p_task = (*p_task)->group_leader;
 319                 gl_nNotifyTgid = (*p_task)->tgid;
 320         } else {
 321                 DPRINTF ("pid for %s not found!", path);
 322         }
 323
 324         return 0;
 325 }
 326
 327 static void set_mapping_file(struct sspt_file *file,
 328                 const struct sspt_procs *procs,
 329                 const struct task_struct *task,
 330                 const struct vm_area_struct *vma);
 331
 332 int install_otg_ip(unsigned long addr,
 333                         kprobe_pre_entry_handler_t pre_handler,
 334                         unsigned long jp_handler,
 335                         uretprobe_handler_t rp_handler)
 336 {
 337         int ret = 0;
 338         struct task_struct *task = current->group_leader;
 339         struct mm_struct *mm = task->mm;
 340
 341         if (mm) {
 342                 struct vm_area_struct *vma = find_vma(mm, addr);
 343                 if (vma && (vma->vm_flags & VM_EXEC) &&
 344                     vma->vm_file && vma->vm_file->f_dentry) {
 345                         unsigned long offset_addr = addr - vma->vm_start;
 346                         struct dentry *dentry = vma->vm_file->f_dentry;
 347                         char *name = dentry->d_iname;
 348                         struct sspt_procs *procs = get_proc_probes_by_task(task);
 349                         struct ip_data pd = {
 350                                         .offset = offset_addr,
 351                                         .pre_handler = pre_handler,
 352                                         .jp_handler = jp_handler,
 353                                         .rp_handler = rp_handler,
 354                                         .flag_retprobe = 1
 355                         };
 356
 357                         struct sspt_file *file = sspt_procs_find_file_or_new(procs, dentry, name);
 358                         struct sspt_page *page = sspt_get_page(file, offset_addr);
 359                         struct us_ip *ip = sspt_find_ip(page, offset_addr & ~PAGE_MASK);
 360
 361                         if (!file->loaded) {
 362                                 set_mapping_file(file, procs, task, vma);
 363                                 file->loaded = 1;
 364                         }
 365
 366                         if (ip == NULL) {
 367                                 // TODO: sspt_procs_find_file_or_new --> sspt_procs_find_file ?!
 368                                 struct sspt_file *file = sspt_procs_find_file_or_new(procs, dentry, name);
 369                                 sspt_file_add_ip(file, &pd);
 370
 371                                 /* if addr mapping, that probe install, else it be installed in do_page_fault handler */
 372                                 if (page_present(mm, addr)) {
 373                                         ip = sspt_find_ip(page, offset_addr & ~PAGE_MASK);
 374                                         sspt_set_ip_addr(ip, page, file);
 375
 376                                         // TODO: error
 377                                         ret = register_usprobe_my(task, ip);
 378                                         if (ret == 0) {
 379                                                 sspt_page_installed(page);
 380                                         } else {
 381                                                 printk("ERROR install_otg_ip: ret=%d\n", ret);
 382                                         }
 383                                 }
 384                         }
 385
 386                         sspt_put_page(page);
 387                 }
 388         }
 389
 390         return ret;
 391 }
 392 EXPORT_SYMBOL_GPL(install_otg_ip);
 393
 394 int init_helper(void)
 395 {
 396         unsigned long addr;
 397         addr = swap_ksyms("do_page_fault");
 398         if (addr == 0) {
 399                 EPRINTF("Cannot find address for page fault function!");
 400                 return -EINVAL;
 401         }
 402         pf_kretprobe.kp.addr = (kprobe_opcode_t *)addr;
 403
 404         addr = swap_ksyms("copy_process");
 405         if (addr == 0) {
 406                 EPRINTF("Cannot find address for copy_process function!");
 407                 return -EINVAL;
 408         }
 409         cp_kretprobe.kp.addr = (kprobe_opcode_t *)addr;
 410
 411         addr = swap_ksyms("mm_release");
 412         if (addr == 0) {
 413                 EPRINTF("Cannot find address for mm_release function!");
 414                 return -EINVAL;
 415         }
 416         mr_jprobe.kp.addr = (kprobe_opcode_t *)addr;
 417
 418         addr = swap_ksyms("do_munmap");
 419         if (addr == 0) {
 420                 EPRINTF("Cannot find address for do_munmap function!");
 421                 return -EINVAL;
 422         }
 423         unmap_jprobe.kp.addr = (kprobe_opcode_t *)addr;
 424
 425         return 0;
 426 }
 427
 428 void uninit_helper(void)
 429 {
 430 }
 431
 432 static int register_helper_ks_probes(void)
 433 {
 434         int ret = 0;
 435
 436         /* install jprobe on 'do_munmap' to detect when for remove user space probes */
 437         ret = dbi_register_jprobe(&unmap_jprobe);
 438         if (ret) {
 439                 EPRINTF("dbi_register_jprobe(do_munmap) result=%d!", ret);
 440                 return ret;
 441         }
 442
 443         /* install jprobe on 'mm_release' to detect when for remove user space probes */
 444         ret = dbi_register_jprobe(&mr_jprobe);
 445         if (ret != 0) {
 446                 EPRINTF("dbi_register_jprobe(mm_release) result=%d!", ret);
 447                 goto unregister_unmap;
 448         }
 449
 450
 451         /* install kretprobe on 'copy_process' */
 452         ret = dbi_register_kretprobe(&cp_kretprobe);
 453         if (ret) {
 454                 EPRINTF("dbi_register_kretprobe(copy_process) result=%d!", ret);
 455                 goto unregister_mr;
 456         }
 457
 458         /* install kretprobe on 'do_page_fault' to detect when they will be loaded */
 459         ret = dbi_register_kretprobe(&pf_kretprobe);
 460         if (ret) {
 461                 EPRINTF("dbi_register_kretprobe(do_page_fault) result=%d!", ret);
 462                 goto unregister_cp;
 463         }
 464
 465         return ret;
 466
 467 unregister_cp:
 468         dbi_unregister_kretprobe(&cp_kretprobe);
 469
 470 unregister_mr:
 471         dbi_unregister_jprobe(&mr_jprobe);
 472
 473 unregister_unmap:
 474         dbi_unregister_jprobe(&unmap_jprobe);
 475
 476         return ret;
 477 }
 478
 479 static void unregister_helper_ks_probes(void)
 480 {
 481         /* uninstall kretprobe with 'do_page_fault' */
 482         dbi_unregister_kretprobe(&pf_kretprobe);
 483
 484         /* uninstall kretprobe with 'copy_process' */
 485         dbi_unregister_kretprobe(&cp_kretprobe);
 486
 487         /* uninstall jprobe with 'mm_release' */
 488         dbi_unregister_jprobe(&mr_jprobe);
 489
 490         /* uninstall jprobe with 'do_munmap' */
 491         dbi_unregister_jprobe(&unmap_jprobe);
 492 }
 493
 494 static int uninstall_us_proc_probes(struct task_struct *task, struct sspt_procs *procs, enum US_FLAGS flag);
 495
 496 int deinst_usr_space_proc (void)
 497 {
 498         int iRet = 0, found = 0;
 499         struct task_struct *task = NULL;
 500
 501         if (!is_us_instrumentation()) {
 502                 return 0;
 503         }
 504
 505         if (iRet)
 506                 EPRINTF ("uninstall_kernel_probe(do_munmap) result=%d!", iRet);
 507
 508         if (is_libonly()) {
 509                 struct sspt_procs *procs;
 510
 511                 for_each_process(task)  {
 512                         procs = get_proc_probes_by_task(task);
 513                         if (procs) {
 514                                 int ret = uninstall_us_proc_probes(task, procs, US_UNREGS_PROBE);
 515                                 if (ret) {
 516                                         EPRINTF ("failed to uninstall IPs (%d)!", ret);
 517                                 }
 518
 519                                 dbi_unregister_all_uprobes(task, 1);
 520                         }
 521                 }
 522         }
 523         else
 524         {
 525                 if (us_proc_info.tgid == 0)
 526                         return 0;
 527                         rcu_read_lock ();
 528                 for_each_process (task)
 529                 {
 530                         if (task->tgid == us_proc_info.tgid)
 531                         {
 532                                 found = 1;
 533                                 get_task_struct (task);
 534                                 break;
 535                         }
 536                 }
 537                 rcu_read_unlock ();
 538                 if (found)
 539                 {
 540                         int i, ret;
 541                         // uninstall IPs
 542                         ret = uninstall_us_proc_probes(task, us_proc_info.pp, US_UNREGS_PROBE);
 543                         if (ret != 0) {
 544                                 EPRINTF ("failed to uninstall IPs %d!", ret);
 545                         }
 546
 547                         put_task_struct (task);
 548
 549                         printk("### 1 ### dbi_unregister_all_uprobes:\n");
 550                         dbi_unregister_all_uprobes(task, 1);
 551                         us_proc_info.tgid = 0;
 552                         for(i = 0; i < us_proc_info.libs_count; i++)
 553                                 us_proc_info.p_libs[i].loaded = 0;
 554                 }
 555         }
 556
 557         unregister_helper_ks_probes();
 558
 559         return iRet;
 560 }
 561
 562 static void install_proc_probes(struct task_struct *task, struct sspt_procs *procs, int atomic);
 563
 564 int inst_usr_space_proc (void)
 565 {
 566         int ret, i;
 567         struct task_struct *task = NULL;
 568
 569         if (!is_us_instrumentation()) {
 570                 return 0;
 571         }
 572
 573         DPRINTF("User space instr");
 574
 575         ret = register_helper_ks_probes();
 576         if (ret) {
 577                 return ret;
 578         }
 579
 580         for (i = 0; i < us_proc_info.libs_count; i++) {
 581                 us_proc_info.p_libs[i].loaded = 0;
 582         }
 583         /* check whether process is already running
 584          * 1) if process is running - look for the libraries in the process maps
 585          * 1.1) check if page for symbol does exist
 586          * 1.1.1) if page exists - instrument it
 587          * 1.1.2) if page does not exist - make sure that do_page_fault handler is installed
 588          * 2) if process is not running - make sure that do_page_fault handler is installed
 589          * */
 590
 591         if (is_libonly())
 592         {
 593                 // FIXME: clear_task_inst_info();
 594                 for_each_process (task) {
 595                         struct sspt_procs *procs;
 596
 597                         if (task->flags & PF_KTHREAD){
 598                                 DPRINTF("ignored kernel thread %d\n",
 599                                         task->pid);
 600                                 continue;
 601                         }
 602
 603                         procs = get_proc_probes_by_task_or_new(task);
 604                         DPRINTF("trying process");
 605                         install_proc_probes(task, procs, 1);
 606                         //put_task_struct (task);
 607                 }
 608         }
 609         else
 610         {
 611                 ret = find_task_by_path (us_proc_info.path, &task, NULL);
 612                 if ( task  )
 613                 {
 614                         DPRINTF("task found. installing probes");
 615                         us_proc_info.tgid = task->pid;
 616                         us_proc_info.pp->sm = create_sm_us(task);
 617                         install_proc_probes(task, us_proc_info.pp, 0);
 618                         put_task_struct (task);
 619                 }
 620         }
 621
 622         return 0;
 623 }
 624
 625 unsigned long imi_sum_time = 0;
 626 unsigned long imi_sum_hit = 0;
 627
 628 static void set_mapping_file(struct sspt_file *file,
 629                 const struct sspt_procs *procs,
 630                 const struct task_struct *task,
 631                 const struct vm_area_struct *vma)
 632 {
 633         int app_flag = (vma->vm_file->f_dentry == procs->dentry);
 634
 635         file->vm_start = vma->vm_start;
 636         file->vm_end = vma->vm_end;
 637
 638         pack_event_info(DYN_LIB_PROBE_ID, RECORD_ENTRY, "dspdd",
 639                         task->tgid, file->name, vma->vm_start,
 640                         vma->vm_end - vma->vm_start, app_flag);
 641 }
 642
 643 void print_vma(struct mm_struct *mm);
 644
 645 static int register_us_page_probe(struct sspt_page *page,
 646                 const struct sspt_file *file,
 647                 struct task_struct *task)
 648 {
 649         int err = 0;
 650         struct us_ip *ip, *n;
 651
 652         spin_lock(&page->lock);
 653
 654         if (sspt_page_is_install(page)) {
 655                 printk("page %lx in %s task[tgid=%u, pid=%u] already installed\n",
 656                                 page->offset, file->dentry->d_iname, task->tgid, task->pid);
 657                 print_vma(task->mm);
 658                 goto unlock;
 659         }
 660
 661         sspt_page_assert_install(page);
 662         sspt_set_all_ip_addr(page, file);
 663
 664         list_for_each_entry_safe(ip, n, &page->ip_list, list) {
 665                 err = register_usprobe_my(task, ip);
 666                 if (err == -ENOEXEC) {
 667                         list_del(&ip->list);
 668                         free_ip(ip);
 669                         continue;
 670                 } else if (err) {
 671                         EPRINTF("Failed to install probe");
 672                 }
 673         }
 674 unlock:
 675         sspt_page_installed(page);
 676         spin_unlock(&page->lock);
 677
 678         return 0;
 679 }
 680
 681 static int unregister_us_page_probe(struct task_struct *task,
 682                 struct sspt_page *page, enum US_FLAGS flag)
 683 {
 684         int err = 0;
 685         struct us_ip *ip;
 686
 687         spin_lock(&page->lock);
 688         if (!sspt_page_is_install(page)) {
 689                 spin_unlock(&page->lock);
 690                 return 0;
 691         }
 692
 693         list_for_each_entry(ip, &page->ip_list, list) {
 694                 err = unregister_usprobe_my(task, ip, flag);
 695                 if (err != 0) {
 696                         //TODO: ERROR
 697                         break;
 698                 }
 699         }
 700
 701         if (flag != US_DISARM) {
 702                 sspt_page_uninstalled(page);
 703         }
 704         spin_unlock(&page->lock);
 705
 706         return err;
 707 }
 708
 709 static void install_page_probes(unsigned long page_addr, struct task_struct *task, struct sspt_procs *procs, int atomic)
 710 {
 711         int lock;
 712         struct mm_struct *mm;
 713         struct vm_area_struct *vma;
 714
 715         mm_read_lock(task, mm, atomic, lock);
 716
 717         vma = find_vma(mm, page_addr);
 718         if (vma && check_vma(vma)) {
 719                 struct dentry *dentry = vma->vm_file->f_dentry;
 720                 struct sspt_file *file = sspt_procs_find_file(procs, dentry);
 721                 if (file) {
 722                         struct sspt_page *page;
 723                         if (!file->loaded) {
 724                                 set_mapping_file(file, procs, task, vma);
 725                                 file->loaded = 1;
 726                         }
 727
 728                         page = sspt_find_page_mapped(file, page_addr);
 729                         if (page) {
 730                                 register_us_page_probe(page, file, task);
 731                         }
 732                 }
 733         }
 734
 735         mm_read_unlock(mm, atomic, lock);
 736 }
 737
 738 static void install_file_probes(struct task_struct *task, struct mm_struct *mm, struct sspt_file *file)
 739 {
 740         struct sspt_page *page = NULL;
 741         struct hlist_node *node = NULL;
 742         struct hlist_head *head = NULL;
 743         int i, table_size = (1 << file->page_probes_hash_bits);
 744
 745         for (i = 0; i < table_size; ++i) {
 746                 head = &file->page_probes_table[i];
 747                 swap_hlist_for_each_entry_rcu(page, node, head, hlist) {
 748                         register_us_page_probe(page, file, task);
 749                 }
 750         }
 751 }
 752
 753 static void install_proc_probes(struct task_struct *task, struct sspt_procs *procs, int atomic)
 754 {
 755         int lock;
 756         struct vm_area_struct *vma;
 757         struct mm_struct *mm;
 758
 759         mm_read_lock(task, mm, atomic, lock);
 760
 761         for (vma = mm->mmap; vma; vma = vma->vm_next) {
 762                 if (check_vma(vma)) {
 763                         struct dentry *dentry = vma->vm_file->f_dentry;
 764                         struct sspt_file *file = sspt_procs_find_file(procs, dentry);
 765                         if (file) {
 766                                 if (!file->loaded) {
 767                                         set_mapping_file(file, procs, task, vma);
 768                                         file->loaded = 1;
 769                                 }
 770
 771                                 install_file_probes(task, mm, file);
 772                         }
 773                 }
 774         }
 775
 776         mm_read_unlock(mm, atomic, lock);
 777 }
 778
 779 static int check_install_pages_in_file(struct task_struct *task, struct sspt_file *file)
 780 {
 781         int i;
 782         int table_size = (1 << file->page_probes_hash_bits);
 783         struct sspt_page *page;
 784         struct hlist_node *node, *tmp;
 785         struct hlist_head *head;
 786
 787         for (i = 0; i < table_size; ++i) {
 788                 head = &file->page_probes_table[i];
 789                 swap_hlist_for_each_entry_safe (page, node, tmp, head, hlist) {
 790                         if (page->install) {
 791                                 return 1;
 792                         }
 793                 }
 794         }
 795
 796         return 0;
 797 }
 798
 799 static int unregister_us_file_probes(struct task_struct *task, struct sspt_file *file, enum US_FLAGS flag)
 800 {
 801         int i, err = 0;
 802         int table_size = (1 << file->page_probes_hash_bits);
 803         struct sspt_page *page;
 804         struct hlist_node *node, *tmp;
 805         struct hlist_head *head;
 806
 807         for (i = 0; i < table_size; ++i) {
 808                 head = &file->page_probes_table[i];
 809                 swap_hlist_for_each_entry_safe (page, node, tmp, head, hlist) {
 810                         err = unregister_us_page_probe(task, page, flag);
 811                         if (err != 0) {
 812                                 // TODO: ERROR
 813                                 return err;
 814                         }
 815                 }
 816         }
 817
 818         if (flag != US_DISARM) {
 819                 file->loaded = 0;
 820         }
 821
 822         return err;
 823 }
 824
 825 static int uninstall_us_proc_probes(struct task_struct *task, struct sspt_procs *procs, enum US_FLAGS flag)
 826 {
 827         int err = 0;
 828         struct sspt_file *file;
 829
 830         list_for_each_entry_rcu(file, &procs->file_list, list) {
 831                 err = unregister_us_file_probes(task, file, flag);
 832                 if (err != 0) {
 833                         // TODO:
 834                         return err;
 835                 }
 836         }
 837
 838         return err;
 839 }
 840
 841 static pid_t find_proc_by_task(const struct task_struct *task, struct dentry *dentry)
 842 {
 843         struct vm_area_struct *vma;
 844         struct mm_struct *mm = task->active_mm;
 845         if (mm == NULL) {
 846                 return 0;
 847         }
 848
 849         for (vma = mm->mmap; vma; vma = vma->vm_next) {
 850                 if (check_vma(vma)) {
 851                         if (vma->vm_file->f_dentry == dentry) {
 852                                 return task->tgid;
 853                         }
 854                 }
 855         }
 856
 857         return 0;
 858 }
 859
 860 static int entry_handler_pf(struct kretprobe_instance *ri, struct pt_regs *regs)
 861 {
 862         struct pf_data *data = (struct pf_data *)ri->data;
 863
 864         /* for ARM arch*/
 865         data->addr = regs->ARM_r0;
 866         return 0;
 867 }
 868
 869 /* Detects when IPs are really loaded into phy mem and installs probes. */
 870 static int ret_handler_pf(struct kretprobe_instance *ri, struct pt_regs *regs)
 871 {
 872         struct task_struct *task = current->group_leader;
 873         struct mm_struct *mm = task->mm;
 874         struct sspt_procs *procs = NULL;
 875         /*
 876          * Because process threads have same address space
 877          * we instrument only group_leader of all this threads
 878          */
 879         struct pf_data *data;
 880         unsigned long addr = 0;
 881         int valid_addr;
 882
 883         // overhead
 884         struct timeval imi_tv1;
 885         struct timeval imi_tv2;
 886 #define USEC_IN_SEC_NUM                         1000000
 887
 888         if (task->flags & PF_KTHREAD) {
 889                 goto out;
 890         }
 891
 892         if (!is_us_instrumentation()) {
 893                 goto out;
 894         }
 895
 896         data = (struct pf_data *)ri->data;
 897         addr = data->addr;
 898
 899         valid_addr = mm && page_present(mm, addr);
 900         if (!valid_addr) {
 901                 goto out;
 902         }
 903
 904         if (is_libonly()) {
 905                 procs = get_proc_probes_by_task_or_new(task);
 906         } else {
 907                 // find task
 908                 if (us_proc_info.tgid == 0) {
 909                         pid_t tgid = find_proc_by_task(task, us_proc_info.m_f_dentry);
 910                         if (tgid) {
 911                                 us_proc_info.tgid = gl_nNotifyTgid = tgid;
 912
 913                                 us_proc_info.pp->sm = create_sm_us(task);
 914                                 /* install probes in already mapped memory */
 915                                 install_proc_probes(task, us_proc_info.pp, 1);
 916                         }
 917                 }
 918
 919                 if (us_proc_info.tgid == task->tgid) {
 920                         procs = us_proc_info.pp;
 921                 }
 922         }
 923
 924         if (procs) {
 925                 unsigned long page = addr & PAGE_MASK;
 926
 927                 // overhead
 928                 do_gettimeofday(&imi_tv1);
 929                 install_page_probes(page, task, procs, 1);
 930                 do_gettimeofday(&imi_tv2);
 931                 imi_sum_hit++;
 932                 imi_sum_time += ((imi_tv2.tv_sec - imi_tv1.tv_sec) *  USEC_IN_SEC_NUM +
 933                                 (imi_tv2.tv_usec - imi_tv1.tv_usec));
 934         }
 935
 936 out:
 937         return 0;
 938 }
 939
 940 void print_vma(struct mm_struct *mm)
 941 {
 942         struct vm_area_struct *vma;
 943         printk("### print_vma: START\n");\
 944         printk("### print_vma: START\n");
 945
 946         for (vma = mm->mmap; vma; vma = vma->vm_next) {
 947                 char *x = vma->vm_flags & VM_EXEC ? "x" : "-";
 948                 char *r = vma->vm_flags & VM_READ ? "r" : "-";
 949                 char *w = vma->vm_flags & VM_WRITE ? "w" : "-";
 950                 char *name = vma->vm_file ? (char *)vma->vm_file->f_dentry->d_iname : "N/A";
 951
 952                 printk("### [%8lx..%8lx] %s%s%s pgoff=\'%8lu\' %s\n",
 953                                 vma->vm_start, vma->vm_end, x, r, w, vma->vm_pgoff, name);
 954         }
 955         printk("### print_vma:  END\n");
 956 }
 957
 958 static int remove_unmap_probes(struct task_struct *task, struct sspt_procs *procs, unsigned long start, size_t len)
 959 {
 960         struct mm_struct *mm = task->mm;
 961         struct vm_area_struct *vma;
 962
 963         if ((start & ~PAGE_MASK) || start > TASK_SIZE || len > TASK_SIZE - start) {
 964                 return -EINVAL;
 965         }
 966
 967         if ((len = PAGE_ALIGN(len)) == 0) {
 968                 return -EINVAL;
 969         }
 970
 971         vma = find_vma(mm, start);
 972         if (vma && check_vma(vma)) {
 973                 struct sspt_file *file;
 974                 unsigned long end = start + len;
 975                 struct dentry *dentry = vma->vm_file->f_dentry;
 976
 977                 file = sspt_procs_find_file(procs, dentry);
 978                 if (file) {
 979                         if (vma->vm_start == start || vma->vm_end == end) {
 980                                 unregister_us_file_probes(task, file, US_UNREGS_PROBE);
 981                                 file->loaded = 0;
 982                         } else {
 983                                 unsigned long page_addr;
 984                                 struct sspt_page *page;
 985
 986                                 for (page_addr = vma->vm_start; page_addr < vma->vm_end; page_addr += PAGE_SIZE) {
 987                                         page = sspt_find_page_mapped(file, page_addr);
 988                                         if (page) {
 989                                                 unregister_us_page_probe(task, page, US_UNREGS_PROBE);
 990                                         }
 991                                 }
 992
 993                                 if (check_install_pages_in_file(task, file)) {
 994                                         file->loaded = 0;
 995                                 }
 996                         }
 997                 }
 998         }
 999
1000         return 0;
1001 }
1002
1003 /* Detects when target removes IPs. */
1004 static int jdo_munmap(struct mm_struct *mm, unsigned long start, size_t len)
1005 {
1006         struct sspt_procs *procs = NULL;
1007         struct task_struct *task = current;
1008
1009         //if user-space instrumentation is not set
1010         if (!is_us_instrumentation()) {
1011                 goto out;
1012         }
1013
1014         if (is_libonly()) {
1015                 procs = get_proc_probes_by_task(task);
1016         } else {
1017                 if (task->tgid == us_proc_info.tgid) {
1018                         procs = us_proc_info.pp;
1019                 }
1020         }
1021
1022         if (procs) {
1023                 if (remove_unmap_probes(task, procs, start, len)) {
1024                         printk("ERROR do_munmap: start=%lx, len=%x\n", start, len);
1025                 }
1026         }
1027
1028 out:
1029         dbi_jprobe_return();
1030         return 0;
1031 }
1032
1033 /* Detects when target process removes IPs. */
1034 static void jmm_release(struct task_struct *task, struct mm_struct *mm)
1035 {
1036         struct sspt_procs *procs = NULL;
1037
1038         if (!is_us_instrumentation() || task->tgid != task->pid) {
1039                 goto out;
1040         }
1041
1042         if (is_libonly()) {
1043                 procs = get_proc_probes_by_task(task);
1044         } else {
1045                 if (task->tgid == us_proc_info.tgid) {
1046                         procs = get_proc_probes_by_task(task);
1047                         us_proc_info.tgid = 0;
1048                 }
1049         }
1050
1051         if (procs) {
1052                 int ret = uninstall_us_proc_probes(task, procs, US_UNREGS_PROBE);
1053                 if (ret != 0) {
1054                         EPRINTF ("failed to uninstall IPs (%d)!", ret);
1055                 }
1056
1057                 dbi_unregister_all_uprobes(task, 1);
1058         }
1059
1060 out:
1061         dbi_jprobe_return();
1062 }
1063
1064 static void recover_child(struct task_struct *child_task, struct sspt_procs *procs)
1065 {
1066         uninstall_us_proc_probes(child_task, procs, US_DISARM);
1067         dbi_disarm_urp_inst_for_task(current, child_task);
1068 }
1069
1070 static void rm_uprobes_child(struct task_struct *new_task)
1071 {
1072         if (is_libonly()) {
1073                 struct sspt_procs *procs = get_proc_probes_by_task(current);
1074                 if(procs) {
1075                         recover_child(new_task, procs);
1076                 }
1077         } else {
1078                 if(us_proc_info.tgid == current->tgid) {
1079                         recover_child(new_task, us_proc_info.pp);
1080                 }
1081         }
1082 }
1083
1084 /* Delete uprobs in children at fork */
1085 static void copy_process_ret_pre_code(struct task_struct *p)
1086 {
1087         if(!p || IS_ERR(p))
1088                 return;
1089
1090         if(p->mm != current->mm)    // check flags CLONE_VM
1091                 rm_uprobes_child(p);
1092 }
1093
1094 static DEFINE_PER_CPU(struct us_ip *, gpCurIp) = NULL;
1095 EXPORT_PER_CPU_SYMBOL_GPL(gpCurIp);
1096 static DEFINE_PER_CPU(struct pt_regs *, gpUserRegs) = NULL;
1097 EXPORT_PER_CPU_SYMBOL_GPL(gpUserRegs);
1098
1099 unsigned long ujprobe_event_pre_handler(struct us_ip *ip, struct pt_regs *regs)
1100 {
1101         __get_cpu_var (gpCurIp) = ip;
1102         __get_cpu_var (gpUserRegs) = regs;
1103         return 0;
1104 }
1105
1106 void ujprobe_event_handler (unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5, unsigned long arg6)
1107 {
1108         struct us_ip *ip = __get_cpu_var(gpCurIp);
1109         unsigned long addr = (unsigned long)ip->jprobe.up.kp.addr;
1110
1111 #if defined(CONFIG_ARM)
1112         if (ip->offset & 0x01)
1113         {
1114                 pack_event_info (US_PROBE_ID, RECORD_ENTRY, "ppppppp", addr | 0x01, arg1, arg2, arg3, arg4, arg5, arg6);
1115         }else{
1116                 pack_event_info (US_PROBE_ID, RECORD_ENTRY, "ppppppp", addr, arg1, arg2, arg3, arg4, arg5, arg6);
1117         }
1118 #else
1119         pack_event_info (US_PROBE_ID, RECORD_ENTRY, "ppppppp", addr, arg1, arg2, arg3, arg4, arg5, arg6);
1120 #endif
1121         // Mr_Nobody: uncomment for valencia
1122         //unregister_usprobe(current, ip, 1);
1123         dbi_uprobe_return ();
1124 }
1125
1126 static void send_plt(struct us_ip *ip)
1127 {
1128         unsigned long addr = (unsigned long)ip->jprobe.up.kp.addr;
1129         struct vm_area_struct *vma = find_vma(current->mm, addr);
1130
1131         if (vma && check_vma(vma)) {
1132                 char *name = NULL;
1133                 unsigned long real_addr;
1134                 unsigned long real_got = current->mm->exe_file == vma->vm_file ?
1135                                 ip->got_addr :
1136                                 ip->got_addr + vma->vm_start;
1137
1138                 if (!read_proc_vm_atomic(current, real_got, &real_addr, sizeof(real_addr))) {
1139                         printk("Failed to read got %lx at memory address %lx!\n", ip->got_addr, real_got);
1140                         return;
1141                 }
1142
1143                 vma = find_vma(current->mm, real_addr);
1144                 if (vma && (vma->vm_start <= real_addr) && (vma->vm_end > real_addr)) {
1145                         name = vma->vm_file ? vma->vm_file->f_dentry->d_iname : NULL;
1146                 } else {
1147                         printk("Failed to get vma, includes %lx address\n", real_addr);
1148                         return;
1149                 }
1150
1151                 if (name) {
1152                         pack_event_info(PLT_ADDR_PROBE_ID, RECORD_RET, "ppsp", addr, real_addr, name, real_addr - vma->vm_start);
1153                 } else {
1154                         pack_event_info(PLT_ADDR_PROBE_ID, RECORD_RET, "ppp", addr, real_addr, real_addr - vma->vm_start);
1155                 }
1156         }
1157 }
1158
1159 int uretprobe_event_handler(struct uretprobe_instance *probe, struct pt_regs *regs, struct us_ip *ip)
1160 {
1161         int retval = regs_return_value(regs);
1162         unsigned long addr = (unsigned long)ip->jprobe.up.kp.addr;
1163
1164         if (ip->got_addr && ip->flag_got == 0) {
1165                 send_plt(ip);
1166                 ip->flag_got = 1;
1167         }
1168
1169 #if defined(CONFIG_ARM)
1170         if (ip->offset & 0x01)
1171         {
1172                 pack_event_info (US_PROBE_ID, RECORD_RET, "pd", addr | 0x01, retval);
1173         }else{
1174                 pack_event_info (US_PROBE_ID, RECORD_RET, "pd", addr, retval);
1175         }
1176 #else
1177         pack_event_info (US_PROBE_ID, RECORD_RET, "pd", addr, retval);
1178 #endif
1179         // Mr_Nobody: uncomment for valencia
1180         //unregister_usprobe(current, ip, 1);
1181         return 0;
1182 }
1183
1184 int register_usprobe(struct task_struct *task, struct us_ip *ip, int atomic)
1185 {
1186         int ret = 0;
1187
1188         if (ip->jprobe.entry == NULL) {
1189                 ip->jprobe.entry = (void *)ujprobe_event_handler;
1190                 DPRINTF("Set default event handler for %x\n", ip->offset);
1191         }
1192
1193         if (ip->jprobe.pre_entry == NULL) {
1194                 ip->jprobe.pre_entry = (uprobe_pre_entry_handler_t)ujprobe_event_pre_handler;
1195                 DPRINTF("Set default pre handler for %x\n", ip->offset);
1196         }
1197
1198         ip->jprobe.priv_arg = ip;
1199         ip->jprobe.up.task = task;
1200         ip->jprobe.up.sm = ip->page->file->procs->sm;
1201         ret = dbi_register_ujprobe(&ip->jprobe, atomic);
1202         if (ret) {
1203                 if (ret == -ENOEXEC) {
1204                         pack_event_info(ERR_MSG_ID, RECORD_ENTRY, "dp",
1205                                         0x1,
1206                                         ip->jprobe.up.kp.addr);
1207                 }
1208                 DPRINTF ("dbi_register_ujprobe() failure %d", ret);
1209                 return ret;
1210         }
1211
1212         if (ip->flag_retprobe) {
1213                 // Mr_Nobody: comment for valencia
1214                 if (ip->retprobe.handler == NULL) {
1215                         ip->retprobe.handler = (uretprobe_handler_t)uretprobe_event_handler;
1216                         DPRINTF("Set default ret event handler for %x\n", ip->offset);
1217                 }
1218
1219                 ip->retprobe.priv_arg = ip;
1220                 ip->retprobe.up.task = task;
1221                 ip->retprobe.up.sm = ip->page->file->procs->sm;
1222                 ret = dbi_register_uretprobe(&ip->retprobe, atomic);
1223                 if (ret) {
1224                         EPRINTF ("dbi_register_uretprobe() failure %d", ret);
1225                         return ret;
1226                 }
1227         }
1228
1229         return 0;
1230 }
1231
1232 int unregister_usprobe(struct task_struct *task, struct us_ip *ip, int atomic)
1233 {
1234         dbi_unregister_ujprobe(&ip->jprobe, atomic);
1235
1236         if (ip->flag_retprobe) {
1237                 dbi_unregister_uretprobe(&ip->retprobe, atomic);
1238         }
1239
1240         return 0;
1241 }