--- /dev/null
+include $(project_pwd)/common.mk
+
+program_name := $(driver_name)
+source_dirs := $(project_pwd)/$(src_path)/$(driver_path)
+include_dirs := $(project_pwd)/$(src_path)/$(common_path) \
+ $(project_pwd)/$(src_path)/$(probes_path)
+include_libs :=
+special_defines := EC_ARCH_$(target_cpu)
+ifeq ($(specific_target), chelsea)
+ special_defines += KERNEL_HAS_ISPAGEPRESENT
+endif
+special_compiling_flags :=
+special_linking_flags :=
+ifeq ($(memory_checker), yes)
+special_defines += MEMORY_CHECKER
+endif
+ifeq ($(thread_profiler), yes)
+special_defines += THREAD_PROFILER
+endif
+
+include $(project_pwd)/rules-m.mk
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: debug.h
+//
+// DESCRIPTION:
+// Debug functions for application
+//
+// SEE ALSO: N/A
+// AUTHOR: L.Komkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.02
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#if !defined(__DEBUG_H__)
+#define __DEBUG_H__
+
+#include <linux/kernel.h>
+#include <linux/string.h> // strrchr
+
+#ifdef __DEBUG
+#define DPRINTF(format, args...) do { \
+ char *f = __FILE__; \
+ char *n = strrchr(f, '/'); \
+ printk("DRIVER[%s:%u:%s] DEBUG: " format "\n" , (n) ? n+1 : f, __LINE__, __FUNCTION__, ##args); \
+ } while(0)
+#else
+#define DPRINTF(format, args...)
+#endif
+
+#define EPRINTF(format, args...) do { \
+ char *f = __FILE__; \
+ char *n = strrchr(f, '/'); \
+ printk("DRIVER[%s:%u:%s] ERROR: " format "\n" , (n) ? n+1 : f, __LINE__, __FUNCTION__, ##args); \
+ } while(0)
+
+
+#endif /* !defined(__DEBUG_H__) */
--- /dev/null
+#ifndef _TNCHK_DEF_HANDLERS_H_
+#define _TNCHK_DEF_HANDLERS_H_
+
+/*void
+ujprobe_event_pre_handler (us_proc_ip_t *, struct pt_regs *);
+
+void
+ujprobe_event_handler (unsigned long, unsigned long, unsigned long,
+ unsigned long, unsigned long, unsigned long);
+
+int
+uretprobe_event_handler (struct kretprobe_instance *, struct pt_regs *, us_proc_ip_t *);*/
+
+#endif /* _TNCHK_DEF_HANDLERS_H_ */
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: device_driver.c
+//
+// DESCRIPTION:
+// This file is C source for SWAP driver.
+//
+// SEE ALSO: device_driver.h
+// AUTHOR: L.Komkov, S.Dianov, S.Grekhov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#include "module.h"
+#include "device_driver.h" // device driver
+#include "CProfile.h"
+
+DECLARE_WAIT_QUEUE_HEAD (notification_waiters_queue);
+volatile unsigned notification_count;
+
+static int device_mmap (struct file *filp, struct vm_area_struct *vma);
+static int device_ioctl (struct inode *inode, struct file *file, unsigned int cmd, unsigned long arg);
+static int device_open(struct inode *, struct file *);
+static int device_release(struct inode *, struct file *);
+static ssize_t device_read(struct file *, char *, size_t, loff_t *);
+static ssize_t device_write(struct file *, const char *, size_t, loff_t *);
+
+static int gl_nDeviceOpened = 0;
+static struct file_operations device_fops = {
+ .owner = THIS_MODULE,
+ .mmap = device_mmap,
+ .ioctl = device_ioctl,
+ .read = device_read,
+ .write = device_write,
+ .open = device_open,
+ .release = device_release
+};
+
+int device_init (void)
+{
+ int nReserved = 0;
+ int nRetVal = register_chrdev(device_major, device_name, &device_fops);
+ if (nRetVal < 0) {
+ EPRINTF("Cannot register character device! [%s, %d]", device_name, device_major);
+ nReserved = register_chrdev(0, device_name, &device_fops);
+ if(nReserved >= 0)
+ EPRINTF("Please, create a new device node with major number [%d],\n\tand pass it as module parameter!", nReserved);
+ return -1;
+ } else if(nRetVal > 0) {
+ EPRINTF("Cannot register this device major number! [%d]\n\tTrying a new one. [%d]", device_major, nRetVal);
+ device_major = nRetVal;
+ }
+ return 0;
+}
+
+void device_down (void)
+{
+ unregister_chrdev(device_major, device_name);
+}
+
+void notify_user (event_id_t event_id)
+{
+ ec_info.events_counters[event_id] += 1;
+
+ if (EVENT_EC_PROBE_RECORD == event_id)
+ {
+ // EC_PROBE_RECORD events happen to often. To reduce overhead user
+ // space will be notified only once per each EVENTS_AGGREGATION_USEC
+ static uint64_t timestamp_usec = 0;
+
+ uint64_t current_usec;
+ uint64_t delta_usec;
+
+ struct timeval tv;
+
+ do_gettimeofday (&tv);
+ current_usec = 1000000ULL * (unsigned) tv.tv_sec + (unsigned) tv.tv_usec;
+
+ if (current_usec < timestamp_usec)
+ {
+ // Note: time from do_gettimeofday() may go backward
+ EPRINTF ("current_usec=%llu timestamp_usec=%llu", current_usec, timestamp_usec);
+ }
+ else
+ {
+ delta_usec = current_usec - timestamp_usec;
+ if (EVENTS_AGGREGATION_USEC > delta_usec)
+ {
+ // wait the time left
+#if defined(__DEBUG)
+ unsigned UNUSED left_usec = EVENTS_AGGREGATION_USEC - delta_usec;
+#endif /* defined(__DEBUG) */
+ return; // supress notification
+ }
+ }
+ timestamp_usec = current_usec; // remember new time for the future use
+ } else if (EVENT_EC_START_CONDITION_SEEN == event_id) {
+ return; // supress notification
+ } else if (EVENT_EC_STOP_CONDITION_SEEN == event_id) {
+ return; // supress notification
+ }
+
+ ++notification_count;
+ wake_up_interruptible (¬ification_waiters_queue);
+}
+
+static int device_mmap (struct file *filp UNUSED, struct vm_area_struct *vma)
+{
+ if(!p_buffer) {
+ EPRINTF("Null pointer to buffer!");
+ return -1;
+ }
+ return remap_vmalloc_range (vma, p_buffer, 0);
+}
+
+static int device_open(struct inode *inode, struct file *file)
+{
+ /*if (gl_nDeviceOpened)
+ return -EBUSY;*/
+ gl_nDeviceOpened++;
+ // TODO
+ try_module_get(THIS_MODULE);
+ return 0;
+}
+
+static int device_release(struct inode *inode, struct file *file)
+{
+ gl_nDeviceOpened--;
+ module_put(THIS_MODULE);
+ return 0;
+}
+
+static ssize_t device_read(struct file *filp, char *buffer, size_t length, loff_t * offset)
+{
+ EPRINTF("Operation <<read>> not supported!");
+ return -1;
+}
+
+static ssize_t device_write(struct file *filp, const char *buff, size_t len, loff_t * off)
+{
+ EPRINTF("Operation <<write>> not supported!");
+ return -1;
+}
+static int device_ioctl (struct inode *inode UNUSED, struct file *file UNUSED, unsigned int cmd, unsigned long arg)
+{
+ unsigned long spinlock_flags = 0L;
+ int result = -1;
+// DPRINTF("Command=%d", cmd);
+ switch (cmd)
+ {
+ case EC_IOCTL_SET_EC_MODE:
+ {
+ ioctl_general_t param;
+ unsigned long nIgnoredBytes = 0;
+ memset(¶m, '0', sizeof(ioctl_general_t));
+ nIgnoredBytes = copy_from_user (¶m, (void*)arg, sizeof(ioctl_general_t));
+ if (nIgnoredBytes > 0) {
+ result = -1;
+ break;
+ }
+ if(SetECMode(param.m_unsignedLong) == -1) {
+ result = -1;
+ break;
+ }
+ result = 0;
+ DPRINTF("Set EC Mode = %lu", param.m_unsignedLong);
+ break;
+ }
+ case EC_IOCTL_GET_EC_MODE:
+ {
+ ioctl_general_t param;
+ unsigned long nIgnoredBytes = 0;
+ memset(¶m, '0', sizeof(ioctl_general_t));
+ param.m_unsignedLong = GetECMode();
+ nIgnoredBytes = copy_to_user ((void*)arg, ¶m, sizeof (ioctl_general_t));
+ if (nIgnoredBytes > 0) {
+ result = -1;
+ break;
+ }
+ result = 0;
+// DPRINTF("Get EC Mode = %lu", param.m_unsignedLong); // Frequent call
+ break;
+ }
+ case EC_IOCTL_SET_BUFFER_SIZE:
+ {
+ ioctl_general_t param;
+ unsigned long nIgnoredBytes = 0;
+ memset(¶m, '0', sizeof(ioctl_general_t));
+ nIgnoredBytes = copy_from_user (¶m, (void*)arg, sizeof(ioctl_general_t));
+ if (nIgnoredBytes > 0) {
+ result = -1;
+ break;
+ }
+ if (SetBufferSize(param.m_unsignedLong) == -1) {
+ result = -1;
+ break;
+ }
+ result = 0;
+ DPRINTF("Set Buffer Size = %lu", param.m_unsignedLong);
+ break;
+ }
+ case EC_IOCTL_GET_BUFFER_SIZE:
+ {
+ ioctl_general_t param;
+ unsigned long nIgnoredBytes = 0;
+ memset(¶m, '0', sizeof(ioctl_general_t));
+ param.m_unsignedLong = GetBufferSize();
+ nIgnoredBytes = copy_to_user ((void*)arg, ¶m, sizeof (ioctl_general_t));
+ if (nIgnoredBytes > 0) {
+ result = -1;
+ break;
+ }
+ result = 0;
+ DPRINTF("Get Buffer Size = %lu", param.m_unsignedLong);
+ break;
+ }
+ case EC_IOCTL_RESET_BUFFER:
+ {
+ if (ResetBuffer() == -1) {
+ result = -1;
+ break;
+ }
+ result = 0;
+ DPRINTF("Reset Buffer");
+ break;
+ }
+ case EC_IOCTL_GET_EC_INFO:
+ {
+ if (copy_ec_info_to_user_space ((ec_info_t *) arg) != 0) {
+ result = -1;
+ break;
+ }
+ result = 0;
+// DPRINTF("Get Buffer Status"); // Frequent call
+ break;
+ }
+ case EC_IOCTL_CONSUME_BUFFER:
+ {
+ static ec_info_t ec_info_copy;
+ int nIgnoredBytes = 0;
+#ifndef __DISABLE_RELAYFS
+ struct rchan* pRelayChannel = NULL;
+ struct rchan_buf *buf = NULL;
+ unsigned int nNumOfSubbufs = 0;
+ void* pConsume = NULL;
+ unsigned int nPaddingLength = 0;
+ unsigned int nSubbufSize = 0;
+ unsigned int nDataSize = 0;
+ unsigned int nEffectSize = 0;
+ unsigned int nSubbufDiscardedCount = 0;
+#endif
+ nIgnoredBytes = copy_from_user (&ec_info_copy, (ec_info_t *) arg, sizeof (ec_info_t));
+ if(nIgnoredBytes > 0)
+ {
+ EPRINTF ("copy_from_user(%08X,%08X)=%d", (unsigned) arg, (unsigned) &ec_info_copy, nIgnoredBytes);
+ result = -1;
+ break;
+ }
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ if((ec_info_copy.m_nMode & MODEMASK_MULTIPLE_BUFFER) == 0) {
+ // Original buffer
+ if(ec_info.after_last < ec_info.first) {
+ ec_info.buffer_effect = ec_info.buffer_size;
+ }
+ ec_info.first = ec_info_copy.after_last;
+ ec_info.trace_size = ec_info.trace_size - ec_info_copy.trace_size;
+ } else {
+ // Relay FS buffer
+#ifndef __DISABLE_RELAYFS
+ pRelayChannel = GetRelayChannel();
+ if(pRelayChannel == NULL) {
+ EPRINTF("Null pointer to relay channel!");
+ result = -1;
+ break;
+ }
+ buf = pRelayChannel->buf[0];
+ nNumOfSubbufs = pRelayChannel->n_subbufs;
+
+ nSubbufSize = pRelayChannel->subbuf_size;
+ pConsume = buf->start + buf->subbufs_consumed % nNumOfSubbufs * nSubbufSize;
+ memcpy(&nPaddingLength, pConsume, sizeof(unsigned int));
+ memcpy(&nSubbufDiscardedCount, pConsume + sizeof(unsigned int), sizeof(unsigned int));
+ nEffectSize = nSubbufSize - nPaddingLength;
+ nDataSize = nEffectSize - RELAY_SUBBUF_HEADER_SIZE;
+ relay_subbufs_consumed(pRelayChannel, 0, 1);
+ ec_info.m_nBeginSubbufNum = buf->subbufs_consumed % nNumOfSubbufs;
+ ec_info.m_nEndSubbufNum = buf->subbufs_produced % nNumOfSubbufs;
+ ec_info.buffer_effect -= nEffectSize;
+ ec_info.trace_size -= nDataSize;
+ buf->dentry->d_inode->i_size = ec_info.trace_size;
+#endif
+ }
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+ result = 0;
+// DPRINTF("Consume Buffer"); // Frequent call
+ break;
+ }
+ case EC_IOCTL_ADD_PROBE:
+ {
+ unsigned long addr = arg;
+ result = add_probe(addr);
+
+ break;
+ }
+ //@AGv: remove_probe expects probe address instead of name
+ /*case EC_IOCTL_REMOVE_PROBE:
+ {
+ char *probe_name = (char *) arg;
+ result = remove_probe (probe_name);
+
+ break;
+ }*/
+ case EC_IOCTL_RESET_PROBES:
+ {
+ result = reset_probes();
+
+ break;
+ }
+ case EC_IOCTL_UPDATE_CONDS:
+ {
+ int args_cnt, i;
+ struct cond *c, *c_tmp, *p_cond;
+ unsigned char *p_data;
+ int err;
+ result = 0;
+ err = copy_from_user(&args_cnt, (void *)arg, sizeof(int));
+ if (err) {
+ result = -1;
+ break;
+ }
+ /* first, delete all the conds */
+ list_for_each_entry_safe(c, c_tmp, &cond_list.list, list) {
+ list_del(&c->list);
+ kfree(c);
+ }
+ /* second, add new conds */
+ p_data = (unsigned char *)(arg + sizeof(int));
+ for (i = 0; i < args_cnt; i++) {
+ p_cond = kmalloc(sizeof(struct cond), GFP_KERNEL);
+ memcpy(&p_cond->tmpl, p_data, sizeof(struct event_tmpl));
+ p_cond->applied = 0;
+ list_add(&(p_cond->list), &(cond_list.list));
+ p_data += sizeof(struct event_tmpl);
+ }
+ break;
+ }
+ case EC_IOCTL_ATTACH:
+ result = ec_user_attach ();
+ DPRINTF("Attach Probes");
+ break;
+ case EC_IOCTL_ACTIVATE:
+ result = ec_user_activate ();
+ DPRINTF("Activate Probes");
+ break;
+ case EC_IOCTL_STOP_AND_DETACH:
+ {
+ unsigned long nIgnoredBytes = 0;
+ if(ec_user_stop() != 0) {
+ result = -1;
+ break;
+ }
+ nIgnoredBytes = copy_ec_info_to_user_space ((ec_info_t*)arg);
+ if(nIgnoredBytes > 0) {
+ result = -1;
+ break;
+ }
+ result = 0;
+ DPRINTF("Stop and Detach Probes");
+ break;
+ }
+ case EC_IOCTL_WAIT_NOTIFICATION:
+ {
+ static ec_info_t ec_info_copy;
+
+ ioctl_wait_notification_t ioctl_args;
+
+ result = copy_from_user (&ioctl_args, (void *) arg, sizeof (ioctl_args));
+ if (result)
+ {
+ result = -1;
+ break;
+ }
+
+ result = wait_event_interruptible (notification_waiters_queue, ioctl_args.notification_count != notification_count);
+ if (result)
+ {
+ result = -EINTR; // woken by signal (ERESTARTSYS 512)
+ break;
+ }
+
+ ioctl_args.notification_count = notification_count;
+
+ result = copy_to_user ((void *) arg, &ioctl_args, sizeof (ioctl_args));
+ if (result)
+ {
+ result = -1;
+ break;
+ }
+
+ // FIXME: synchronization is necessary here (ec_info must be locked).
+ // ENTER_CRITICAL_SECTION
+ memcpy (&ec_info_copy, &ec_info, sizeof (ec_info_copy));
+ // LEAVE_CRITICAL_SECTION
+
+ result = copy_to_user ((void *) ioctl_args.p_ec_info, &ec_info_copy, sizeof (ec_info_t));
+ if (result)
+ {
+ EPRINTF ("copy_to_user(%08X,%08X)=%d", (unsigned) ioctl_args.p_ec_info, (unsigned) &ec_info_copy, result);
+ result = -1;
+ break;
+ }
+ DPRINTF("Wake up");
+ break;
+ }
+ case EC_IOCTL_INST_USR_SPACE_PROC:
+ {
+ ioctl_inst_usr_space_proc_t ioctl_args;
+ result = copy_from_user (&ioctl_args, (void *) arg, sizeof (ioctl_args));
+ if (result)
+ {
+ result = -1;
+ EPRINTF ("copy_from_user() failure");
+ }
+ else
+ {
+ result = set_us_proc_inst_info (&ioctl_args);
+ }
+ DPRINTF("Instrument User Space Procedures");
+ break;
+ }
+ case EC_IOCTL_DEINST_USR_SPACE_PROC:
+ {
+ release_us_proc_inst_info ();
+ result = 0;
+ DPRINTF("Deinstrument User Space Procedures");
+ break;
+ }
+ case EC_IOCTL_US_EVENT:
+ {
+ ioctl_us_event_t ioctl_args;
+ result = copy_from_user (&ioctl_args, (void *) arg, sizeof (ioctl_args));
+ if (result)
+ {
+ result = -1;
+ EPRINTF ("copy_from_user() failure");
+ }
+ else
+ {
+ if(ioctl_args.len == 0){
+ result = -EINVAL;
+ EPRINTF ("invalid event length!");
+ }
+ else {
+ char *buf = kmalloc(ioctl_args.len, GFP_KERNEL);
+ if(!buf){
+ result = -ENOMEM;
+ EPRINTF ("failed to alloc mem for event!");
+ }
+ else {
+ result = copy_from_user (buf, (void *) ioctl_args.data, ioctl_args.len);
+ if (result){
+ result = -1;
+ EPRINTF ("failed to copy event from user space!");
+ }
+ else
+ result = put_us_event(buf, ioctl_args.len);
+ kfree(buf);
+ }
+ }
+ }
+// DPRINTF("User Space Event"); // Frequent call
+ break;
+ }
+
+ case EC_IOCTL_SET_EVENT_MASK:
+ {
+ int mask;
+ result = copy_from_user (&mask, (void *) arg, sizeof (mask));
+ if (result)
+ {
+ result = -EFAULT;
+ break;
+ }
+
+ result = set_event_mask (mask);
+ if (result)
+ {
+ break;
+ }
+ DPRINTF("Set Event Mask = %d", mask);
+ break;
+ }
+
+ case EC_IOCTL_GET_EVENT_MASK:
+ {
+ int mask = 0;
+ result = get_event_mask(&mask);
+ if (result)
+ {
+ result = -EFAULT;
+ }
+ result = copy_to_user ((void *) arg, &mask, sizeof (mask));
+ if (result)
+ {
+ result = -EFAULT;
+ }
+ DPRINTF("Get Event Mask = %d", mask);
+ break;
+ }
+
+ case EC_IOCTL_SET_PREDEF_UPROBES:
+ {
+ ioctl_predef_uprobes_info_t data;
+ result = copy_from_user (&data, (void *) arg, sizeof (data));
+ if (result)
+ {
+ result = -EFAULT;
+ break;
+ }
+
+ result = set_predef_uprobes (&data);
+ if (result)
+ {
+ break;
+ }
+ DPRINTF("Set Predefined User Space Probes");
+ break;
+ }
+
+ case EC_IOCTL_GET_PREDEF_UPROBES:
+ {
+ result = get_predef_uprobes((ioctl_predef_uprobes_info_t *)arg);
+ if (result)
+ {
+ result = -EFAULT;
+ }
+ DPRINTF("Get Predefined User Space Probes");
+ break;
+ }
+
+ case EC_IOCTL_GET_PREDEF_UPROBES_SIZE:
+ {
+ int size = 0;
+ result = get_predef_uprobes_size(&size);
+ if (result)
+ {
+ result = -EFAULT;
+ }
+ result = copy_to_user ((void *) arg, &size, sizeof (size));
+ if (result)
+ {
+ result = -EFAULT;
+ }
+ DPRINTF("Get Size of Predefined User Space Probes");
+ break;
+ }
+
+ default:
+ EPRINTF ("Unknown driver command = %u", cmd);
+ result = -EINVAL;
+ break;
+ }
+
+ return result;
+}
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: device_driver.h
+//
+// DESCRIPTION:
+//
+// SEE ALSO: device_driver.c
+// AUTHOR: L.Komkov, S.Dianov, S.Grekhov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#if !defined(device_driver_h)
+#define device_driver_h
+
+#include "event.h" // event_id_t
+#include "ec_info.h" // ec_info_t
+#include "ec_probe.h" // probe_id_t
+
+#define DEFAULT_DEVICE_NAME "inperfa_drv"
+#define DEFAULT_DEVICE_MAJOR 250
+#define EVENTS_AGGREGATION_USEC (5 * 1000000UL)
+
+extern int device_init (void);
+extern void device_down (void);
+extern void notify_user (event_id_t event_id);
+
+#endif /* !defined(device_driver_h) */
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: ec.c
+//
+// DESCRIPTION:
+// This file is C++ source for SWAP driver.
+//
+// SEE ALSO: ec.h
+// AUTHOR: L.Komkov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#include "module.h"
+#include "ec.h"
+#include "CProfile.h"
+
+////////////////////////////////////////////////////////////////////////////////////////////
+
+ec_info_t ec_info = {
+ .ec_state = EC_STATE_IDLE,
+ .m_nMode = 0L,
+ .buffer_size = EC_BUFFER_SIZE_DEFAULT,
+ .ignored_events_count = 0,
+ .m_nNumOfSubbuffers = 0,
+ .m_nSubbufSize = DEFAULT_SUBBUF_SIZE,
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////
+
+spinlock_t ec_spinlock = SPIN_LOCK_UNLOCKED; // protects 'ec_info'
+
+ec_probe_info_t ec_probe_info = {
+ .probe_id = -1,
+ .probe_selected = 0,
+ .jprobe_active = 0,
+ .retprobe_active = 0,
+ .address = 0,
+};
+
+spinlock_t ec_probe_spinlock = SPIN_LOCK_UNLOCKED; // protects 'ec_probe_info'
+
+ec_state_t GetECState(void) { return ec_info.ec_state; };
+
+void ResetECInfo(void) {
+ unsigned long spinlock_flags = 0L;
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ec_info.trace_size = 0;
+ ec_info.first = 0;
+ ec_info.after_last = 0;
+ ec_info.ignored_events_count = 0;
+ ec_info.saved_events_count = 0;
+ ec_info.discarded_events_count = 0;
+ ec_info.collision_count = 0;
+ ec_info.lost_events_count = 0;
+ ec_info.m_nBeginSubbufNum = 0;
+ ec_info.m_nEndSubbufNum = 0;
+ ec_info.m_nEndOffset = 0;
+ ec_info.m_nSubbufSavedEvents = 0;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+}
+
+void CleanECInfo(void) {
+ unsigned long spinlock_flags = 0L;
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ec_info.buffer_effect = 0;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ ResetECInfo();
+
+}
+
+int IsECMode(unsigned long nMask) { return ((ec_info.m_nMode & nMask) != 0); }
+
+int IsMultipleBuffer() { return IsECMode(MODEMASK_MULTIPLE_BUFFER); }
+
+int IsContinuousRetrieval() { return IsECMode(MODEMASK_CONTINUOUS_RETRIEVAL); }
+
+int SetECMode(unsigned long nECMode) {
+ unsigned long spinlock_flags = 0L;
+
+ if((nECMode & MODEMASK_MULTIPLE_BUFFER) != 0) {
+ if(EnableMultipleBuffer() == -1) {
+ EPRINTF("Cannot enable multiple buffer!");
+ return -1;
+ }
+ } else {
+ if(DisableMultipleBuffer() == -1) {
+ EPRINTF("Cannot disable multiple buffer!");
+ return -1;
+ }
+ }
+ if((nECMode & MODEMASK_CONTINUOUS_RETRIEVAL) != 0) {
+ if(EnableContinuousRetrieval() == -1) {
+ EPRINTF("Cannot enable continuous retrieval!");
+ return -1;
+ }
+ } else {
+ if(DisableContinuousRetrieval() == -1) {
+ EPRINTF("Cannot disable continuous retrieval!");
+ return -1;
+ }
+ }
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ec_info.m_nMode = nECMode;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ return 0;
+}
+
+unsigned long GetECMode(void) { return ec_info.m_nMode; }
+
+unsigned int GetNumOfSubbuffers(unsigned long nBufferSize)
+{
+ if(nBufferSize % ec_info.m_nSubbufSize > 0)
+ EPRINTF("The buffer size is not divisible by a subbuffer size!");
+ return nBufferSize / ec_info.m_nSubbufSize;
+};
+
+#if defined(__DEBUG)
+static UNUSED char * ec_state_name (ec_state_t ec_state)
+{
+ static char *ec_state_names[EC_STATE_TAG_COUNT] = { "IDLE", "ATTACHED", "ACTIVE", "STOPPED" };
+
+ if (((unsigned) ec_info.ec_state) < EC_STATE_TAG_COUNT)
+ {
+ return ec_state_names[ec_info.ec_state];
+ }
+ else
+ {
+ return "<unknown>";
+ }
+}
+#endif /* defined(__DEBUG) */
+
+
+/*
+ On user request user space EC may change state in the following order:
+ IDLE -> ATTACHED (on "attach")
+ IDLE | ATTACHED -> ACTIVE (on "activate")
+ ATTACHED | ACTIVE | STOPPED -> IDLE (on "stop"/"detach")
+*/
+int ec_user_attach (void)
+{
+ unsigned long spinlock_flags;
+ int result;
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags); // make other CPUs wait
+ if (EC_STATE_IDLE == ec_info.ec_state)
+ {
+ int tmp;
+ ec_info.ec_state = EC_STATE_ATTACHED;
+
+ /* save 'start' time */
+ struct timeval tv;
+ do_gettimeofday(&tv);
+ memcpy(&last_attach_time, &tv, sizeof(struct timeval));
+
+ /* unpause if paused */
+ paused = 0;
+
+ /* if there is at least one start condition in the list
+ we are paused at the beginning */
+ struct cond *p_cond;
+ struct event_tmpl *p_tmpl;
+ list_for_each_entry(p_cond, &cond_list.list, list) {
+ p_tmpl = &p_cond->tmpl;
+ if (p_tmpl->type == ET_TYPE_START_COND) {
+ paused = 1;
+ break;
+ }
+ }
+
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+
+ //first of all put event with event format
+ tmp = event_mask;
+ event_mask = 0;
+ pack_event_info(EVENT_FMT_PROBE_ID, RECORD_ENTRY, "x", tmp);
+ event_mask = tmp;
+
+ result = attach_selected_probes ();
+ if (result == 0) // instrument user space process
+ result = inst_usr_space_proc ();
+ // FIXME: SAFETY CHECK
+ if (result)
+ { // return to safe state
+ detach_selected_probes ();
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags); // make other CPUs wait
+ ec_info.ec_state = EC_STATE_IDLE;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+ }
+ // FIXME: SAFETY CHECK
+
+ notify_user (EVENT_EC_STATE_CHANGE);
+
+ }
+ else
+ {
+
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+ result = -EINVAL;
+
+ }
+
+ return result;
+}
+
+int ec_user_activate (void)
+{
+ unsigned long spinlock_flags;
+ int result;
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags); // make other CPUs wait
+ if (EC_STATE_IDLE == ec_info.ec_state)
+ {
+ int tmp;
+ ec_info.ec_state = EC_STATE_ACTIVE;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+ //first of all put event with event format
+ tmp = event_mask;
+ event_mask = 0;
+ pack_event_info(EVENT_FMT_PROBE_ID, RECORD_ENTRY, "x", tmp);
+ event_mask = tmp;
+
+ result = attach_selected_probes ();
+ if (result == 0) // instrument user space process
+ result = inst_usr_space_proc ();
+ // FIXME: SAFETY CHECK
+ if (result)
+ { // return to safe state
+ detach_selected_probes ();
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags); // make other CPUs wait
+ ec_info.ec_state = EC_STATE_IDLE;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+ }
+ // FIXME: SAFETY CHECK
+
+ notify_user (EVENT_EC_STATE_CHANGE);
+
+ }
+ else if (EC_STATE_ATTACHED == ec_info.ec_state)
+ {
+
+ ec_info.ec_state = EC_STATE_ACTIVE;
+ result = 0;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+
+ notify_user (EVENT_EC_STATE_CHANGE);
+
+ }
+ else
+ {
+
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+ result = -EINVAL;
+ }
+
+ return result;
+}
+
+int ec_user_stop (void)
+{
+ unsigned long spinlock_flags;
+ int result = 0, ret = 0;
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags); // make other CPUs wait
+ if (EC_STATE_ATTACHED == ec_info.ec_state || EC_STATE_ACTIVE == ec_info.ec_state || EC_STATE_STOPPED == ec_info.ec_state)
+ {
+
+ ec_info.ec_state = EC_STATE_IDLE;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+
+ ret = deinst_usr_space_proc ();
+ result = detach_selected_probes ();
+ if (result == 0)
+ result = ret;
+
+ notify_user (EVENT_EC_STATE_CHANGE);
+
+ }
+ else
+ {
+
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+ result = -EINVAL;
+
+ }
+
+ return result;
+}
+
+/*
+ Kernel space EC may change state in the following order:
+ ATTACHED -> ACTIVE (when start condition is satisfied)
+ ACTIVE -> STOPPED (when stop condition is satisfied)
+*/
+int ec_kernel_activate (void)
+{
+ unsigned long spinlock_flags;
+ int result;
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags); // make other CPUs wait
+ if (EC_STATE_ATTACHED == ec_info.ec_state)
+ {
+ ec_info.ec_state = EC_STATE_ACTIVE;
+ result = 0;
+ }
+ else
+ {
+ result = -EINVAL;
+ }
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+
+ notify_user (EVENT_EC_STATE_CHANGE);
+
+ return result;
+}
+
+int ec_kernel_stop (void)
+{
+ unsigned long spinlock_flags;
+ int result;
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags); // make other CPUs wait
+ if (EC_STATE_ACTIVE == ec_info.ec_state)
+ {
+ ec_info.ec_state = EC_STATE_STOPPED;
+ result = 0;
+ }
+ else
+ {
+ result = -EINVAL;
+ }
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+
+ notify_user (EVENT_EC_STATE_CHANGE);
+
+ return result;
+}
+
+// Copies EC info to user space
+// Since "copy_to_user" may block, an intermediate copy of ec_info is used here
+int copy_ec_info_to_user_space (ec_info_t * p_user_ec_info)
+{
+ /*
+ WARNING: to avoid stack overflow the following data structure was made
+ static. As result, simultaneous users of this function will share it
+ and must use additional synchronization to avoid collisions.
+ */
+ // FIXME: synchronization is necessary here (ec_info_copy must be locked).
+ static ec_info_t ec_info_copy;
+ unsigned long spinlock_flags;
+ int result;
+
+ // ENTER_CRITICAL_SECTION
+ // lock semaphore here
+
+
+ // ENTER_CRITICAL_SECTION
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags); // make other CPUs wait
+
+ // copy
+ memcpy (&ec_info_copy, &ec_info, sizeof (ec_info_copy));
+
+ // LEAVE_CRITICAL_SECTION
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags); // open our data for other CPUs
+
+
+ result = copy_to_user (p_user_ec_info, &ec_info_copy, sizeof (ec_info_t));
+
+ // LEAVE_CRITICAL_SECTION
+ // unlock semaphore here
+
+ if (result)
+ {
+ EPRINTF ("copy_to_user(%08X,%08X)=%d", (unsigned) p_user_ec_info, (unsigned) &ec_info_copy, result);
+ result = -EFAULT;
+ }
+ return result;
+}
+
+int copy_ec_probe_info_to_user_space (ec_probe_info_t * p_user_ec_probe_info)
+{
+ /*
+ WARNING: to avoid stack overflow the following data structure was made
+ static. As result, simultaneous users of this function will share it
+ and must use additional synchronization to avoid collisions.
+ */
+ // FIXME: synchronization is necessary here (ec_info_copy must be locked).
+ ec_probe_info_t ec_probe_info_copy;
+ unsigned long spinlock_flags;
+ int result;
+
+ // ENTER_CRITICAL_SECTION
+ // lock semaphore here
+
+
+ // ENTER_CRITICAL_SECTION
+ spin_lock_irqsave (&ec_probe_spinlock, spinlock_flags); // make other CPUs wait
+
+ // copy
+ memcpy (&ec_probe_info_copy, &ec_probe_info, sizeof (ec_probe_info_copy));
+
+ // LEAVE_CRITICAL_SECTION
+ spin_unlock_irqrestore (&ec_probe_spinlock, spinlock_flags); // open our data for other CPUs
+
+
+ result = copy_to_user (p_user_ec_probe_info, &ec_probe_info_copy, sizeof (ec_probe_info_t));
+
+ // LEAVE_CRITICAL_SECTION
+ // unlock semaphore here
+
+ if (result)
+ {
+ EPRINTF ("copy_to_user(%08X,%08X)=%d", (unsigned) p_user_ec_probe_info, (unsigned) &ec_probe_info_copy, result);
+ result = -EFAULT;
+ }
+ return result;
+}
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: ec.h
+//
+// DESCRIPTION:
+//
+// SEE ALSO: ec.c
+// AUTHOR: L.Komkov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#if !defined(__EC_H__)
+#define __EC_H__
+
+#include "ec_ioctl.h"
+#include "ec_info.h"
+#include "ec_module.h"
+#include "ec_probe.h"
+#include "picl.h"
+
+extern ec_info_t ec_info;
+extern ec_probe_info_t ec_probe_info;
+extern spinlock_t ec_spinlock;
+
+extern int ec_user_attach (void);
+extern int ec_user_activate (void);
+extern int ec_user_stop (void);
+extern int ec_kernel_activate (void);
+extern int ec_kernel_stop (void);
+
+extern int copy_ec_info_to_user_space (ec_info_t * p_user_ec_info);
+extern int copy_ec_probe_info_to_user_space (ec_probe_info_t * p_user_ec_probe_info);
+
+extern ec_state_t GetECState(void);
+extern void ResetECInfo(void);
+extern void CleanECInfo(void);
+extern int IsECMode(unsigned long nMask);
+extern int IsMultipleBuffer(void);
+extern int IsContinuousRetrieval(void);
+extern int SetECMode(unsigned long nECMode);
+extern unsigned long GetECMode(void);
+extern unsigned int GetNumOfSubbuffers(unsigned long nBufferSize);
+extern struct timeval last_attach_time;
+extern int paused;
+
+#endif /* !defined(__EC_H__) */
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: ec_module.h
+//
+// DESCRIPTION:
+//
+// SEE ALSO: N/A
+// AUTHOR: L.Komkov, S.Dianov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.02
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#if !defined(ec_ioctl_h)
+#define ec_ioctl_h
+
+#include "ec_info.h" // ec_info_t
+#include "picl.h"
+
+typedef enum
+{
+ // modes
+ EC_IOCTL_SET_EC_MODE,
+ EC_IOCTL_GET_EC_MODE,
+
+ // buffer manipulation
+ EC_IOCTL_SET_BUFFER_SIZE,
+ EC_IOCTL_GET_BUFFER_SIZE,
+ EC_IOCTL_RESET_BUFFER,
+
+ // probes management
+ EC_IOCTL_SELECT_PROBE,
+ EC_IOCTL_DESELECT_PROBE,
+ EC_IOCTL_GET_PROBE_INFO,
+ EC_IOCTL_ADD_PROBE,
+ //EC_IOCTL_REMOVE_PROBE,
+ EC_IOCTL_RESET_PROBES,
+ EC_IOCTL_SET_COMPLEX_PROBES,
+
+ // tracing
+ EC_IOCTL_ATTACH, // attaches selected probes
+ EC_IOCTL_ACTIVATE, // START
+
+ // stop is common for both tracing and profiling
+ EC_IOCTL_STOP_AND_DETACH, // STOP (and detach for TRACING)
+
+ // kernel to user notification delivery
+ EC_IOCTL_WAIT_NOTIFICATION,
+
+ // get ec_info
+ EC_IOCTL_GET_EC_INFO,
+ EC_IOCTL_GET_COMPLEX_STATUS,
+ EC_IOCTL_CONSUME_BUFFER,
+
+ // instrument user space process
+ EC_IOCTL_INST_USR_SPACE_PROC,
+ // deinstrument user space process
+ EC_IOCTL_DEINST_USR_SPACE_PROC,
+
+ // conditions
+ EC_IOCTL_UPDATE_CONDS,
+
+ //user space event
+ EC_IOCTL_US_EVENT,
+
+ //event format
+ EC_IOCTL_SET_EVENT_MASK,
+ EC_IOCTL_GET_EVENT_MASK,
+
+ // pre-defined user space probes
+ EC_IOCTL_SET_PREDEF_UPROBES,
+ EC_IOCTL_GET_PREDEF_UPROBES,
+ EC_IOCTL_GET_PREDEF_UPROBES_SIZE,
+
+} EC_IOCTL_CMD;
+
+typedef struct
+{
+ unsigned notification_count;
+ ec_info_t *p_ec_info;
+} ioctl_wait_notification_t;
+
+typedef struct
+{
+ int m_signedInt;
+ unsigned int m_unsignedInt;
+ long m_signedLong;
+ unsigned long m_unsignedLong;
+ char* m_ptrChar;
+} ioctl_general_t;
+
+typedef enum
+{
+ OPERATION_ANY, // means do not check value
+ OPERATION_EQUAL,
+ OPERATION_NOT_EQUAL,
+ OPERATION_LESS,
+ OPERATION_GREATER
+} operation_t;
+
+typedef struct
+{
+ unsigned m_condition_always_false;
+ unsigned m_op_time;
+ unsigned m_op_pid;
+ unsigned m_op_tid;
+ unsigned m_op_probe_id;
+// char m_probe_name[SWAP_COMMON_STRING_SIZE];
+ unsigned m_op_record_type;
+ unsigned m_time_sec;
+ unsigned m_time_usec;
+ unsigned m_pid;
+ unsigned m_tid;
+ unsigned m_probe_id;
+ unsigned m_record_type;
+} condition_t;
+
+
+
+// condition matching any event
+#define CONDITION_ANY \
+{ \
+ .m_condition_always_false = 0, \
+ .m_op_time = OPERATION_ANY, \
+ .m_op_pid = OPERATION_ANY, \
+ .m_op_tid = OPERATION_ANY, \
+ .m_op_probe_id = OPERATION_ANY, \
+ .m_op_record_type = OPERATION_ANY, \
+} \
+
+// never matching condition
+#define CONDITION_FALSE \
+{ \
+ .m_condition_always_false = 1, \
+ .m_op_time = OPERATION_ANY, \
+ .m_op_pid = OPERATION_ANY, \
+ .m_op_tid = OPERATION_ANY, \
+ .m_op_probe_id = OPERATION_ANY, \
+ .m_op_record_type = OPERATION_ANY, \
+} \
+
+// default start condition - start immediately
+#define DEFAULT_START_CONDITION CONDITION_ANY
+// default stop condition - never stop
+#define DEFAULT_STOP_CONDITION CONDITION_FALSE
+
+
+typedef struct
+{
+ unsigned count;
+ int *p_pids;
+} ioctl_set_pids_to_ignore_t;
+
+
+typedef struct
+{
+ char *name;
+ unsigned long addr;
+ char type;
+ unsigned long size;
+ signed char reg; // -1 - memory, 0..127 - register number
+ long off;
+} ioctl_usr_space_vtp_t;
+
+typedef struct
+{
+ //char *name;
+ //unsigned name_len;
+ unsigned long addr;
+} ioctl_usr_space_ip_t;
+
+typedef struct
+{
+ char *path;
+ //unsigned path_len;
+ unsigned ips_count;
+ ioctl_usr_space_ip_t *p_ips;
+ unsigned vtps_count;
+ ioctl_usr_space_vtp_t *p_vtps;
+} ioctl_usr_space_lib_t;
+
+typedef struct
+{
+ char *path;
+ //unsigned path_len;
+ unsigned libs_count;
+ ioctl_usr_space_lib_t *p_libs;
+} ioctl_inst_usr_space_proc_t;
+
+typedef struct
+{
+ char *proc_name;
+ unsigned count;
+} ioctl_set_proc_to_ignore_t;
+
+typedef struct
+{
+ char *data;
+ unsigned len;
+} ioctl_us_event_t;
+
+// exclude entry events
+#define IOCTL_EMASK_ENTRY 0x01
+// exclude exit events
+#define IOCTL_EMASK_EXIT 0x02
+// timestamp
+#define IOCTL_EMASK_TIME 0x04
+// PID
+#define IOCTL_EMASK_PID 0x08
+// TID
+#define IOCTL_EMASK_TID 0x10
+// CPU
+#define IOCTL_EMASK_CPU 0x20
+// Args
+#define IOCTL_EMASK_ARGS 0x40
+
+typedef struct
+{
+ unsigned probes_count;
+ char *p_probes;
+} ioctl_predef_uprobes_info_t;
+
+
+#endif /* !defined(ec_ioctl_h) */
--- /dev/null
+#ifndef EVENTS_H_
+#define EVENTS_H_
+
+#define UNUSED __attribute__((unused))
+
+#ifdef __KERNEL__
+#include <asm/uaccess.h> // copy_from_user
+#include "debug.h" // //DPRINTF
+#else
+#include <string.h>
+#include <stdarg.h>
+#endif
+#include "picl.h"
+#include "ec_ioctl.h"
+#include "ec_probe.h"
+
+static TYPE_ARGUMENT GetArgumentType (char ch)
+{
+ switch (ch)
+ {
+ case 'c':
+ return AT_CHAR;
+ break;
+ case 'h':
+ return AT_SHORT;
+ break;
+ case 'd':
+ return AT_INT;
+ break;
+ case 'x':
+ return AT_LONG;
+ break;
+ case 'p':
+ return AT_PTR;
+ break;
+ case 'f':
+ return AT_FLOAT;
+ break;
+ case 'w':
+ return AT_DOUBLE;
+ break;
+ case 's':
+ return AT_STRING;
+ break;
+ case 'a':
+ return AT_ARRAY;
+ break;
+ };
+ return AT_UNKNOWN;
+}
+
+static char *PackArguments (char *pBuffer, unsigned long nLen, const char *szFormat, va_list args)
+{
+ TYPE_ARGUMENT nArgType = AT_UNKNOWN;
+ const char *pChar = NULL;
+ char chCode = '\0';
+ char *pResult = pBuffer;
+ unsigned long nLengthOfDescriptor = 0, nFree = nLen;
+ unsigned long nSizeOfDescriptor = 0;
+
+ // Descriptor
+ nLengthOfDescriptor = strlen (szFormat) + 1;
+ nSizeOfDescriptor = ALIGN_VALUE(nLengthOfDescriptor);
+ if(nFree < nSizeOfDescriptor)
+ return NULL; // no space for descriptor
+ memcpy (pResult, szFormat, nLengthOfDescriptor);
+ pResult += nSizeOfDescriptor;
+ nFree -= nSizeOfDescriptor;
+
+ for (pChar = szFormat; (chCode = *pChar) != '\0'; pChar++)
+ {
+ nArgType = GetArgumentType(chCode);
+ switch (nArgType)
+ {
+ case AT_CHAR:
+ {
+ int ch = va_arg (args, int);
+ if(nFree < sizeof(ch))
+ return NULL; // no space for arg
+ memcpy(pResult, &ch, sizeof (ch));
+ pResult += sizeof (ch);
+ nFree -= sizeof (ch);
+ }
+ break;
+ case AT_SHORT:
+ {
+ int nShort = va_arg (args, int);
+ if(nFree < sizeof(nShort))
+ return NULL; // no space for arg
+ memcpy(pResult, &nShort, sizeof (nShort));
+ pResult += sizeof (nShort);
+ nFree -= sizeof (nShort);
+ }
+ break;
+ case AT_INT:
+ {
+ int nInt = va_arg (args, int);
+ if(nFree < sizeof(nInt))
+ return NULL; // no space for arg
+ memcpy(pResult, &nInt, sizeof (nInt));
+ pResult += sizeof (nInt);
+ nFree -= sizeof (nInt);
+ }
+ break;
+ case AT_LONG:
+ {
+ long nLong = va_arg (args, long);
+ if(nFree < sizeof(nLong))
+ return NULL; // no space for arg
+ memcpy (pResult, &nLong, sizeof (nLong));
+ pResult += sizeof (nLong);
+ nFree -= sizeof (nLong);
+ }
+ break;
+ case AT_PTR:
+ {
+ void *p = va_arg (args, void *);
+ if(nFree < sizeof(p))
+ return NULL; // no space for arg
+ memcpy (pResult, &p, sizeof (p));
+ pResult += sizeof (p);
+ nFree -= sizeof (p);
+ }
+ break;
+ case AT_FLOAT:
+ {
+ double fValue = va_arg (args, double);
+ if(nFree < sizeof(fValue))
+ return NULL; // no space for arg
+ memcpy (pResult, &fValue, sizeof (fValue));
+ pResult += sizeof (fValue);
+ nFree -= sizeof (fValue);
+ }
+ break;
+ case AT_DOUBLE:
+ {
+ double fDouble = va_arg (args, double);
+ if(nFree < sizeof(fDouble))
+ return NULL; // no space for arg
+ memcpy (pResult, &fDouble, sizeof (fDouble));
+ pResult += sizeof (fDouble);
+ nFree -= sizeof (fDouble);
+ }
+ break;
+ case AT_STRING:
+ {
+ const char *s = va_arg (args, const char *);
+ int nLengthOfString = 0, nSizeOfString;
+#ifdef __KERNEL__
+ if((void *)s < (void *)TASK_SIZE) {
+ nLengthOfString = strlen_user (s);
+ if(nFree < nLengthOfString)
+ return NULL; // no space for arg
+ if(strncpy_from_user(pResult, s, nLengthOfString) != (nLengthOfString-1))
+ EPRINTF ("failed to copy string from user %p, bytes %d", s, nLengthOfString);
+ }
+ else
+#endif
+ {
+ nLengthOfString = strlen (s) + 1;
+ if(nFree < nLengthOfString)
+ return NULL; // no space for arg
+ memcpy (pResult, s, nLengthOfString);
+ }
+ nSizeOfString = ALIGN_VALUE (nLengthOfString);
+ if(nFree < nSizeOfString)
+ return NULL; // no space for arg
+ pResult += nSizeOfString;
+ nFree -= nSizeOfString;
+ }
+ break;
+ case AT_ARRAY:
+ {
+ int nLength = va_arg (args, int);
+ void *p = NULL;
+ int nSize = 0;
+ nSize = ALIGN_VALUE (nLength);
+ if(nFree < nSize)
+ return NULL; // no space for arg
+ memcpy (pResult, &nLength, sizeof (int));
+ pResult += sizeof (int);
+ p = va_arg (args, void *);
+#ifdef __KERNEL__
+ if(p < (void *)TASK_SIZE) {
+ if(copy_from_user(pResult, p, nLength)!= 0)
+ EPRINTF ("failed to copy array from user %p, bytes %d", p, nLength);
+ }
+ else
+#endif
+ memcpy (pResult, p, nLength);
+ pResult += nSize;
+ nFree -= nSize;
+ }
+ break;
+ default:
+ break;
+ };
+ }
+ return pResult;
+}
+
+static UNUSED TYPEOF_EVENT_LENGTH VPackEvent(char *buf, unsigned long buf_len, int mask, TYPEOF_PROBE_ID probe_id,
+ TYPEOF_EVENT_TYPE record_type, TYPEOF_TIME *tv, TYPEOF_PROCESS_ID pid,
+ TYPEOF_THREAD_ID tid, TYPEOF_CPU_NUMBER cpu, const char *fmt, va_list args)
+{
+ char *cur = buf;
+ SWAP_TYPE_EVENT_HEADER *pEventHeader = (SWAP_TYPE_EVENT_HEADER *)buf;
+
+ if(buf_len < sizeof(SWAP_TYPE_EVENT_HEADER))
+ return 0; // no space for header
+
+ pEventHeader->m_nLength = 0;
+ cur += sizeof(TYPEOF_EVENT_LENGTH);
+ pEventHeader->m_nType = record_type;
+ cur += sizeof(TYPEOF_EVENT_TYPE);
+ pEventHeader->m_nProbeID = probe_id;
+ cur += sizeof(TYPEOF_PROBE_ID);
+ //pEventHeader->m_time.tv_sec = tv->tv_sec;
+ //pEventHeader->m_time.tv_usec = tv->tv_usec;
+ if((probe_id == EVENT_FMT_PROBE_ID) || !(mask & IOCTL_EMASK_TIME)){
+ memcpy(cur, tv, sizeof(TYPEOF_TIME));
+ cur += sizeof(TYPEOF_TIME);
+ }
+ //pEventHeader->m_nProcessID = pid;
+ if((probe_id == EVENT_FMT_PROBE_ID) || !(mask & IOCTL_EMASK_PID)){
+ (*(TYPEOF_PROCESS_ID *)cur) = pid;
+ cur += sizeof(TYPEOF_PROCESS_ID);
+ }
+ //pEventHeader->m_nThreadID = tid;
+ if((probe_id == EVENT_FMT_PROBE_ID) || !(mask & IOCTL_EMASK_TID)){
+ (*(TYPEOF_THREAD_ID *)cur) = tid;
+ cur += sizeof(TYPEOF_THREAD_ID);
+ }
+ //pEventHeader->m_nCPU = cpu;
+ if((probe_id == EVENT_FMT_PROBE_ID) || !(mask & IOCTL_EMASK_CPU)){
+ (*(TYPEOF_CPU_NUMBER *)cur) = cpu;
+ cur += sizeof(TYPEOF_CPU_NUMBER);
+ }
+ // dyn lib event should have all args, it is for internal use and not visible to user
+ if((probe_id == EVENT_FMT_PROBE_ID) || (probe_id == DYN_LIB_PROBE_ID) || !(mask & IOCTL_EMASK_ARGS)){
+ (*(TYPEOF_NUMBER_OF_ARGS *)cur) = strlen(fmt);
+ cur += sizeof(TYPEOF_NUMBER_OF_ARGS);
+ cur = PackArguments(cur, buf_len-(cur-buf), fmt, args);
+ if(!cur) return 0; // no space for args
+ }
+ else {
+ // user space and dynamic kernel probes should have at least one argument
+ // to identify them
+ if((probe_id == US_PROBE_ID) || (probe_id == VTP_PROBE_ID) || (probe_id == KS_PROBE_ID)){
+ char fmt2[2];
+ (*(TYPEOF_NUMBER_OF_ARGS *)cur) = 1;
+ cur += sizeof(TYPEOF_NUMBER_OF_ARGS);
+ // pack args using format string for the 1st arg only
+ fmt2[0] = fmt[0]; fmt2[1] = '\0';
+ cur = PackArguments(cur, buf_len-(cur-buf), fmt2, args);
+ if(!cur) return 0; // no space for args
+ }
+ else {
+ (*(TYPEOF_NUMBER_OF_ARGS *)cur) = 0;
+ cur += sizeof(TYPEOF_NUMBER_OF_ARGS);
+ }
+ }
+
+ pEventHeader->m_nLength = cur - buf + sizeof(TYPEOF_EVENT_LENGTH);
+ if(buf_len < pEventHeader->m_nLength)
+ return 0;// no space for back length
+ //memcpy(cur, &pEventHeader->m_nLength, sizeof(TYPEOF_EVENT_LENGTH));
+ *((TYPEOF_EVENT_LENGTH *)cur) = pEventHeader->m_nLength;
+
+ return pEventHeader->m_nLength;
+}
+
+/*static TYPEOF_EVENT_LENGTH PackEvent(char *buf, unsigned long buf_len, TYPEOF_PROBE_ID probe_id,
+ TYPEOF_EVENT_TYPE record_type, TYPEOF_TIME *tv, TYPEOF_PROCESS_ID pid,
+ TYPEOF_THREAD_ID tid, TYPEOF_CPU_NUMBER cpu, const char *fmt, ...)
+{
+ va_list args;
+ TYPEOF_EVENT_LENGTH len;
+
+ va_start (args, fmt);
+ len = VPackEvent(buf, buf_len, probe_id, record_type, tv, pid, tid, cpu, fmt, args);
+ va_end (args);
+
+ return len;
+}*/
+
+#endif /*EVENTS_H_*/
--- /dev/null
+#!/bin/sh
+
+
+# name of the file with module
+MODULE_FILE=inperfa_driver
+
+# device name
+DEVICE=inperfa_drv
+DEFAULT_MAJOR=250
+
+# name of device visible in /proc/devices
+DEVICE_NAME=${DEVICE}
+
+# name of the device file for /dev/
+DEVICE_FILE=${DEVICE}
+
+KSYMS=kallsyms_lookup_name
+
+# ADDRESS for "kallsyms_lookup_name" function taken from /proc/kallsyms
+ADDRESS=0x`sed "/ kallsyms_lookup_name/ ! d" /proc/kallsyms | sed "s/ T kallsyms_lookup_name//"`
+
+if [ "${ADDRESS}" = "0x" ]; then
+ if [ "$1" = "" ]; then
+ echo "Enter kallsyms_lookup_name as parameter:"
+ echo "insmod.sh <kallsyms_lookup_name address>"
+ exit
+ else
+ ADDRESS=$1
+ echo "kallsyms_lookup_name address is ${ADDRESS}"
+ fi
+fi
+
+MAJOR=`sed "/${DEVICE_NAME}/ ! d" /proc/devices | sed "s/ ${DEVICE_NAME}//"`
+if [ "${MAJOR}" != "" ] ; then
+ echo "Inperfa Driver is already loaded!"
+ exit 1
+ rmmod ${MODULE_FILE}
+ MAJOR=`sed "/${DEVICE_NAME}/ ! d" /proc/devices | sed "s/ ${DEVICE_NAME}//"`
+ if [ "${MAJOR}" != "" ] ; then
+ echo "Error: Unable to unload driver module '${MODULE_FILE}'"
+ exit 1
+ fi
+fi
+
+if [ ! -c /dev/${DEVICE_FILE} ] ; then
+ echo "WARNING: Creating device node with major number [${DEFAULT_MAJOR}]!"
+ mknod /dev/${DEVICE_FILE} c ${DEFAULT_MAJOR} 0
+ if [ $? -ne 0 ]; then
+ echo "Error: Unable to create device node!"
+ exit
+ fi
+ chmod a+r /dev/${DEVICE_FILE}
+fi
+
+# load driver module
+echo "loading module '${MODULE_FILE}'"
+insmod ${MODULE_FILE}.ko fp_kallsyms_lookup_name=${ADDRESS} device_name=${DEVICE_NAME} device_major=${DEFAULT_MAJOR}
+if [ $? -ne 0 ]; then
+ echo "Error: Unable to load Swap Driver!"
+fi
+MAJOR=`sed "/${DEVICE_NAME}/ ! d" /proc/devices | sed "s/ ${DEVICE_NAME}//"`
+
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: legacy.c
+//
+// DESCRIPTION:
+// This file is C source for SWAP driver.
+//
+// SEE ALSO: legacy.h
+// AUTHOR: L.Komkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.02
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#include "module.h"
+#include "legacy.h"
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 14)
+signed long
+schedule_timeout_interruptible (signed long timeout)
+{
+ __set_current_state (TASK_INTERRUPTIBLE);
+ return schedule_timeout (timeout);
+}
+#endif /* kernel without schedule_timeout_interruptible */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
+int
+remap_vmalloc_range (struct vm_area_struct *vma, void *addr, unsigned long pgoff)
+{
+ unsigned long uaddr = vma->vm_start;
+ unsigned long usize = vma->vm_end - vma->vm_start;
+ int ret = -EINVAL;
+
+ if ((PAGE_SIZE - 1) & (unsigned long) addr)
+ {
+ return -EINVAL;
+ }
+
+ if (pgoff)
+ {
+ return -EINVAL;
+ }
+
+ while (usize)
+ {
+ ret = remap_pfn_range (vma, uaddr, vmalloc_to_pfn (addr), PAGE_SIZE, PAGE_SHARED);
+ if (ret)
+ {
+ break;
+ }
+ uaddr += PAGE_SIZE;
+ addr += PAGE_SIZE;
+ usize -= PAGE_SIZE;
+ };
+
+ return ret;
+}
+#endif /* kernel without remap_vmalloc_range() */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
+/*
+ To be mappable to user space, pages of memory allocated via "vmalloc" must
+ be marked with "PG_reserved" flag. Memory allocated via "vmalloc_user"
+ doesn't need it.
+*/
+static void
+_reserve_pages (void *p, unsigned size)
+{
+ unsigned pages = (size + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
+ while (pages)
+ {
+ SetPageReserved (vmalloc_to_page (p));
+
+ p += PAGE_SIZE;
+ --pages;
+ }
+}
+
+static void
+_unreserve_pages (void *p, unsigned size)
+{
+ unsigned pages = (size + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
+ while (pages)
+ {
+ ClearPageReserved (vmalloc_to_page (p));
+
+ p += PAGE_SIZE;
+ --pages;
+ }
+}
+
+void *
+vmalloc_user (unsigned long size)
+{
+ void *p = __vmalloc (size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
+ if (p)
+ {
+ memset (p, 0, size);
+ _reserve_pages (p, size);
+ }
+ return p;
+}
+
+void
+vfree_user (void *address, unsigned long size)
+{
+ _unreserve_pages (address, size);
+ vfree (address);
+}
+#endif /* kernel without vmalloc_user() */
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: legacy.h
+//
+// DESCRIPTION:
+//
+// SEE ALSO: legacy.c
+// AUTHOR: L.Komkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+/*
+ Support for legacy Linux kernel versions
+*/
+#if !defined(__LEGACY_H__)
+#define __LEGACY_H__
+
+#include <linux/mm.h>
+#include <linux/version.h>
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 14)
+extern signed long schedule_timeout_interruptible (signed long timeout);
+#endif /* kernel without schedule_timeout_interruptible */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
+extern int remap_vmalloc_range (struct vm_area_struct *vma, void *addr, unsigned long pgoff);
+#endif /* kernel without remap_vmalloc_range() */
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
+extern void *vmalloc_user (unsigned long size);
+extern void vfree_user (void *address, unsigned long size);
+#define VFREE_USER(address, size) \
+ { \
+ if(address != NULL) { \
+ vfree_user(address, size); \
+ address = NULL; \
+ } \
+ }
+#else
+#define VFREE_USER(address, size) \
+ { \
+ if(address != NULL) { \
+ vfree(address); \
+ address = NULL; \
+ } \
+ }
+#endif /* kernel without vmalloc_user() */
+
+#endif /* !defined(__LEGACY_H__) */
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: module.c
+//
+// DESCRIPTION:
+// This file is C source for SWAP driver.
+//
+// SEE ALSO: module.h
+// AUTHOR: L.Komkov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#include "module.h"
+
+int fp_kallsyms_lookup_name = 0;
+module_param (fp_kallsyms_lookup_name, int, 0);
+MODULE_PARM_DESC (fp_kallsyms_lookup_name, "address of 'kallsyms_lookup_name' function");
+
+char gl_szDefaultDeviceName[128] = DEFAULT_DEVICE_NAME;
+char* device_name = NULL;
+module_param (device_name, charp, 0);
+MODULE_PARM_DESC (device_name, "device name for '/proc/devices'");
+
+unsigned int device_major = 0;
+module_param (device_major, uint, 0);
+MODULE_PARM_DESC (device_major, "default device major number");
+
+fp_kallsyms_lookup_name_t lookup_name;
+
+#if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
+void (*__real_put_task_struct) (struct task_struct * tsk);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11))
+#define SWAPDRV_PUT_TASK_STRUCT "put_task_struct"
+void
+put_task_struct (struct task_struct *tsk)
+{
+ __real_put_task_struct (tsk);
+}
+#else
+#define SWAPDRV_PUT_TASK_STRUCT "__put_task_struct"
+void
+__put_task_struct (struct task_struct *tsk)
+{
+ __real_put_task_struct (tsk);
+}
+#endif
+#else /*2.6.16 */
+void (*__real_put_task_struct) (struct rcu_head * rhp);
+#define SWAPDRV_PUT_TASK_STRUCT "__put_task_struct_cb"
+void
+__put_task_struct_cb (struct rcu_head *rhp)
+{
+ __real_put_task_struct (rhp);
+}
+#endif
+/*void (*__real_put_task_struct)(struct task_struct *tsk);
+void __put_task_struct(struct task_struct *tsk)
+{
+ __real_put_task_struct(tsk);
+}*/
+
+#if defined(CONFIG_MIPS)
+void (*flush_cache_page) (struct vm_area_struct * vma, unsigned long page);
+#endif
+
+static int __init InitializeModule(void)
+{
+ if(!fp_kallsyms_lookup_name) {
+ EPRINTF("fp_kallsyms_lookup_name parameter undefined!");
+ return -1;
+ }
+ if(device_name == NULL) {
+ EPRINTF("Using default device name!");
+ device_name = gl_szDefaultDeviceName;
+ }
+ if(device_major == 0) {
+ EPRINTF("Using default device major number!");
+ device_major = DEFAULT_DEVICE_MAJOR;
+ }
+
+ lookup_name = (fp_kallsyms_lookup_name_t) fp_kallsyms_lookup_name;
+ __real_put_task_struct = (void *) lookup_name (SWAPDRV_PUT_TASK_STRUCT);
+ if (!__real_put_task_struct)
+ {
+ EPRINTF (SWAPDRV_PUT_TASK_STRUCT " is not found! Oops. Where is it?");
+ return -ESRCH;
+ }
+
+#if defined(CONFIG_MIPS)
+ flush_cache_page = (void *) fp ("r4k_flush_cache_page");
+ if (!flush_cache_page)
+ {
+ EPRINTF ("failed to resolve 'flush_cache_page'!\n");
+ return -ESRCH;
+ }
+#endif
+
+ if(probes_manager_init() < 0) {
+ EPRINTF ("Cannot initialize probe manager!");
+ return -1;
+ }
+ if(device_init() < 0) {
+ EPRINTF ("Cannot initialize device!");
+ return -1;
+ }
+
+ INIT_LIST_HEAD(&cond_list.list);
+
+ DPRINTF ("is successfully initialized.");
+ return 0;
+}
+
+static void __exit UninitializeModule (void)
+{
+ ec_user_stop ();
+ device_down ();
+ probes_manager_down ();
+ DPRINTF ("is successfully finished.");
+}
+
+module_init (InitializeModule);
+module_exit (UninitializeModule);
+MODULE_LICENSE ("GPL");
+MODULE_AUTHOR("Adwanced Software Group (SRC, Moscow)");
+MODULE_DESCRIPTION("InPerfA Device Driver");
+MODULE_VERSION("4:1.0");
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: module.h
+//
+// DESCRIPTION:
+//
+// SEE ALSO: module.c
+// AUTHOR: L.Komkov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#if !defined(__MODULE_H__)
+#define __MODULE_H__
+
+#define UNUSED __attribute__((unused))
+
+#include <linux/types.h>
+#include <asm/current.h>
+#include <asm/page.h>
+#include <asm/uaccess.h>
+#include <asm/local.h>
+#include <asm/string.h>
+#include <asm/mman.h>
+
+#include <net/sock.h>
+#include <net/tcp.h>
+#include <net/inet_common.h>
+
+#include <linux/gfp.h>
+#include <linux/mm.h>
+#include <linux/page-flags.h>
+#include <linux/vmalloc.h>
+#include <linux/errno.h>
+#include <linux/spinlock.h>
+#include <linux/version.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/genhd.h>
+#include <linux/fs.h>
+#include <linux/bio.h>
+#include <linux/file.h>
+#include <linux/tcp.h>
+#include <linux/skbuff.h>
+#include <linux/ipsec.h>
+#include <linux/sysctl.h>
+#include <linux/dcache.h>
+#include <linux/kernel.h>
+#include <linux/stddef.h>
+#include <linux/audit.h>
+#include <linux/namei.h>
+#include <linux/signal.h>
+#include <linux/highmem.h>
+#include <linux/pagemap.h>
+#include <linux/types.h>
+#include <linux/cdev.h>
+#include <linux/jiffies.h>
+#include <linux/time.h>
+#include <linux/proc_fs.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 14))
+#ifndef __DISABLE_RELAYFS
+#define __DISABLE_RELAYFS
+#warning "RELAY FS was disabled since not supported!"
+#endif // __DISABLE_RELAYFS
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 14) & LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17))
+#include <linux/debugfs.h>
+#include <linux/relayfs_fs.h>
+#else
+#include <linux/debugfs.h>
+#include <linux/relay.h>
+#endif
+
+#include "events.h"
+
+#include "debug.h"
+#include "ec.h"
+#include "event.h"
+#include "legacy.h"
+#include "storage.h"
+#include "us_proc_inst.h"
+#include "device_driver.h"
+#include "probes_manager.h"
+#include "probes.h"
+#include "stdfunc.h"
+
+typedef unsigned long (*fp_kallsyms_lookup_name_t) (const char *name);
+extern int fp_kallsyms_lookup_name;
+extern fp_kallsyms_lookup_name_t lookup_name;
+extern char *device_name;
+extern unsigned int device_major;
+
+#endif /* !defined(module_h) */
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: PICL.h
+//
+// DESCRIPTION:
+//
+// SEE ALSO: PICL.cpp
+// AUTHOR: L.Komkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.02
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#if !defined(PICL_h)
+#define PICL_h
+
+#ifdef INPERFA_HOST
+#include <sys/time.h> // struct timeval
+#else // INPERFA_TARGET
+#ifdef __KERNEL__
+#include <linux/time.h> // struct timeval
+#else
+#include <sys/time.h> // struct timeval
+#endif
+#endif
+
+#define MAX_ARGS_DATA 200
+
+#define EVENT_MAX_SIZE 2048*5
+
+/* EvenRecord format
+ ______________________________________________________________________________________________________________
+| Len | RecType | EventType | Time | Pid | Tid | CPU | ArgsCount | ArgTypes | ArgValues | PreEventLen |
+ __4b_____4b________4b________8b_____4b____4b_____4b______4b________ArgCount___variable-length._______4b_____
+
+*/
+
+typedef enum
+{
+ RECORD_ENTRY = 0,
+ RECORD_RET,
+ RECORD_PROFILE,
+
+ RECORD_TYPE_COUNT // fictional record type used to count real record types
+} record_type_t;
+
+const char *ec_record_type_to_name (record_type_t record_type);
+int ec_record_type_by_name (record_type_t * p_record_type, const char *record_type_name);
+
+// old Picl.h
+
+#define _TRACE_TYPE 0
+#define _TRACE_STATS_TYPE 2
+
+typedef union tagArgValueType
+{
+ char m_char;
+ int m_int;
+ long m_long;
+ float m_float;
+ double m_double;
+ void *m_pointer;
+ char *m_string;
+} _ARG_VALUE_TYPE;
+
+/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <sys/types.h>
+#endif
+
+struct common_timeval {
+ u_int32_t tv_sec;
+ u_int32_t tv_usec;
+};
+
+typedef u_int32_t TYPEOF_EVENT_LENGTH;
+typedef u_int32_t TYPEOF_EVENT_TYPE;
+typedef u_int32_t TYPEOF_PROBE_ID;
+typedef struct common_timeval TYPEOF_TIME;
+typedef u_int32_t TYPEOF_PROCESS_ID;
+typedef u_int32_t TYPEOF_THREAD_ID;
+typedef u_int32_t TYPEOF_CPU_NUMBER;
+typedef u_int32_t TYPEOF_NUMBER_OF_ARGS;
+
+typedef struct tagEventHeader
+{
+ TYPEOF_EVENT_LENGTH m_nLength;
+ TYPEOF_EVENT_TYPE m_nType;
+ TYPEOF_PROBE_ID m_nProbeID;
+ TYPEOF_TIME m_time;
+ TYPEOF_PROCESS_ID m_nProcessID;
+ TYPEOF_THREAD_ID m_nThreadID;
+ TYPEOF_CPU_NUMBER m_nCPU;
+ TYPEOF_NUMBER_OF_ARGS m_nNumberOfArgs;
+} SWAP_TYPE_EVENT_HEADER;
+
+typedef struct tagEvent
+{
+ SWAP_TYPE_EVENT_HEADER *m_pHeader;
+ char *m_pDescriptor;
+ char *m_pData;
+} SWAP_TYPE_EVENT;
+
+/*
+ Argument Descriptors
+
+ c - char
+ h - short
+ d - int
+ x - long
+ p - pointer
+ f - float
+ w - double
+ s - string
+*/
+
+typedef enum
+{
+ AT_UNKNOWN,
+ AT_CHAR,
+ AT_SHORT,
+ AT_INT,
+ AT_LONG,
+ AT_PTR,
+ AT_FLOAT,
+ AT_DOUBLE,
+ AT_STRING,
+ AT_ARRAY
+} TYPE_ARGUMENT;
+
+#define ALIGN_VALUE(x) ((x) + (4 - ((x) % 4)))
+
+#endif /* !defined(PICL_h) */
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: probes.c
+//
+// DESCRIPTION:
+// This file is C source for SWAP driver.
+//
+// SEE ALSO: probes.h
+// AUTHOR: L.Komkov, S.Grekhov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#include "module.h"
+#include "probes.h"
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
+#define tcp_opt tcp_sock
+#endif
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20)
+#define kmem_cache_t struct kmem_cache
+#endif
+
+/*
+#define UNUSED __attribute__((unused))
+
+#define USER_PRIO(p) ((p)-MAX_RT_PRIO)
+#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))
+
+#define PRIO_BONUS_RATIO 25
+#define MAX_BONUS (MAX_USER_PRIO * PRIO_BONUS_RATIO / 100)
+#define INTERACTIVE_DELTA 2
+
+#define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20)
+#define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio)
+
+#define SCALE(v1,v1_max,v2_max) (v1) * (v2_max) / (v1_max)
+
+#define DELTA(p) (SCALE(TASK_NICE(p), 40, MAX_BONUS) + INTERACTIVE_DELTA)
+
+#define TASK_INTERACTIVE(p) ((p)->prio <= (p)->static_prio - DELTA(p))
+*/
+
+const char *ec_probe_name[] = {
+ "ks_probe_id",
+ "us_probe_id",
+ "vtp_probe_id",
+ "dyn_lib_probe_id",
+ "event_fmt_probe_id",
+ "rq_profile",
+ "pid_rq_profile"
+};
+
+//TODO: the same function should be used from utils.cpp
+int name2index (unsigned *p_index, unsigned count, const char **names, const char *name)
+{
+ unsigned index;
+ for (index = 0; index < count; ++index) {
+ if (!strcmp (names[index], name)) {
+ *p_index = index;
+ return 0;
+ }
+ }
+ return -EINVAL;
+}
+
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: probes.h
+//
+// DESCRIPTION:
+//
+// SEE ALSO: probes.c
+// AUTHOR: L.Komkov, S.Grekhov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#ifndef __PROBES_H__
+#define __PROBES_H__
+
+#include "ec_probe.h"
+#include "storage.h"
+#include "../kprobe/kprobes.h"
+
+#ifndef regs_return_value
+/* "regs_return_value" is ARCH-dependent. ARCH is passed via "EC_ARCH_*" */
+
+#ifdef EC_ARCH_arm
+/* ARCH == arm */
+#define regs_return_value(regs) ((regs)->ARM_r0)
+#endif /* def EC_ARCH_arm */
+
+#ifdef EC_ARCH_i386
+/* ARCH == i386 */
+//#include <linux/kprobes.h>
+#define regs_return_value(regs) ((regs)->ax)
+#endif /* def EC_ARCH_i386 */
+
+#ifdef EC_ARCH_mips
+/* ARCH == mips */
+#define regs_return_value(regs) ((regs)->regs[2])
+#endif /* def EC_ARCH_mips */
+
+#endif /* ndef regs_return_value */
+
+extern struct jprobe my_jprobe[];
+extern const char *ec_probe_name[];
+
+extern struct kretprobe my_kretprobe[];
+
+#define MY_JPROBE_ENTRY(handler_entry) { .entry = JPROBE_ENTRY(handler_entry) }
+
+/* Probe up to 20 instances concurrently. */
+#define MAXACTIVE 20
+
+#define MY_RETPROBE_HANDLER(handler_entry) { .handler = (handler_entry), .maxactive = MAXACTIVE }
+
+#define MY_UAPP(_ips_arr) { .path="", .m_f_dentry=NULL, \
+ .ips_count=sizeof(_ips_arr)/sizeof(us_proc_ip_t), .p_ips=_ips_arr, \
+ .vtps_count=0, .p_vtps=NULL, .loaded=0}
+#define MY_ULIB(_lib, _ips_arr) { .path=#_lib, .m_f_dentry=NULL, \
+ .ips_count=sizeof(_ips_arr)/sizeof(us_proc_ip_t), .p_ips=_ips_arr, \
+ .vtps_count=0, .p_vtps=NULL, .loaded=0}
+#define MY_UPROBE_ENTRY(_name, _entry_hand, _exit_hand) {.name = #_name, \
+ .jprobe.entry = JPROBE_ENTRY(_entry_hand), \
+ .retprobe.handler = (kretprobe_handler_t)_exit_hand}
+#define MY_UPROBE_ENTRY_EXT(_name, _pre_entry_hand, _entry_hand, _exit_hand) {.name = #_name, .jprobe.pre_entry = (kprobe_pre_entry_handler_t)_pre_entry_hand, .jprobe.entry = JPROBE_ENTRY(_entry_hand), .retprobe.handler = (kretprobe_handler_t)_exit_hand}
+
+#endif // __PROBES_H__
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: probes_manager.c
+//
+// DESCRIPTION:
+// This file is C source for SWAP driver.
+//
+// SEE ALSO: probes_manager.h
+// AUTHOR: L.Komkov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#include <linux/percpu.h>
+#include "module.h"
+#include "probes_manager.h"
+
+#ifdef EC_ARCH_arm
+/* ARCH == arm */
+#include "../kprobe/kprobes.h"
+#endif /* def EC_ARCH_arm */
+
+#ifdef EC_ARCH_i386
+/* ARCH == i386 */
+//#include <linux/kprobes.h>
+#include "../kprobe/kprobes.h"
+#endif /* def EC_ARCH_i386 */
+
+#ifdef EC_ARCH_mips
+/* ARCH == mips */
+#include "../kprobe/kprobes.h"
+#endif /* def EC_ARCH_mips */
+
+unsigned long pf_addr;
+unsigned long exit_addr;
+kernel_probe_t *pf_probe = NULL;
+kernel_probe_t *exit_probe = NULL;
+unsigned int probes_flags = 0;
+
+#ifdef MEMORY_CHECKER
+EXPORT_SYMBOL_GPL(pf_addr);
+#endif
+
+int
+probes_manager_init (void)
+{
+#ifdef CONFIG_X86
+ pf_addr = lookup_name("handle_mm_fault");
+#else
+ pf_addr = lookup_name("do_page_fault");
+#endif
+ if (pf_addr == 0) {
+ EPRINTF("Cannot find address for page fault function!");
+ return -EINVAL;
+ }
+
+ exit_addr = lookup_name("do_exit");
+ if (exit_addr == 0) {
+ EPRINTF("Cannot find address for do_exit function!");
+ return -EINVAL;
+ }
+
+ return storage_init ();
+}
+
+void
+probes_manager_down (void)
+{
+ detach_selected_probes ();
+ storage_down ();
+}
+
+static int
+register_kernel_jprobe (kernel_probe_t * probe)
+{
+ int result;
+ if (((probe == pf_probe) && (us_proc_probes & US_PROC_PF_INSTLD)) ||
+ ((probe == exit_probe) && (us_proc_probes & US_PROC_EXIT_INSTLD)))
+ {
+ return 0; // probe is already registered
+ }
+ result = register_jprobe (&probe->jprobe, 0);
+ if (result)
+ {
+ EPRINTF ("register_kernel_jprobe(0x%lx) failure %d", probe->addr, result);
+ return result;
+ }
+ return 0;
+}
+
+static int
+unregister_kernel_jprobe (kernel_probe_t * probe)
+{
+ if (((probe == pf_probe) && (us_proc_probes & US_PROC_PF_INSTLD)) ||
+ ((probe == exit_probe) && (us_proc_probes & US_PROC_EXIT_INSTLD)))
+ {
+ return 0; // probe is necessary for user space instrumentation
+ }
+ unregister_jprobe (&probe->jprobe, 0);
+ return 0;
+}
+
+static int
+register_kernel_retprobe (kernel_probe_t * probe)
+{
+ int result;
+ if (((probe == pf_probe) && (us_proc_probes & US_PROC_PF_INSTLD)) ||
+ ((probe == exit_probe) && (us_proc_probes & US_PROC_EXIT_INSTLD))/* ||
+ ((probe == fork_probe) && (us_proc_probes & US_PROC_FORK_INSTLD)) ||
+ ((probe == ss_probe) && (us_proc_probes & US_PROC_SS_INSTLD))*/)
+ {
+ return 0; // probe is already registered
+ }
+
+ result = register_kretprobe (&probe->retprobe, 0);
+ if (result)
+ {
+ EPRINTF ("register_kernel_retprobe(0x%lx) failure %d", probe->addr, result);
+ return result;
+ }
+ return 0;
+}
+
+static int
+unregister_kernel_retprobe (kernel_probe_t * probe)
+{
+ if (((probe == pf_probe) && (us_proc_probes & US_PROC_PF_INSTLD)) ||
+ ((probe == exit_probe) && (us_proc_probes & US_PROC_EXIT_INSTLD))/* ||
+ ((probe == fork_probe) && (us_proc_probes & US_PROC_FORK_INSTLD)) ||
+ ((probe == ss_probe) && (us_proc_probes & US_PROC_SS_INSTLD))*/)
+ {
+ return 0; // probe is necessary for user space instrumentation
+ }
+ unregister_kretprobe (&probe->retprobe, 0);
+ return 0;
+}
+
+int
+register_kernel_probe (kernel_probe_t * probe)
+{
+ register_kernel_jprobe (probe);
+ register_kernel_retprobe (probe);
+ return 0;
+}
+
+int
+unregister_kernel_probe (kernel_probe_t * probe)
+{
+ unregister_kernel_jprobe (probe);
+ unregister_kernel_retprobe (probe);
+ return 0;
+}
+
+int
+attach_selected_probes (void)
+{
+ int result = 0;
+ int partial_result = 0;
+ kernel_probe_t *p;
+ struct hlist_node *node;
+
+ hlist_for_each_entry_rcu (p, node, &kernel_probes, hlist)
+ {
+ partial_result = register_kernel_probe (p);
+ if (partial_result)
+ {
+ result = partial_result;
+ detach_selected_probes (); // return into safe state
+ break;
+ }
+ }
+
+ return result;
+}
+
+int
+detach_selected_probes (void)
+{
+ kernel_probe_t *p;
+ struct hlist_node *node;
+
+ hlist_for_each_entry_rcu (p, node, &kernel_probes, hlist)
+ unregister_kernel_probe (p);
+
+ return 0;
+}
+
+int
+add_probe (unsigned long addr)
+{
+ int result = 0;
+ kernel_probe_t **pprobe = NULL;
+
+ if (EC_STATE_IDLE != ec_info.ec_state)
+ {
+ EPRINTF("Probes addition is allowed in IDLE state only.");
+ return -EINVAL;
+ }
+
+ if (addr == pf_addr) {
+ probes_flags |= PROBE_FLAG_PF_INSTLD;
+ if (us_proc_probes & US_PROC_PF_INSTLD)
+ {
+ return 0;
+ }
+ pprobe = &pf_probe;
+ }
+ else if (addr == exit_addr) {
+ probes_flags |= PROBE_FLAG_EXIT_INSTLD;
+ if (us_proc_probes & US_PROC_EXIT_INSTLD)
+ {
+ return 0;
+ }
+ pprobe = &exit_probe;
+ }
+
+ result = add_probe_to_list (addr, pprobe);
+ if (result) {
+ if (addr == pf_addr)
+ probes_flags &= ~PROBE_FLAG_PF_INSTLD;
+ else if (addr == exit_addr)
+ probes_flags &= ~PROBE_FLAG_EXIT_INSTLD;
+ }
+ return result;
+}
+
+int reset_probes()
+{
+ struct hlist_node *node, *tnode;
+ kernel_probe_t *p;
+
+ hlist_for_each_entry_safe (p, node, tnode, &kernel_probes, hlist) {
+ if (p->addr == pf_addr) {
+ probes_flags &= ~PROBE_FLAG_PF_INSTLD;
+ pf_probe = NULL;
+ } else if (p->addr == exit_addr) {
+ probes_flags &= ~PROBE_FLAG_EXIT_INSTLD;
+ exit_probe = NULL;
+ }
+ hlist_del(node);
+ kfree(p);
+ }
+
+ return 0;
+}
+
+int
+remove_probe (unsigned long addr)
+{
+ int result = 0;
+
+ if (EC_STATE_IDLE != ec_info.ec_state)
+ {
+ EPRINTF("Probes addition is allowed in IDLE state only.");
+ return -EINVAL;
+ }
+
+ if (addr == pf_addr) {
+ probes_flags &= ~PROBE_FLAG_PF_INSTLD;
+ if (us_proc_probes & US_PROC_PF_INSTLD)
+ {
+ return 0;
+ }
+ pf_probe = NULL;
+ }
+ else if (addr == exit_addr) {
+ probes_flags &= ~PROBE_FLAG_EXIT_INSTLD;
+ if (us_proc_probes & US_PROC_EXIT_INSTLD)
+ {
+ return 0;
+ }
+ exit_probe = NULL;
+ }
+
+ result = remove_probe_from_list (addr);
+
+ return result;
+}
+
+DEFINE_PER_CPU (kernel_probe_t *, gpKernProbe) = NULL;
+EXPORT_PER_CPU_SYMBOL_GPL(gpKernProbe);
+DEFINE_PER_CPU(struct pt_regs *, gpKernRegs) = NULL;
+EXPORT_PER_CPU_SYMBOL_GPL(gpKernRegs);
+
+unsigned long
+def_jprobe_event_pre_handler (kernel_probe_t * probe, struct pt_regs *regs)
+{
+ __get_cpu_var (gpKernProbe) = probe;
+ __get_cpu_var (gpKernRegs) = regs;
+
+ return 0;
+}
+
+void
+def_jprobe_event_handler (unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5, unsigned long arg6)
+{
+ //static int nCount;
+ kernel_probe_t *probe = __get_cpu_var(gpKernProbe);
+ int skip = 0;
+
+ if (pf_probe == probe)
+ {
+ if (!(probes_flags & PROBE_FLAG_PF_INSTLD))
+ skip = 1;
+ }
+ else if (exit_probe == probe)
+ {
+ if (us_proc_probes & US_PROC_EXIT_INSTLD)
+ do_exit_probe_pre_code ();
+ if (!(probes_flags & PROBE_FLAG_EXIT_INSTLD))
+ skip = 1;
+ }
+
+ if (!skip)
+ pack_event_info (KS_PROBE_ID, RECORD_ENTRY, "pxxxxxx", probe->addr, arg1, arg2, arg3, arg4, arg5, arg6);
+ jprobe_return ();
+}
+
+int
+def_retprobe_event_handler (struct kretprobe_instance *pi, struct pt_regs *regs, kernel_probe_t * probe)
+{
+ int skip = 0;
+
+ if (pf_probe == probe)
+ {
+ if (us_proc_probes & US_PROC_PF_INSTLD)
+ do_page_fault_ret_pre_code ();
+ if (!(probes_flags & PROBE_FLAG_PF_INSTLD))
+ skip = 1;
+ }
+ else if (exit_probe == probe)
+ {
+ if (!(probes_flags & PROBE_FLAG_EXIT_INSTLD))
+ skip = 1;
+ }
+
+ if (!skip)
+ pack_event_info (KS_PROBE_ID, RECORD_RET, "p", probe->addr);
+ return 0;
+}
+
+/* This is a callback that is called by module 'inperfa_handlers'
+ * in order to register user defined handlers */
+void install_user_handlers(void)
+{
+ kernel_probe_t *probe;
+ struct hlist_node *node;
+ unsigned long pre_handler_addr, jp_handler_addr, rp_handler_addr;
+
+ /* We must perform this lookup whenever this function is called
+ * because the addresses of find_*_handler functions may differ. */
+ // MCPP inperfa_handlers removed
+ unsigned long (*find_jp_handler)(unsigned long) =
+ // MCPP inperfa_handlers removed
+ (unsigned long (*)(unsigned long))lookup_name("find_jp_handler");
+ unsigned long (*find_rp_handler)(unsigned long) =
+ (unsigned long (*)(unsigned long))lookup_name("find_rp_handler");
+ unsigned long (*find_pre_handler)(unsigned long) =
+ (unsigned long (*)(unsigned long))lookup_name("find_pre_handler");
+ hlist_for_each_entry_rcu (probe, node, &kernel_probes, hlist) {
+ if(find_pre_handler)
+ {
+ pre_handler_addr = find_pre_handler(probe->addr);
+ if (find_pre_handler != 0) {
+ DPRINTF("Added user pre handler for 0x%lx: 0x%lx",
+ probe->addr, find_pre_handler);
+ probe->jprobe.pre_entry = (kprobe_pre_entry_handler_t)pre_handler_addr;
+ }
+ }
+ jp_handler_addr = find_jp_handler(probe->addr);
+ if (jp_handler_addr != 0) {
+ DPRINTF("Added user jp handler for 0x%lx: 0x%lx",
+ probe->addr, jp_handler_addr);
+ probe->jprobe.entry = (kprobe_opcode_t *)jp_handler_addr;
+ }
+ rp_handler_addr = find_rp_handler(probe->addr);
+ if (rp_handler_addr != 0)
+ probe->retprobe.handler = (kretprobe_handler_t)rp_handler_addr;
+ }
+}
+EXPORT_SYMBOL_GPL(install_user_handlers);
+
+void uninstall_user_handlers(void)
+{
+ kernel_probe_t *probe;
+ struct hlist_node *node;
+
+ hlist_for_each_entry_rcu (probe, node, &kernel_probes, hlist) {
+ DPRINTF("Removed user jp handler for 0x%lx", probe->addr);
+ probe->jprobe.pre_entry = (kprobe_pre_entry_handler_t)def_jprobe_event_pre_handler;
+ probe->jprobe.entry = (kprobe_opcode_t *)def_jprobe_event_handler;
+ probe->retprobe.handler = (kretprobe_handler_t)def_retprobe_event_handler;
+ }
+}
+EXPORT_SYMBOL_GPL(uninstall_user_handlers);
+
+int is_pf_installed_by_user(void)
+{
+ return (probes_flags & PROBE_FLAG_PF_INSTLD) ? 1: 0;
+}
+EXPORT_SYMBOL_GPL(is_pf_installed_by_user);
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: probes_manager.h
+//
+// DESCRIPTION:
+//
+// SEE ALSO: probes_manager.c
+// AUTHOR: L.Komkov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#if !defined(__PROBES_MANAGER_H__)
+#define __PROBES_MANAGER_H__
+
+#include "ec_module.h"
+#include "ec_probe.h"
+#include "ec_module.h"
+#include "probes.h"
+
+typedef struct
+{
+ unsigned long addr;
+ struct jprobe jprobe;
+ struct kretprobe retprobe;
+ struct hlist_node hlist;
+} kernel_probe_t;
+
+extern int probes_manager_init (void);
+extern void probes_manager_down (void);
+
+extern int add_probe (unsigned long addr);
+extern int reset_probes(void);
+extern int remove_probe (unsigned long addr);
+
+extern int attach_selected_probes (void);
+extern int detach_selected_probes (void);
+
+extern int register_kernel_probe (kernel_probe_t * probe);
+extern int unregister_kernel_probe (kernel_probe_t * probe);
+
+extern unsigned long def_jprobe_event_pre_handler (kernel_probe_t * probe, struct pt_regs *regs);
+extern void def_jprobe_event_handler (unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5, unsigned long arg6);
+extern int def_retprobe_event_handler (struct kretprobe_instance *probe, struct pt_regs *regs, kernel_probe_t * p);
+void install_user_handlers(void);
+void uninstall_user_handlers(void);
+int is_pf_installed_by_user(void);
+
+extern unsigned long pf_addr;
+extern unsigned long exit_addr;
+extern kernel_probe_t *pf_probe;
+extern kernel_probe_t *exit_probe;
+extern unsigned int probes_flags;
+
+#define PROBE_FLAG_PF_INSTLD 0x1
+#define PROBE_FLAG_EXIT_INSTLD 0x2
+//#define PROBE_FLAG_SS_INSTLD 0x4
+//#define PROBE_FLAG_FORK_INSTLD 0x8
+
+#endif // !defined(__PROBES_MANAGER_H__)
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: storage.c
+//
+// DESCRIPTION:
+// This file is C source for SWAP.
+//
+// SEE ALSO: storage.h
+// AUTHOR: L.Komkov, S.Dianov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#include <linux/types.h>
+#include <linux/hash.h>
+#include <linux/list.h>
+#include <linux/unistd.h>
+#include "module.h"
+#include "storage.h"
+#include "CProfile.h"
+//#include "def_handlers.h"
+
+#define after_buffer ec_info.buffer_size
+
+
+char *p_buffer = NULL;
+inst_us_proc_t us_proc_info;
+struct hlist_head kernel_probes;
+int event_mask = 0L;
+struct cond cond_list;
+int paused = 0; /* a state after a stop condition (events are not collected) */
+struct timeval last_attach_time = {0, 0};
+
+#ifdef MEMORY_CHECKER
+#define M_BITS 6
+#define M_SIZE (1 << M_BITS)
+
+static struct hlist_head malloc_table[M_SIZE];
+//void *sys_call_table[];
+asmlinkage int (*orig_sys_mprotect)(unsigned long start, size_t len, unsigned long prot);
+
+typedef struct {
+ void * address;
+ size_t size;
+ struct hlist_node hlist;
+} mec_object_t;
+
+EXPORT_SYMBOL_GPL(us_proc_info);
+#endif
+
+unsigned copy_into_cyclic_buffer (char *buffer, unsigned dst_offset, char *src, unsigned size)
+{
+ unsigned nOffset = dst_offset;
+ char* pSource = src;
+ while (size--)
+ buffer[nOffset++] = *pSource++;
+ return nOffset;
+}
+
+unsigned copy_from_cyclic_buffer (char *dst, char *buffer, unsigned src_offset, unsigned size)
+{
+ unsigned nOffset = src_offset;
+ char* pDestination = dst;
+ while (size--)
+ *pDestination++ = buffer[nOffset++];
+ return nOffset;
+}
+
+int CheckBufferSize (unsigned int nSize)
+{
+ if (nSize < EC_BUFFER_SIZE_MIN) {
+ EPRINTF("Too small buffer size! [Size=%u KB]", nSize / 1024);
+ return -1;
+ }
+ if (nSize > EC_BUFFER_SIZE_MAX) {
+ EPRINTF("Too big buffer size! [Size=%u KB]", nSize / 1024);
+ return -1;
+ }
+ return 0;
+}
+
+int AllocateSingleBuffer(unsigned int nSize)
+{
+ unsigned long spinlock_flags = 0L;
+
+ unsigned int nSubbufferSize = ec_info.m_nSubbufSize;
+ unsigned int nNumOfSubbufers = GetNumOfSubbuffers(nSize);
+ unsigned long nAllocatedSize = nSubbufferSize * nNumOfSubbufers;
+
+ p_buffer = vmalloc_user(nAllocatedSize);
+ if(!p_buffer) {
+ EPRINTF("Memory allocation error! [Size=%lu KB]", nAllocatedSize / 1024);
+ return -1;
+ }
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ec_info.m_nNumOfSubbuffers = nNumOfSubbufers;
+ ec_info.buffer_effect = ec_info.buffer_size = nAllocatedSize;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ return 0;
+}
+
+void FreeSingleBuffer (void)
+{
+ VFREE_USER(p_buffer, ec_info.buffer_size);
+ CleanECInfo();
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+
+int EnableContinuousRetrieval() {
+ unsigned long spinlock_flags = 0L;
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ec_info.m_nMode |= MODEMASK_CONTINUOUS_RETRIEVAL;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ return 0;
+}
+
+int DisableContinuousRetrieval() {
+ unsigned long spinlock_flags = 0L;
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ec_info.m_nMode &= ~MODEMASK_CONTINUOUS_RETRIEVAL;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ return 0;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+
+#ifndef __DISABLE_RELAYFS
+
+struct rchan* gl_pRelayChannel = NULL;
+struct rchan* GetRelayChannel(void) { return gl_pRelayChannel; };
+
+struct dentry* gl_pdirRelay = NULL;
+struct dentry* GetRelayDir(void) { return gl_pdirRelay; };
+
+#ifdef __USE_PROCFS
+
+struct proc_dir_entry* alt_pde = NULL;
+
+static inline struct dentry *_dir_create (const char *dirname, struct dentry *parent, struct proc_dir_entry **p2pde)
+{
+ struct dentry *dir;
+ struct proc_dir_entry *pde;
+
+ pde = proc_mkdir (dirname, PDE (parent->d_inode));
+ if (pde == NULL)
+ {
+ dir = NULL;
+ }
+ else
+ {
+ mutex_lock (&parent->d_inode->i_mutex);
+ dir = lookup_one_len (dirname, parent, strlen (dirname));
+ mutex_unlock (&parent->d_inode->i_mutex);
+
+ if (IS_ERR (dir))
+ {
+ dir = NULL;
+ remove_proc_entry (dirname, PDE (parent->d_inode));
+ }
+
+ *p2pde = pde;
+ }
+
+ return dir;
+}
+
+static inline struct dentry *_get_proc_root (void)
+{
+ struct file_system_type *procfs_type;
+ struct super_block *procfs_sb;
+
+ procfs_type = get_fs_type ("proc");
+
+ if (!procfs_type || list_empty (&procfs_type->fs_supers))
+ return NULL;
+
+ procfs_sb = list_entry (procfs_type->fs_supers.next, \
+ struct super_block, s_instances);
+
+ return procfs_sb->s_root;
+
+}
+
+static struct dentry *create_buf (const char *filename, struct dentry *parent, int mode, struct rchan_buf *buf, int *is_global)
+{
+ struct proc_dir_entry *pde;
+ struct proc_dir_entry *parent_pde = NULL;
+ struct dentry *dentry;
+
+ if (parent)
+ parent_pde = PDE (parent->d_inode);
+ else
+ parent = _get_proc_root ();
+
+ pde = create_proc_entry (filename, S_IFREG|S_IRUSR, parent_pde);
+
+ if(unlikely(!pde))
+ return NULL;
+
+ pde->proc_fops = &relay_file_operations;
+
+ mutex_lock (&parent->d_inode->i_mutex);
+ dentry = lookup_one_len (filename, parent, strlen (filename));
+ mutex_unlock (&parent->d_inode->i_mutex);
+
+ if (IS_ERR(dentry)) {
+ remove_proc_entry (filename, parent_pde);
+ }
+
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18))
+ dentry->d_inode->i_private = buf;
+#else
+ dentry->d_inode->u.generic_ip = buf;
+#endif
+
+ return dentry;
+
+}
+
+static int remove_buf (struct dentry *dentry)
+{
+ if (dentry != NULL)
+ {
+ struct proc_dir_entry *pde = PDE (dentry->d_inode);
+ dput (dentry);
+ remove_proc_entry (pde->name, pde->parent);
+ }
+
+ return 0;
+}
+
+#endif // __USE_PROCFS
+ /*
+ * subbuf_start - called on buffer-switch to a new sub-buffer
+ * @buf: the channel buffer containing the new sub-buffer
+ * @subbuf: the start of the new sub-buffer
+ * @prev_subbuf: the start of the previous sub-buffer
+ * @prev_padding: unused space at the end of previous sub-buffer
+ *
+ * The client should return 1 to continue logging, 0 to stop
+ * logging.
+ *
+ * NOTE: subbuf_start will also be invoked when the buffer is
+ * created, so that the first sub-buffer can be initialized
+ * if necessary. In this case, prev_subbuf will be NULL.
+ *
+ * NOTE: the client can reserve bytes at the beginning of the new
+ * sub-buffer by calling subbuf_start_reserve() in this callback.
+ */
+int RelayCallbackSubbufStart(struct rchan_buf *buf,
+ void *subbuf,
+ void *prev_subbuf,
+ size_t prev_padding)
+{
+ struct rchan* pRelayChannel = NULL;
+ unsigned int nNumOfSubbufs = 0;
+
+ unsigned long spinlock_flags = 0L;
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+
+ subbuf_start_reserve(buf, RELAY_SUBBUF_HEADER_SIZE);
+ ec_info.buffer_effect += RELAY_SUBBUF_HEADER_SIZE;
+ ec_info.m_nEndOffset = RELAY_SUBBUF_HEADER_SIZE;
+
+ if(prev_subbuf == NULL) {
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+ return 1;
+ }
+ memcpy(prev_subbuf, &prev_padding, sizeof(unsigned int));
+ memcpy(prev_subbuf + sizeof(unsigned int), &ec_info.m_nSubbufSavedEvents, sizeof(unsigned int));
+ ec_info.m_nSubbufSavedEvents = 0;
+ pRelayChannel = GetRelayChannel();
+ if(pRelayChannel == NULL) {
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+ EPRINTF("Null pointer to relay channel!");
+ return 0;
+ }
+ nNumOfSubbufs = pRelayChannel->n_subbufs;
+ ec_info.m_nBeginSubbufNum = buf->subbufs_consumed % nNumOfSubbufs;
+ ec_info.m_nEndSubbufNum = buf->subbufs_produced % nNumOfSubbufs;
+ if(relay_buf_full(buf)) {
+ void* pConsume = NULL;
+ unsigned int nPaddingLength = 0;
+ unsigned int nSubbufSize = 0;
+ unsigned int nDataSize = 0;
+ unsigned int nEffectSize = 0;
+ unsigned int nSubbufDiscardedCount = 0;
+ nSubbufSize = pRelayChannel->subbuf_size;
+ pConsume = buf->start + buf->subbufs_consumed % nNumOfSubbufs * nSubbufSize;
+ memcpy(&nPaddingLength, pConsume, sizeof(unsigned int));
+ memcpy(&nSubbufDiscardedCount, pConsume + sizeof(unsigned int), sizeof(unsigned int));
+ nEffectSize = nSubbufSize - nPaddingLength;
+ nDataSize = nEffectSize - RELAY_SUBBUF_HEADER_SIZE;
+ ec_info.discarded_events_count += nSubbufDiscardedCount;
+ relay_subbufs_consumed(pRelayChannel, 0, 1);
+ ec_info.m_nBeginSubbufNum = buf->subbufs_consumed % nNumOfSubbufs;
+ ec_info.m_nEndSubbufNum = buf->subbufs_produced % nNumOfSubbufs;
+ ec_info.buffer_effect -= nEffectSize;
+ ec_info.trace_size -= nDataSize;
+ buf->dentry->d_inode->i_size = ec_info.trace_size;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+ return 1; // Overwrite mode
+ }
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+ return 1;
+}
+
+ /*
+ * buf_mapped - relay buffer mmap notification
+ * @buf: the channel buffer
+ * @filp: relay file pointer
+ *
+ * Called when a relay file is successfully mmapped
+ */
+void RelayCallbackBufMapped(struct rchan_buf *buf,
+ struct file *filp)
+{
+}
+
+ /*
+ * buf_unmapped - relay buffer unmap notification
+ * @buf: the channel buffer
+ * @filp: relay file pointer
+ *
+ * Called when a relay file is successfully unmapped
+ */
+void RelayCallbackBufUnmapped(struct rchan_buf *buf,
+ struct file *filp)
+{
+}
+ /*
+ * create_buf_file - create file to represent a relay channel buffer
+ * @filename: the name of the file to create
+ * @parent: the parent of the file to create
+ * @mode: the mode of the file to create
+ * @buf: the channel buffer
+ * @is_global: outparam - set non-zero if the buffer should be global
+ *
+ * Called during relay_open(), once for each per-cpu buffer,
+ * to allow the client to create a file to be used to
+ * represent the corresponding channel buffer. If the file is
+ * created outside of relay, the parent must also exist in
+ * that filesystem.
+ *
+ * The callback should return the dentry of the file created
+ * to represent the relay buffer.
+ *
+ * Setting the is_global outparam to a non-zero value will
+ * cause relay_open() to create a single global buffer rather
+ * than the default set of per-cpu buffers.
+ *
+ * See Documentation/filesystems/relayfs.txt for more info.
+ */
+struct dentry * RelayCallbackCreateBufFile(const char *filename,
+ struct dentry *parent,
+ int mode,
+ struct rchan_buf *buf,
+ int *is_global)
+{
+ *is_global = 1;
+#ifdef __USE_PROCFS
+ DPRINTF("\"%s\" is creating in procfs...!", filename);
+ return create_buf(filename, parent, mode, buf, is_global);
+#else
+ DPRINTF("\"%s\" is creating in debugfs...!", filename);
+ return debugfs_create_file(filename, (mode_t)mode, parent, buf, &relay_file_operations);
+#endif // __USE_PROCFS
+}
+
+ /*
+ * remove_buf_file - remove file representing a relay channel buffer
+ * @dentry: the dentry of the file to remove
+ *
+ * Called during relay_close(), once for each per-cpu buffer,
+ * to allow the client to remove a file used to represent a
+ * channel buffer.
+ *
+ * The callback should return 0 if successful, negative if not.
+ */
+int RelayCallbackRemoveBufFile(struct dentry *dentry)
+{
+#ifdef __USE_PROCFS
+ remove_buf(dentry);
+#else
+ debugfs_remove(dentry);
+#endif // __USE_PROCFS
+ return 0;
+}
+
+struct rchan_callbacks gl_RelayCallbacks = {
+ .subbuf_start = RelayCallbackSubbufStart,
+ .buf_mapped = RelayCallbackBufMapped,
+ .buf_unmapped = RelayCallbackBufUnmapped,
+ .create_buf_file = RelayCallbackCreateBufFile,
+ .remove_buf_file = RelayCallbackRemoveBufFile
+};
+#endif //__DISABLE_RELAYFS
+
+int AllocateMultipleBuffer(unsigned int nSize) {
+#ifndef __DISABLE_RELAYFS
+ unsigned long spinlock_flags = 0L;
+
+ unsigned int nSubbufferSize = ec_info.m_nSubbufSize;
+ unsigned int nNumOfSubbufers = GetNumOfSubbuffers(nSize);
+
+ gl_pRelayChannel = relay_open(DEFAULT_RELAY_BASE_FILENAME,
+ GetRelayDir(),
+ nSubbufferSize,
+ nNumOfSubbufers,
+ &gl_RelayCallbacks
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18))
+ ,NULL
+#endif
+ );
+ if(gl_pRelayChannel == NULL) {
+ EPRINTF("Cannot create relay buffer channel! [%d subbufers by %u Kb = %u Kb]",
+ nNumOfSubbufers, nSubbufferSize / 1024, nSize / 1024);
+ return -1;
+ }
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ec_info.m_nNumOfSubbuffers = nNumOfSubbufers;
+ ec_info.buffer_effect = ec_info.buffer_size = nSubbufferSize * nNumOfSubbufers;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ return 0;
+#else
+ EPRINTF("RelayFS not supported!");
+ return -1;
+#endif //__DISABLE_RELAYFS
+}
+
+void FreeMultipleBuffer(void) {
+#ifndef __DISABLE_RELAYFS
+ relay_close(gl_pRelayChannel);
+ CleanECInfo();
+#else
+ EPRINTF("RelayFS not supported!");
+#endif //__DISABLE_RELAYFS
+}
+
+int InitializeBuffer(unsigned int nSize) {
+ if(IsMultipleBuffer())
+ return AllocateMultipleBuffer(nSize);
+ return AllocateSingleBuffer(nSize);
+}
+
+int UninitializeBuffer(void) {
+ if(IsMultipleBuffer())
+ FreeMultipleBuffer();
+ FreeSingleBuffer();
+ return 0;
+}
+
+int EnableMultipleBuffer() {
+ unsigned long spinlock_flags = 0L;
+
+ if(IsMultipleBuffer())
+ return 0;
+
+ if(UninitializeBuffer() == -1)
+ EPRINTF("Cannot uninitialize buffer!");
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ec_info.m_nMode |= MODEMASK_MULTIPLE_BUFFER;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ if(InitializeBuffer(GetBufferSize()) == -1) {
+ EPRINTF("Cannot initialize buffer!");
+ return -1;
+ }
+ return 0;
+}
+
+int DisableMultipleBuffer() {
+ unsigned long spinlock_flags = 0L;
+
+ if(!IsMultipleBuffer())
+ return 0;
+
+ if(UninitializeBuffer() == -1)
+ EPRINTF("Cannot uninitialize buffer!");
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ec_info.m_nMode &= ~MODEMASK_MULTIPLE_BUFFER;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ if(InitializeBuffer(GetBufferSize()) == -1) {
+ EPRINTF("Cannot initialize buffer!");
+ return -1;
+ }
+ return 0;
+}
+
+unsigned int GetBufferSize(void) { return ec_info.buffer_size; };
+
+int SetBufferSize(unsigned int nSize) {
+ if (GetECState() != EC_STATE_IDLE) {
+ EPRINTF("Buffer changes are allowed in IDLE state only (%d)!", GetECState());
+ return -1;
+ }
+ if(GetBufferSize() == nSize)
+ return 0;
+ if(CheckBufferSize(nSize) == -1) {
+ EPRINTF("Invalid buffer size!");
+ return -1;
+ }
+ detach_selected_probes ();
+ if(UninitializeBuffer() == -1)
+ EPRINTF("Cannot uninitialize buffer!");
+ if(InitializeBuffer(nSize) == -1) {
+ EPRINTF("Cannot initialize buffer! [Size=%u KB]", nSize / 1024);
+ return -1;
+ }
+ return 0;
+}
+
+void ResetSingleBuffer(void) {
+}
+
+void ResetMultipleBuffer(void) {
+#ifndef __DISABLE_RELAYFS
+ relay_reset(gl_pRelayChannel);
+#else
+ EPRINTF("RelayFS not supported!");
+#endif //__DISABLE_RELAYFS
+}
+
+int ResetBuffer(void) {
+ unsigned long spinlock_flags = 0L;
+
+ if (GetECState() != EC_STATE_IDLE) {
+ EPRINTF("Buffer changes are allowed in IDLE state only!");
+ return -1;
+ }
+
+ if(IsMultipleBuffer())
+ ResetMultipleBuffer();
+ else
+ ResetSingleBuffer();
+
+ detach_selected_probes ();
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ec_info.buffer_effect = ec_info.buffer_size;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ ResetECInfo();
+
+ return 0;
+}
+
+int WriteEventIntoSingleBuffer(char* pEvent, unsigned long nEventSize) {
+ unsigned long spinlock_flags = 0L;
+ int bCopied = 0;
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ if(!p_buffer) {
+ EPRINTF("Invalid pointer to buffer!");
+ ++ec_info.lost_events_count;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+ return -1;
+ }
+ while (!bCopied) {
+ unsigned unused_space;
+ if(ec_info.saved_events_count == 0 || ec_info.after_last > ec_info.first) {
+ unused_space = ec_info.buffer_size - ec_info.after_last;
+ if(unused_space > nEventSize) {
+ ec_info.after_last = copy_into_cyclic_buffer (p_buffer, ec_info.after_last, pEvent, nEventSize);
+ ++ec_info.saved_events_count;
+ bCopied = 1;
+ ec_info.buffer_effect = ec_info.buffer_size;
+ ec_info.trace_size = ec_info.after_last - ec_info.first;
+ } else {
+ bCopied = 0;
+ ec_info.buffer_effect = ec_info.after_last;
+ ec_info.after_last = 0;
+ }
+ } else {
+ unused_space = ec_info.first - ec_info.after_last;
+ if(unused_space > nEventSize) {
+ ec_info.after_last = copy_into_cyclic_buffer (p_buffer, ec_info.after_last, pEvent, nEventSize);
+ ++ec_info.saved_events_count;
+ bCopied = 1;
+ ec_info.trace_size = ec_info.buffer_effect - ec_info.first + ec_info.after_last;
+ } else {
+ if(ec_info.first < ec_info.buffer_effect) {
+ TYPEOF_EVENT_LENGTH discard_len = 0;
+ copy_from_cyclic_buffer ((char *) &discard_len, p_buffer, ec_info.first, sizeof (TYPEOF_EVENT_LENGTH));
+ ec_info.first = ec_info.first + discard_len;
+ ++ec_info.discarded_events_count;
+ bCopied = 0;
+ ec_info.trace_size = ec_info.buffer_effect - ec_info.first + ec_info.after_last;
+ } else {
+ bCopied = 0;
+ ec_info.buffer_effect = ec_info.buffer_size;
+ ec_info.first = 0;
+ }
+ }
+ }
+ }
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+ return 0;
+}
+
+int WriteEventIntoMultipleBuffer(char* pEvent, unsigned long nEventSize) {
+#ifndef __DISABLE_RELAYFS
+ unsigned long spinlock_flags = 0L;
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ __relay_write(GetRelayChannel(), pEvent, nEventSize);
+ ec_info.buffer_effect += nEventSize;
+ ec_info.trace_size += nEventSize;
+ ec_info.saved_events_count++;
+ ec_info.m_nEndOffset += nEventSize;
+ ec_info.m_nSubbufSavedEvents++;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+ return 0;
+#else
+ EPRINTF("RelayFS not supported!");
+ return -1;
+#endif //__DISABLE_RELAYFS
+}
+
+int WriteEventIntoBuffer(char* pEvent, unsigned long nEventSize) {
+
+ /*unsigned long i;
+ for(i = 0; i < nEventSize; i++)
+ printk("%02X ", pEvent[i]);
+ printk("\n");*/
+
+ if(IsMultipleBuffer())
+ return WriteEventIntoMultipleBuffer(pEvent, nEventSize);
+ return WriteEventIntoSingleBuffer(pEvent, nEventSize);
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////
+
+int set_event_mask (int new_mask)
+{
+ unsigned long spinlock_flags = 0L;
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ event_mask = new_mask;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+ return 0;
+}
+
+int
+get_event_mask (int *mask)
+{
+ *mask = event_mask;
+ return 0;
+}
+
+static void
+generic_swap (void *a, void *b, int size)
+{
+ char t;
+ do {
+ t = *(char *) a;
+ *(char *) a++ = *(char *) b;
+ *(char *) b++ = t;
+ } while (--size > 0);
+}
+
+static void sort (void *base, size_t num, size_t size, int (*cmp) (const void *, const void *), void (*fswap) (void *, void *, int size))
+{
+ /* pre-scale counters for performance */
+ int i = (num / 2) * size, n = num * size, c, r;
+
+ /* heapify */
+ for (; i >= 0; i -= size)
+ {
+ for (r = i; r * 2 < n; r = c)
+ {
+ c = r * 2;
+ if (c < n - size && cmp (base + c, base + c + size) < 0)
+ c += size;
+ if (cmp (base + r, base + c) >= 0)
+ break;
+ fswap (base + r, base + c, size);
+ }
+ }
+
+ /* sort */
+ for (i = n - size; i >= 0; i -= size)
+ {
+ fswap (base, base + i, size);
+ for (r = 0; r * 2 < i; r = c)
+ {
+ c = r * 2;
+ if (c < i - size && cmp (base + c, base + c + size) < 0)
+ c += size;
+ if (cmp (base + r, base + c) >= 0)
+ break;
+ fswap (base + r, base + c, size);
+ }
+ }
+}
+
+static int addr_cmp (const void *a, const void *b)
+{
+ return *(unsigned long *) a > *(unsigned long *) b ? -1 : 1;
+}
+
+int set_us_proc_inst_info (ioctl_inst_usr_space_proc_t * inst_info)
+{
+ int i, k, j, l, is_app = 0;
+ long len;
+ us_proc_lib_t *d_lib, *pd_lib;
+ ioctl_usr_space_lib_t s_lib;
+ us_proc_ip_t *d_ip;
+ ioctl_usr_space_ip_t s_ip;
+ us_proc_vtp_t *mvtp;
+ ioctl_usr_space_vtp_t s_vtp;
+ struct nameidata nd;
+ char *ptr;
+
+ inst_us_proc_t *my_uprobes_info = (inst_us_proc_t *)lookup_name("my_uprobes_info");
+ inst_us_proc_t empty_uprobes_info =
+ {
+ .libs_count = 0,
+ .p_libs = NULL,
+ };
+ if (my_uprobes_info == 0)
+ my_uprobes_info = &empty_uprobes_info;
+
+ if (us_proc_info.path)
+ release_us_proc_inst_info ();
+
+ memset ((void *) &us_proc_info, 0, sizeof (us_proc_info));
+ len = strlen_user (inst_info->path);
+ us_proc_info.path = kmalloc (len, GFP_KERNEL);
+ DPRINTF ("us_proc_info.path=%p", us_proc_info.path);
+ if (!us_proc_info.path)
+ return -ENOMEM;
+ if (strncpy_from_user (us_proc_info.path, inst_info->path, len) != (len-1))
+ {
+ EPRINTF ("strncpy_from_user app path failed %p (%ld)", us_proc_info.path, len);
+ return -EFAULT;
+ }
+
+ if (path_lookup (us_proc_info.path, LOOKUP_FOLLOW, &nd) != 0)
+ {
+ EPRINTF ("failed to lookup dentry for path %s!", us_proc_info.path);
+ return -EFAULT;
+ }
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)
+ us_proc_info.m_f_dentry = nd.dentry;
+ path_release (&nd);
+#else
+ us_proc_info.m_f_dentry = nd.path.dentry;
+ path_put (&nd.path);
+#endif
+
+ us_proc_info.libs_count = inst_info->libs_count;
+ us_proc_info.p_libs = kmalloc (us_proc_info.libs_count * sizeof (us_proc_lib_t), GFP_KERNEL);
+ DPRINTF ("us_proc_info.p_libs=%p/%u", us_proc_info.p_libs, us_proc_info.libs_count);
+ if (!us_proc_info.p_libs)
+ return -ENOMEM;
+ memset (us_proc_info.p_libs, 0, us_proc_info.libs_count * sizeof (us_proc_lib_t));
+ for (i = 0; i < us_proc_info.libs_count; i++)
+ {
+ d_lib = &us_proc_info.p_libs[i];
+ if (copy_from_user ((void *) &s_lib, &inst_info->p_libs[i], sizeof (ioctl_usr_space_lib_t)))
+ {
+ EPRINTF ("copy_from_user lib failed %p", &inst_info->p_libs[i]);
+ return -EFAULT;
+ }
+
+ len = strlen_user (s_lib.path);
+ d_lib->path = kmalloc(len, GFP_KERNEL);
+ DPRINTF ("d_lib[%i]->path=%p", i, d_lib->path);
+ if (!d_lib->path)
+ {
+ return -ENOMEM;
+ }
+ if (strncpy_from_user (d_lib->path, s_lib.path, len) != (len-1))
+ {
+ EPRINTF ("strncpy_from_user lib path failed %p (%ld)", d_lib->path, len);
+ return -EFAULT;
+ }
+ if (path_lookup (d_lib->path, LOOKUP_FOLLOW, &nd) != 0)
+ {
+ EPRINTF ("failed to lookup dentry for path %s!", d_lib->path);
+ continue;
+ }
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)
+ d_lib->m_f_dentry = nd.dentry;
+ path_release (&nd);
+#else
+ d_lib->m_f_dentry = nd.path.dentry;
+ path_put (&nd.path);
+#endif
+
+ if(strcmp(inst_info->path, d_lib->path) == 0)
+ is_app = 1;
+ else
+ is_app = 0;
+
+ pd_lib = NULL;
+ ptr = strrchr(d_lib->path, '/');
+ if(ptr) ptr++;
+ else ptr = d_lib->path;
+ for(l = 0; l < my_uprobes_info->libs_count; l++){
+ //DPRINTF("LIB %s/%s", ptr, my_uprobes_info->p_libs[l].path);
+ if((strcmp(ptr, my_uprobes_info->p_libs[l].path) == 0) ||
+ (is_app && *(my_uprobes_info->p_libs[l].path) == '\0')){
+ pd_lib = &my_uprobes_info->p_libs[l];
+ //DPRINTF("predef lib %s", pd_lib->path);
+ break;
+ }
+ }
+
+ if (s_lib.ips_count > 0)
+ {
+ us_proc_info.unres_ips_count += s_lib.ips_count;
+ d_lib->ips_count = s_lib.ips_count;
+ d_lib->p_ips = vmalloc (d_lib->ips_count * sizeof (us_proc_ip_t));
+ DPRINTF ("d_lib[%i]->p_ips=%p/%u [%s]", i, d_lib->p_ips, us_proc_info.unres_ips_count, d_lib->path);
+ if (!d_lib->p_ips)
+ {
+ return -ENOMEM;
+ }
+ memset (d_lib->p_ips, 0, d_lib->ips_count * sizeof (us_proc_ip_t));
+ for (k = 0; k < d_lib->ips_count; k++)
+ {
+ d_ip = &d_lib->p_ips[k];
+ if (copy_from_user ((void *) &s_ip, &s_lib.p_ips[k], sizeof (ioctl_usr_space_ip_t)))
+ {
+ EPRINTF ("copy_from_user ip failed %p", &s_lib.p_ips[k]);
+ return -EFAULT;
+ }
+ d_ip->offset = s_ip.addr;
+
+ /* TNCHK */
+ /*d_ip->jprobe.pre_entry = ujprobe_event_pre_handler;
+ d_ip->jprobe.entry = ujprobe_event_handler;
+ d_ip->retprobe.handler = uretprobe_event_handler;*/
+ /* TNCHK */
+
+ if(pd_lib){
+ for(l = 0; l < pd_lib->ips_count; l++){
+ if(d_ip->offset == pd_lib->p_ips[l].offset){
+ DPRINTF("predefined probe %s in %s at %lx", pd_lib->p_ips[l].name, pd_lib->path, pd_lib->p_ips[l].offset);
+ d_ip->jprobe.pre_entry = pd_lib->p_ips[l].jprobe.pre_entry;
+ d_ip->jprobe.entry = pd_lib->p_ips[l].jprobe.entry;
+ d_ip->retprobe.handler = pd_lib->p_ips[l].retprobe.handler;
+ break;
+ }
+ }
+ }
+ }
+ }
+ if (s_lib.vtps_count > 0)
+ {
+ unsigned long ucount = 1, pre_addr;
+ // array containing elements like (addr, index)
+ unsigned long *addrs = kmalloc (s_lib.vtps_count * 2 * sizeof (unsigned long), GFP_KERNEL);
+// DPRINTF ("addrs=%p/%u", addrs, s_lib.vtps_count);
+ if (!addrs)
+ {
+ //note: storage will released next time or at clean-up moment
+ return -ENOMEM;
+ }
+ memset (addrs, 0, s_lib.vtps_count * 2 * sizeof (unsigned long));
+ // fill the array in
+ for (k = 0; k < s_lib.vtps_count; k++)
+ {
+ if (copy_from_user ((void *) &s_vtp, &s_lib.p_vtps[k], sizeof (ioctl_usr_space_vtp_t)))
+ {
+ //note: storage will released next time or at clean-up moment
+ EPRINTF ("copy_from_user VTP failed %p", &s_lib.p_vtps[k]);
+ kfree (addrs);
+ return -EFAULT;
+ }
+ addrs[2 * k] = s_vtp.addr;
+ addrs[2 * k + 1] = k;
+ }
+ // sort by VTP addresses, i.e. make VTPs with the same addresses adjacent;
+ // organize them into bundles
+ sort (addrs, s_lib.vtps_count, 2 * sizeof (unsigned long), addr_cmp, generic_swap);
+
+ // calc number of VTPs with unique addresses
+ for (k = 1, pre_addr = addrs[0]; k < s_lib.vtps_count; k++)
+ {
+ if (addrs[2 * k] != pre_addr)
+ ucount++; // count different only
+ pre_addr = addrs[2 * k];
+ }
+ us_proc_info.unres_vtps_count += ucount;
+ d_lib->vtps_count = ucount;
+ d_lib->p_vtps = kmalloc (ucount * sizeof (us_proc_vtp_t), GFP_KERNEL);
+ DPRINTF ("d_lib[%i]->p_vtps=%p/%lu", i, d_lib->p_vtps, ucount); //, d_lib->path);
+ if (!d_lib->p_vtps)
+ {
+ //note: storage will released next time or at clean-up moment
+ kfree (addrs);
+ return -ENOMEM;
+ }
+ memset (d_lib->p_vtps, 0, d_lib->vtps_count * sizeof (us_proc_vtp_t));
+ // go through sorted VTPS.
+ for (k = 0, j = 0, pre_addr = 0, mvtp = NULL; k < s_lib.vtps_count; k++)
+ {
+ us_proc_vtp_data_t *vtp_data;
+ // copy VTP data
+ if (copy_from_user ((void *) &s_vtp, &s_lib.p_vtps[addrs[2 * k + 1]], sizeof (ioctl_usr_space_vtp_t)))
+ {
+ //note: storage will released next time or at clean-up moment
+ EPRINTF ("copy_from_user VTP failed %p", &s_lib.p_vtps[addrs[2 * k + 1]]);
+ kfree (addrs);
+ return -EFAULT;
+ }
+ // if this is the first VTP in bundle (master VTP)
+ if (addrs[2 * k] != pre_addr)
+ {
+ // data are in the array of master VTPs
+ mvtp = &d_lib->p_vtps[j++];
+ mvtp->addr = s_vtp.addr;
+ INIT_LIST_HEAD (&mvtp->list);
+ }
+ // data are in the list of slave VTPs
+ vtp_data = kmalloc (sizeof (us_proc_vtp_data_t), GFP_KERNEL);
+ if (!vtp_data)
+ {
+ //note: storage will released next time or at clean-up moment
+ kfree (addrs);
+ return -ENOMEM;
+ }
+
+ /*len = strlen_user (s_vtp.name);
+ vtp_data->name = kmalloc (len, GFP_KERNEL);
+ if (!vtp_data->name)
+ {
+ //note: storage will released next time or at clean-up moment
+ kfree (vtp_data);
+ kfree (addrs);
+ return -ENOMEM;
+ }
+ if (strncpy_from_user (vtp_data->name, s_vtp.name, len) != (len-1))
+ {
+ //note: storage will released next time or at clean-up moment
+ EPRINTF ("strncpy_from_user VTP name failed %p (%ld)", vtp_data->name, len);
+ kfree (vtp_data->name);
+ kfree (vtp_data);
+ kfree (addrs);
+ return -EFAULT;
+ }
+ //vtp_data->name[len] = 0;*/
+ vtp_data->type = s_vtp.type;
+ vtp_data->size = s_vtp.size;
+ vtp_data->reg = s_vtp.reg;
+ vtp_data->off = s_vtp.off;
+ list_add_tail_rcu (&vtp_data->list, &mvtp->list);
+ pre_addr = addrs[2 * k];
+ }
+ kfree (addrs);
+ }
+ }
+
+ //us_proc_info.mapped_codelets = (kprobe_opcode_t *) - 1;
+
+ return 0;
+}
+
+#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 27)
+#define list_for_each_rcu(pos, head) __list_for_each_rcu(pos, head)
+#endif
+
+// release storage of user space process instrumentation info
+void release_us_proc_inst_info (void)
+{
+ int i, k;
+ us_proc_lib_t *d_lib;
+ char *path;
+ struct list_head *pos; //, *tmp;
+
+ path = us_proc_info.path;
+ us_proc_info.path = 0;
+
+ // first make sure "d_lib" is not used any more and only
+ // then release storage
+ if (us_proc_info.p_libs)
+ {
+ int count1 = us_proc_info.libs_count;
+ us_proc_info.libs_count = 0;
+ for (i = 0; i < count1; i++)
+ {
+ d_lib = &us_proc_info.p_libs[i];
+ if (d_lib->p_ips)
+ {
+ // first make sure "d_lib->p_ips" is not used any more and only
+ // then release storage
+ //int count2 = d_lib->ips_count;
+ d_lib->ips_count = 0;
+ /*for (k = 0; k < count2; k++)
+ kfree ((void *) d_lib->p_ips[k].name);*/
+ vfree ((void *) d_lib->p_ips);
+ }
+ if (d_lib->p_vtps)
+ {
+ // first make sure "d_lib->p_vtps" is not used any more and only
+ // then release storage
+ int count2 = d_lib->vtps_count;
+ d_lib->vtps_count = 0;
+ for (k = 0; k < count2; k++)
+ {
+ //list_for_each_safe_rcu(pos, tmp, &d_lib->p_vtps[k].list) {
+ list_for_each_rcu (pos, &d_lib->p_vtps[k].list)
+ {
+ us_proc_vtp_data_t *vtp = list_entry (pos, us_proc_vtp_data_t, list);
+ list_del_rcu (pos);
+ //kfree (vtp->name);
+ kfree (vtp);
+ }
+ }
+ kfree ((void *) d_lib->p_vtps);
+ }
+ kfree ((void *) d_lib->path);
+ //putname(d_lib->path);
+ }
+ kfree ((void *) us_proc_info.p_libs);
+ us_proc_info.p_libs = 0;
+ }
+ if (path)
+ {
+ kfree ((void *) path);
+ //putname(path);
+ }
+
+ us_proc_info.tgid = 0;
+}
+
+
+//++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
+int storage_init (void)
+{
+ unsigned long spinlock_flags = 0L;
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ec_info.m_nMode = 0; // MASK IS CLEAR (SINGLE NON_CONTINUOUS BUFFER)
+// ec_info.m_nMode |= ECMODEMASK_MULTIPLE_BUFFER;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+#ifndef __DISABLE_RELAYFS
+
+#ifdef __USE_PROCFS
+ gl_pdirRelay = _dir_create (DEFAULT_RELAY_BASE_DIR, _get_proc_root(), &alt_pde);
+ if(gl_pdirRelay == NULL) {
+ EPRINTF("Cannot create procfs directory for relay buffer!");
+ return -1;
+ }
+#else
+ gl_pdirRelay = debugfs_create_dir(DEFAULT_RELAY_BASE_DIR, NULL);
+ if(gl_pdirRelay == NULL) {
+ EPRINTF("Cannot create directory for relay buffer!");
+ return -1;
+ }
+
+#endif // __USE_PROCFS
+
+#endif //__DISABLE_RELAYFS
+
+ if(InitializeBuffer(EC_BUFFER_SIZE_DEFAULT) == -1) {
+ EPRINTF("Cannot initialize buffer! [Size=%u KB]", EC_BUFFER_SIZE_DEFAULT / 1024 );
+ return -1;
+ }
+
+ INIT_HLIST_HEAD (&kernel_probes);
+
+#ifdef MEMORY_CHECKER
+ {
+ void **sys_call_table_ptr = (void **)lookup_name("sys_call_table");
+ if(!sys_call_table_ptr){
+ EPRINTF("Cannot find sys_call_table in kernel!");
+ return -1;
+ }
+ orig_sys_mprotect = sys_call_table_ptr[__NR_mprotect];
+
+ int i;
+ for(i = 0; i < M_SIZE; i++)
+ {
+ INIT_HLIST_HEAD(&malloc_table[i]);
+ }
+ }
+#endif
+
+ return 0;
+}
+
+/*
+ Shuts down "storage".
+ Assumes that all probes are already deactivated.
+*/
+void storage_down (void)
+{
+ if(UninitializeBuffer() == -1)
+ EPRINTF("Cannot uninitialize buffer!");
+
+#ifndef __DISABLE_RELAYFS
+
+#ifdef __USE_PROCFS
+// remove_buf(gl_pdirRelay);
+#else
+ debugfs_remove(gl_pdirRelay);
+#endif // __USE_PROCFS
+
+#endif //__DISABLE_RELAYFS
+
+ release_us_proc_inst_info ();
+
+ if (ec_info.collision_count)
+ EPRINTF ("ec_info.collision_count=%d", ec_info.collision_count);
+ if (ec_info.lost_events_count)
+ EPRINTF ("ec_info.lost_events_count=%d", ec_info.lost_events_count);
+}
+
+static u_int32_t get_probe_func_addr(const char *fmt, va_list args)
+{
+ if (fmt[0] != 'p')
+ return 0;
+
+ return va_arg(args, u_int32_t);
+}
+
+/*
+ Note: Experiment shows that on x86 jprobe and retprobe are handled with
+ interrupts disabled. Thus collision is possible only due to hardware
+ exception happening inside our code or due to parallel execution on 2 or
+ more physical CPUs. Experiment shows that on x86 with single CPU collisions
+ do not happen.
+
+ mk: We have dual situation. SMP (non-UP) Spinlock will help against
+ cross-CPU collisions but will lock-up even single CPU in case of hardware
+ exception like pagefault.
+
+ Per-CPU counter will allow us to catch nesting for the same CPU.
+ "pack_event_info()" is a preemption-safe, thus we can use "__get_cpu_var()"
+ directly.
+
+ NOTE: real_per_cpu_in_use_count = per_cpu_in_use_count + 1
+*/
+static DEFINE_PER_CPU (local_t, per_cpu_in_use_count) = LOCAL_INIT (-1);
+
+void pack_event_info (probe_id_t probe_id, record_type_t record_type, const char *fmt, ...)
+{
+ unsigned long spinlock_flags = 0L;
+ static char buf[EVENT_MAX_SIZE] = "";
+ //char* probe_name = NULL;
+ TYPEOF_EVENT_LENGTH event_len = 0L;
+ TYPEOF_TIME tv = { 0, 0 };
+ TYPEOF_THREAD_ID current_pid = current->pid;
+ TYPEOF_PROCESS_ID current_tgid = current->tgid;
+ unsigned current_cpu = task_cpu(current);
+ va_list args;
+ unsigned long addr = 0;
+ struct cond *p_cond;
+ struct event_tmpl *p_tmpl;
+
+ memset(buf, 0, EVENT_MAX_SIZE);
+ do_gettimeofday (&tv);
+
+ if (probe_id == KS_PROBE_ID) {
+ va_start(args, fmt);
+ addr = get_probe_func_addr(fmt, args);
+ va_end(args);
+ if (!find_probe(addr))
+ return;
+ if (((addr == pf_addr) && !(probes_flags & PROBE_FLAG_PF_INSTLD)) ||
+ ((addr == exit_addr) && !(probes_flags & PROBE_FLAG_EXIT_INSTLD)))
+ return;
+ }
+ if (probe_id == US_PROBE_ID) {
+ va_start(args, fmt);
+ addr = get_probe_func_addr(fmt, args);
+ va_end(args);
+ }
+
+ /* Checking for all the conditions
+ * except stop condition that we process after saving the event */
+ list_for_each_entry(p_cond, &cond_list.list, list) {
+ p_tmpl = &p_cond->tmpl;
+ switch (p_tmpl->type) {
+ case ET_TYPE_START_COND:
+ if ((!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_ADDR) ||
+ (addr == p_tmpl->addr)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_PID) ||
+ (current_tgid == p_tmpl->pid)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_TID) ||
+ (current_pid == p_tmpl->tid)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_CPU_NUM) ||
+ (current_cpu == p_tmpl->cpu_num)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_BIN_NAME) ||
+ (strcmp(current->comm, p_tmpl->bin_name) == 0)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_TIME) ||
+ (tv.tv_sec > last_attach_time.tv_sec + p_tmpl->sec) ||
+ (tv.tv_sec == last_attach_time.tv_sec + p_tmpl->sec &&
+ tv.tv_usec >= last_attach_time.tv_usec + p_tmpl->usec)) &&
+ !p_cond->applied) {
+ paused = 0;
+ p_cond->applied = 1;
+ }
+ break;
+ case ET_TYPE_IGNORE_COND:
+ /* if (probe_id == PROBE_SCHEDULE) */
+ /* break; */
+ if ((!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_ADDR) ||
+ (addr == p_tmpl->addr)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_PID) ||
+ (current_tgid == p_tmpl->pid)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_TID) ||
+ (current_pid == p_tmpl->tid)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_CPU_NUM) ||
+ (current_cpu == p_tmpl->cpu_num)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_BIN_NAME) ||
+ (strcmp(current->comm, p_tmpl->bin_name) == 0))) {
+ ec_info.ignored_events_count++;
+ return;
+ }
+ break;
+ }
+ }
+
+ if (paused && probe_id != EVENT_FMT_PROBE_ID) {
+ ec_info.ignored_events_count++;
+ return;
+ }
+
+ // We are in a preemption-safe context (because interrupts are disabled on
+ // local CPU), thus we can use "__get_cpu_var()" directly.
+ local_inc (&__get_cpu_var (per_cpu_in_use_count));
+ if (0 != local_read (&__get_cpu_var (per_cpu_in_use_count))) {
+ // nested entry for the same CPU means page fault or something
+ // similarly terrific in our code
+ local_dec (&__get_cpu_var (per_cpu_in_use_count)); // roll back
+ EPRINTF ("nested %s() for the same CPU", __FUNCTION__);
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ++ec_info.collision_count;
+ ++ec_info.lost_events_count;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ return; // bail out
+ }
+
+ /*
+ The same CPU can't get here second time thus we can safely use spinlock
+ to separate multiple CPUs accessing our data.
+ */
+ va_start (args, fmt);
+ event_len = VPackEvent(buf, sizeof(buf), event_mask, probe_id, record_type, &tv,
+ current_tgid, current_pid, current_cpu, fmt, args);
+ va_end (args);
+
+ if(event_len == 0) {
+ EPRINTF ("ERROR: failed to pack event!");
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ++ec_info.lost_events_count;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ } else if(WriteEventIntoBuffer(buf, event_len) == -1) {
+ EPRINTF("Cannot write event into buffer!");
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ++ec_info.lost_events_count;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+ }
+
+ local_dec (&__get_cpu_var (per_cpu_in_use_count)); // roll back
+
+ /* Check for stop condition. We pause collecting the trace right after
+ * storing this event */
+ list_for_each_entry(p_cond, &cond_list.list, list) {
+ p_tmpl = &p_cond->tmpl;
+ switch (p_tmpl->type) {
+ case ET_TYPE_STOP_COND:
+ if ((!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_ADDR) ||
+ (addr == p_tmpl->addr)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_PID) ||
+ (current_tgid == p_tmpl->pid)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_TID) ||
+ (current_pid == p_tmpl->tid)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_CPU_NUM) ||
+ (current_cpu == p_tmpl->cpu_num)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_BIN_NAME) ||
+ (strcmp(current->comm, p_tmpl->bin_name) == 0)) &&
+ (!ET_FIELD_ISSET(p_tmpl->flags, ET_MATCH_TIME) ||
+ (tv.tv_sec > last_attach_time.tv_sec + p_tmpl->sec) ||
+ (tv.tv_sec == last_attach_time.tv_sec + p_tmpl->sec &&
+ tv.tv_usec >= last_attach_time.tv_usec + p_tmpl->usec)) &&
+ !p_cond->applied) {
+ paused = 1;
+ p_cond->applied = 1;
+ }
+ break;
+ }
+ }
+}
+EXPORT_SYMBOL_GPL(pack_event_info);
+
+kernel_probe_t* find_probe (unsigned long addr)
+{
+ kernel_probe_t *p;
+ struct hlist_node *node;
+
+ //check if such probe does exist
+ hlist_for_each_entry_rcu (p, node, &kernel_probes, hlist)
+ if (p->addr == addr)
+ break;
+
+ return node ? p : NULL;
+}
+
+int add_probe_to_list (unsigned long addr, kernel_probe_t ** pprobe)
+{
+ kernel_probe_t *new_probe;
+ unsigned long jp_handler_addr, rp_handler_addr, pre_handler_addr;
+ unsigned long (*find_jp_handler)(unsigned long) =
+ (unsigned long (*)(unsigned long))lookup_name("find_jp_handler");
+ unsigned long (*find_rp_handler)(unsigned long) =
+ (unsigned long (*)(unsigned long))lookup_name("find_rp_handler");
+ unsigned long (*find_pre_handler)(unsigned long) =
+ (unsigned long (*)(unsigned long))lookup_name("find_pre_handler");
+
+ if (pprobe)
+ *pprobe = NULL;
+ //check if such probe does already exist
+ if (find_probe (addr)) {
+ /* It is not a problem if we have already registered
+ this probe before */
+ return 0;
+ }
+
+ new_probe = kmalloc (sizeof (kernel_probe_t), GFP_KERNEL);
+ if (!new_probe)
+ {
+ EPRINTF ("no memory for new probe!");
+ return -ENOMEM;
+ }
+ memset (new_probe, 0, sizeof (kernel_probe_t));
+
+ new_probe->addr = addr;
+ new_probe->jprobe.kp.addr = new_probe->retprobe.kp.addr = (kprobe_opcode_t *)addr;
+ new_probe->jprobe.priv_arg = new_probe->retprobe.priv_arg = new_probe;
+
+ //new_probe->jprobe.pre_entry = (kprobe_pre_entry_handler_t) def_jprobe_event_pre_handler;
+ if (find_pre_handler == 0 ||
+ (pre_handler_addr = find_pre_handler(new_probe->addr)) == 0)
+ new_probe->jprobe.pre_entry = (kprobe_pre_entry_handler_t) def_jprobe_event_pre_handler;
+ else
+ new_probe->jprobe.pre_entry = (kprobe_pre_entry_handler_t)pre_handler_addr;
+
+ if (find_jp_handler == 0 ||
+ (jp_handler_addr = find_jp_handler(new_probe->addr)) == 0)
+ new_probe->jprobe.entry = (kprobe_opcode_t *) def_jprobe_event_handler;
+ else
+ new_probe->jprobe.entry = (kprobe_opcode_t *)jp_handler_addr;
+
+ if (find_rp_handler == 0 ||
+ (rp_handler_addr = find_rp_handler(new_probe->addr)) == 0)
+ new_probe->retprobe.handler =
+ (kretprobe_handler_t) def_retprobe_event_handler;
+ else
+ new_probe->retprobe.handler = (kretprobe_handler_t)rp_handler_addr;
+
+ INIT_HLIST_NODE (&new_probe->hlist);
+ hlist_add_head_rcu (&new_probe->hlist, &kernel_probes);
+
+ if (pprobe)
+ *pprobe = new_probe;
+
+ return 0;
+}
+
+int remove_probe_from_list (unsigned long addr)
+{
+ kernel_probe_t *p;
+
+ //check if such probe does exist
+ p = find_probe (addr);
+ if (!p) {
+ /* We do not care about it. Nothing bad. */
+ return 0;
+ }
+
+ hlist_del_rcu (&p->hlist);
+
+ kfree (p);
+
+ return 0;
+}
+
+#ifdef MEMORY_CHECKER
+
+// active (unprotected) memory object
+static mec_object_t *curr_mec_obj;
+// protects 'curr_mec_obj'
+static spinlock_t mec_spinlock = SPIN_LOCK_UNLOCKED;
+
+void *mec_find_in_object_range(void *addr, int *rz)
+{
+ mec_object_t *obj = NULL;
+ struct hlist_node *node = NULL;
+ struct hlist_head *head = &malloc_table[0];//hash_ptr(addr, M_BITS)];
+ void *obj_zone_start, *obj_zone_end;
+ unsigned long spinlock_flags = 0;
+
+ *rz = 0;
+ //DPRINTF("mec_find_in_object_range %p", addr);
+ hlist_for_each_entry(obj, node, head, hlist)
+ {
+ // hit into another object
+ if((addr >= obj->address) && (addr < (obj->address + obj->size)))
+ {
+ //DPRINTF("mec_find_in_object_range found %lx, %u", obj->address, obj->size);
+ return obj;
+ }
+ obj_zone_start = (void *)((unsigned long)(obj->address) & ~(PAGE_SIZE-1));
+ obj_zone_end = (char *)obj_zone_start + PAGE_ALIGN(obj->size);
+ // hit into red zone of another object
+ if(((addr >= obj_zone_start) && (addr < obj->address)) ||
+ ((addr >= (obj->address+obj->size)) && (addr < obj_zone_end)))
+ {
+ *rz = 1;
+ //DPRINTF("mec_find_in_object_range %lx found in red zone of %lx, %u. [%p..%p]",
+ // addr, obj->address, obj->size, obj_zone_start, obj_zone_end);
+ return obj;
+ }
+ }
+ spin_lock_irqsave (&mec_spinlock, spinlock_flags);
+ if(curr_mec_obj)
+ {
+ // hit into unallocated zone near active object but within max underflow/overflow zone
+ obj_zone_start = (void *)((unsigned long)(curr_mec_obj->address) & ~(PAGE_SIZE-1));
+ obj_zone_end = (char *)obj_zone_start + PAGE_ALIGN(curr_mec_obj->size);
+ spin_unlock_irqrestore (&mec_spinlock, spinlock_flags);
+ if((addr < obj_zone_start) && (addr >= (obj_zone_start - MEC_MAX_OVERFLOW_SIZE)))
+ {
+ *rz = 1;
+ //DPRINTF("mec_find_in_object_range %lx found in unalloc pre-zone of %lx, %u. [%p..%p]",
+ // addr, curr_mec_obj->address, curr_mec_obj->size, obj_zone_start, obj_zone_end);
+ return curr_mec_obj;
+ }
+ if((addr >= obj_zone_end) && (addr < (obj_zone_end + MEC_MAX_OVERFLOW_SIZE)))
+ {
+ *rz = 1;
+ //DPRINTF("mec_find_in_object_range %lx found in unalloc post-zone of %lx, %u. [%p..%p]",
+ // addr, curr_mec_obj->address, curr_mec_obj->size, obj_zone_start, obj_zone_end);
+ return curr_mec_obj;
+ }
+ }
+ else
+ spin_unlock_irqrestore (&mec_spinlock, spinlock_flags);
+
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(mec_find_in_object_range);
+
+int mec_mprotect(unsigned long start, size_t len, unsigned long prot)
+{
+ //DPRINTF("mec_mprotect %lx, %u, %lx", start, len, prot);
+ return orig_sys_mprotect(start, len, prot);
+}
+
+void mec_change_active_object(void *obj)
+{
+ mec_object_t *new_obj = (mec_object_t *)obj;
+ unsigned long sz, addr, spinlock_flags = 0;
+
+ spin_lock_irqsave (&mec_spinlock, spinlock_flags);
+
+ if(curr_mec_obj != NULL)
+ {
+ // protect old object
+ //DPRINTF("mec_change_active_object protect old obj %lx, %u", curr_obj->address, curr_obj->size);
+ addr = (unsigned long)(curr_mec_obj->address) & ~(PAGE_SIZE-1);
+ sz = PAGE_ALIGN((unsigned long)(curr_mec_obj->address) + curr_mec_obj->size) - addr;
+ mec_mprotect(addr, sz, PROT_NONE);
+ }
+
+ // remove protection from new object
+ //DPRINTF("mec_change_active_object unprotect new obj %lx, %u", new_obj->address, new_obj->size);
+ addr = (unsigned long)(new_obj->address) & ~(PAGE_SIZE-1);
+ sz = PAGE_ALIGN((unsigned long)(new_obj->address) + new_obj->size) - addr;
+ mec_mprotect(addr, sz, PROT_READ|PROT_WRITE);
+
+ curr_mec_obj = new_obj;
+
+ spin_unlock_irqrestore (&mec_spinlock, spinlock_flags);
+}
+EXPORT_SYMBOL_GPL(mec_change_active_object);
+
+void *mec_find_object(void *addr)
+{
+ mec_object_t *obj = NULL;
+ struct hlist_node *node = NULL;
+ struct hlist_head *head = &malloc_table[0];//hash_ptr(addr, M_BITS)];
+
+ hlist_for_each_entry(obj, node, head, hlist)
+ {
+ if(obj->address == addr)
+ return obj;
+ }
+
+ return NULL;
+}
+
+void *mec_add_object(void *addr, unsigned long size)
+{
+ mec_object_t *new;
+
+ if (mec_find_object(addr))
+ return NULL;
+
+ DPRINTF("Add object %p/%lu!", addr, size);
+ new = (mec_object_t *)kmalloc(sizeof(mec_object_t), GFP_KERNEL);
+ if(new != NULL)
+ {
+ new->address = addr;
+ new->size = size;
+ INIT_HLIST_NODE (&new->hlist);
+ hlist_add_head(&new->hlist, &malloc_table[0]);//hash_ptr(addr, M_BITS)]);
+ }
+
+ return new;
+}
+
+void mec_delete_object(void *addr)
+{
+ struct hlist_head *head = &malloc_table[0];//hash_ptr(addr, M_BITS)];
+ unsigned long spinlock_flags = 0;
+
+ mec_object_t *obj = (mec_object_t *)mec_find_object(addr);
+ if(!obj)
+ return;
+
+ DPRINTF("Delete object %p!", addr);
+ hlist_del(&obj->hlist);
+ if(hlist_empty(head))
+ {
+ spin_lock_irqsave (&mec_spinlock, spinlock_flags);
+ curr_mec_obj = NULL;
+ spin_unlock_irqrestore (&mec_spinlock, spinlock_flags);
+ }
+ kfree(obj);
+}
+
+void mec_remove_objects()
+{
+ mec_object_t *obj = NULL;
+ struct hlist_node *node = NULL, *tmp;
+ struct hlist_head *head = &malloc_table[0];//hash_ptr(addr, M_BITS)];
+ unsigned long spinlock_flags = 0;
+
+ spin_lock_irqsave (&mec_spinlock, spinlock_flags);
+ curr_mec_obj = NULL;
+ spin_unlock_irqrestore (&mec_spinlock, spinlock_flags);
+
+ hlist_for_each_entry_safe(obj, node, tmp, head, hlist)
+ {
+ hlist_del(&obj->hlist);
+ kfree(obj);
+ }
+}
+
+void mec_resize_object(void *hnd, unsigned long size)
+{
+ mec_object_t *obj = (mec_object_t *)hnd;
+
+ DPRINTF("Resize object %p/%u->%lu!", obj->address, obj->size, size);
+ obj->size = size;
+}
+
+void mec_post_event(char *data, unsigned long len)
+{
+ SWAP_TYPE_EVENT_HEADER *pEventHeader = (SWAP_TYPE_EVENT_HEADER *)data;
+ void **oaddr;
+ // point to the first arg
+ char *arg = (char *)&(pEventHeader->m_nNumberOfArgs) + sizeof(TYPEOF_NUMBER_OF_ARGS);
+ arg += ALIGN_VALUE(strlen(arg) + 1);
+ //func_addr, ret_addr, frame_ptr, new_addr, new_size, [old_addr]
+ oaddr = (void **)(arg + 3*sizeof(void *));
+ if((pEventHeader->m_nProbeID >= MEC_ALLOC_PROBE_ID_MIN) && (pEventHeader->m_nProbeID <= MEC_ALLOC_PROBE_ID_MAX))
+ {
+ if(pEventHeader->m_nType == RECORD_RET)
+ {
+ unsigned long *osize = (unsigned long *)(arg + 4*sizeof(void *));
+ if((pEventHeader->m_nProbeID == MEC_REALLOC_PROBE_ID) || (pEventHeader->m_nProbeID == MEC_MREMAP_PROBE_ID))
+ {
+ void **addr = (void *)(arg + 4*sizeof(void *) + sizeof(unsigned long));
+ if(*oaddr == *addr)
+ {
+ void *obj = mec_find_object(*addr);
+ if(obj)
+ mec_resize_object(obj, *osize);
+ else if(!mec_add_object(*oaddr, *osize)) // normally should not get here
+ EPRINTF("Failed to add new object %p/%lu!", *oaddr, *osize);
+ }
+ else
+ {
+ mec_delete_object(*addr); // delete old
+ if(!mec_add_object(*oaddr, *osize)) // add new
+ EPRINTF("Failed to add new object %p/%lu!", *oaddr, *osize);
+ }
+ }
+ else if(!mec_add_object(*oaddr, *osize))
+ EPRINTF("Failed to add new object %p/%lu!", *oaddr, *osize);
+ }
+ }
+ else if((pEventHeader->m_nProbeID >= MEC_DEALLOC_PROBE_ID_MIN) && (pEventHeader->m_nProbeID <= MEC_DEALLOC_PROBE_ID_MAX))
+ {
+ if(pEventHeader->m_nType == RECORD_ENTRY)
+ mec_delete_object(*oaddr);
+ }
+ pEventHeader->m_nProbeID = US_PROBE_ID;
+}
+#endif
+
+int put_us_event (char *data, unsigned long len)
+{
+ unsigned long spinlock_flags = 0L;
+
+ SWAP_TYPE_EVENT_HEADER *pEventHeader = (SWAP_TYPE_EVENT_HEADER *)data;
+ char *cur = data + sizeof(TYPEOF_EVENT_LENGTH) + sizeof(TYPEOF_EVENT_TYPE)
+ + sizeof(TYPEOF_PROBE_ID);
+ TYPEOF_NUMBER_OF_ARGS nArgs = pEventHeader->m_nNumberOfArgs;
+ TYPEOF_PROBE_ID probe_id = pEventHeader->m_nProbeID;
+ //int i;
+
+ /*if(probe_id == US_PROBE_ID){
+ printk("esrc %p/%d[", data, len);
+ for(i = 0; i < len; i++)
+ printk("%02x ", data[i]);
+ printk("]\n");
+ }*/
+
+ // set pid/tid/cpu/time i
+ //pEventHeader->m_time.tv_sec = tv.tv_sec;
+ //pEventHeader->m_time.tv_usec = tv.tv_usec;
+
+#ifdef MEMORY_CHECKER
+ //TODO: move this part to special MEC event posting routine, new IOCTL is needed
+ if((probe_id >= MEC_PROBE_ID_MIN) && (probe_id <= MEC_PROBE_ID_MAX))
+ mec_post_event(data, len);
+#endif
+
+ if((probe_id == EVENT_FMT_PROBE_ID) || !(event_mask & IOCTL_EMASK_TIME)){
+ TYPEOF_TIME tv = { 0, 0 };
+ do_gettimeofday (&tv);
+ memcpy(cur, &tv, sizeof(TYPEOF_TIME));
+ cur += sizeof(TYPEOF_TIME);
+ }
+ //pEventHeader->m_nProcessID = current_tgid;
+ if((probe_id == EVENT_FMT_PROBE_ID) || !(event_mask & IOCTL_EMASK_PID)){
+ //TYPEOF_PROCESS_ID current_tgid = current->tgid;
+ (*(TYPEOF_PROCESS_ID *)cur) = current->tgid;
+ cur += sizeof(TYPEOF_PROCESS_ID);
+ }
+ //pEventHeader->m_nThreadID = current_pid;
+ if((probe_id == EVENT_FMT_PROBE_ID) || !(event_mask & IOCTL_EMASK_TID)){
+ //TYPEOF_THREAD_ID current_pid = current->pid;
+ (*(TYPEOF_THREAD_ID *)cur) = current->pid;
+ cur += sizeof(TYPEOF_THREAD_ID);
+ }
+ //pEventHeader->m_nCPU = current_cpu;
+ if((probe_id == EVENT_FMT_PROBE_ID) || !(event_mask & IOCTL_EMASK_CPU)){
+ //TYPEOF_CPU_NUMBER current_cpu = task_cpu(current);
+ (*(TYPEOF_CPU_NUMBER *)cur) = task_cpu(current);
+ cur += sizeof(TYPEOF_CPU_NUMBER);
+ }
+ //printk("%d %x", probe_id, event_mask);
+ // dyn lib event should have all args, it is for internal use and not visible to user
+ if((probe_id == EVENT_FMT_PROBE_ID) || (probe_id == DYN_LIB_PROBE_ID) || !(event_mask & IOCTL_EMASK_ARGS)){
+ // move only if any of prev fields has been skipped
+ if(event_mask & (IOCTL_EMASK_TIME|IOCTL_EMASK_PID|IOCTL_EMASK_TID|IOCTL_EMASK_CPU)){
+ memmove(cur, data+sizeof(SWAP_TYPE_EVENT_HEADER)-sizeof(TYPEOF_NUMBER_OF_ARGS),
+ len-sizeof(SWAP_TYPE_EVENT_HEADER)+sizeof(TYPEOF_NUMBER_OF_ARGS)
+ -sizeof(TYPEOF_EVENT_LENGTH));
+ }
+ cur += len-sizeof(SWAP_TYPE_EVENT_HEADER)+sizeof(TYPEOF_NUMBER_OF_ARGS)
+ -sizeof(TYPEOF_EVENT_LENGTH);
+ }
+ else{
+ // user space probes should have at least one argument to identify them
+ if((probe_id == US_PROBE_ID) || (probe_id == VTP_PROBE_ID)){
+ char *pArg1;
+ (*(TYPEOF_NUMBER_OF_ARGS *)cur) = 1;
+ cur += sizeof(TYPEOF_NUMBER_OF_ARGS);
+ // pack args using format string for the 1st arg only
+ memset(cur, 0, ALIGN_VALUE(2));
+ cur[0] = 'p'; cur[1] = '\0';
+ cur += ALIGN_VALUE(2);
+ pArg1 = data + sizeof(SWAP_TYPE_EVENT_HEADER)+ALIGN_VALUE(nArgs+1);
+ memmove(cur, pArg1, sizeof(unsigned long));
+ cur += sizeof(unsigned long);
+ }
+ else {
+ (*(TYPEOF_NUMBER_OF_ARGS *)cur) = 0;
+ cur += sizeof(TYPEOF_NUMBER_OF_ARGS);
+ }
+ }
+ pEventHeader->m_nLength = cur - data + sizeof(TYPEOF_EVENT_LENGTH);
+ *((TYPEOF_EVENT_LENGTH *)cur) = pEventHeader->m_nLength;
+ len = pEventHeader->m_nLength;
+
+ /*if(probe_id == US_PROBE_ID){
+ printk("edst[");
+ for(i = 0; i < len; i++)
+ printk("%02x ", data[i]);
+ printk("]\n");
+ }*/
+
+ // We are in a preemption-safe context (because interrupts are disabled on
+ // local CPU), thus we can use "__get_cpu_var()" directly.
+ local_inc (&__get_cpu_var (per_cpu_in_use_count));
+ if (0 != local_read (&__get_cpu_var (per_cpu_in_use_count)))
+ {
+ // nested entry for the same CPU means page fault or something
+ // similarly terrific in our code
+ local_dec (&__get_cpu_var (per_cpu_in_use_count)); // roll back
+ EPRINTF ("nested %s() for the same CPU", __FUNCTION__);
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ++ec_info.collision_count;
+ ++ec_info.lost_events_count;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+
+ return -1; // bail out
+ }
+
+ /*
+ The same CPU can't get here second time thus we can safely use spinlock
+ to separate multiple CPUs accessing our data.
+ */
+ if(WriteEventIntoBuffer(data, len) == -1) {
+ EPRINTF("Cannot write event into buffer!");
+
+ spin_lock_irqsave (&ec_spinlock, spinlock_flags);
+ ++ec_info.lost_events_count;
+ spin_unlock_irqrestore (&ec_spinlock, spinlock_flags);
+ }
+
+ local_dec (&__get_cpu_var (per_cpu_in_use_count)); // roll back
+
+ return 0;
+}
+
+int set_predef_uprobes (ioctl_predef_uprobes_info_t *data)
+{
+ int i, k, size = 0, probe_size, result, j;
+ char *buf, *sep1, *sep2;
+
+ inst_us_proc_t *my_uprobes_info = (inst_us_proc_t *)lookup_name("my_uprobes_info");
+ inst_us_proc_t empty_uprobes_info =
+ {
+ .libs_count = 0,
+ .p_libs = NULL,
+ };
+ if (my_uprobes_info == 0)
+ my_uprobes_info = &empty_uprobes_info;
+
+ for(j = 0; j < data->probes_count; j++){
+ probe_size = strlen_user(data->p_probes+size);
+ buf = kmalloc(probe_size, GFP_KERNEL);
+ if(!buf){
+ EPRINTF("failed to alloc mem!");
+ return -EFAULT;
+ }
+ result = strncpy_from_user(buf, data->p_probes+size, probe_size);
+ if (result != (probe_size-1))
+ {
+ EPRINTF("failed to copy from user!");
+ kfree(buf);
+ return -EFAULT;
+ }
+ //DPRINTF("%s", buf);
+ sep1 = strchr(buf, ':');
+ if(!sep1){
+ EPRINTF("skipping invalid predefined uprobe string '%s'!", buf);
+ kfree(buf);
+ size += probe_size;
+ continue;
+ }
+ sep2 = strchr(sep1+1, ':');
+ if(!sep2 || (sep2 == sep1) || (sep2+2 == buf+probe_size)){
+ EPRINTF("skipping invalid predefined uprobe string '%s'!", buf);
+ kfree(buf);
+ size += probe_size;
+ continue;
+ }
+ for(i = 0; i < my_uprobes_info->libs_count; i++){
+ if(strncmp(buf, my_uprobes_info->p_libs[i].path, sep1-buf) != 0)
+ continue;
+ for(k = 0; k < my_uprobes_info->p_libs[i].ips_count; k++){
+ if(strncmp(sep1+1, my_uprobes_info->p_libs[i].p_ips[k].name, sep2-sep1-1) != 0)
+ continue;
+ my_uprobes_info->p_libs[i].p_ips[k].offset = simple_strtoul(sep2+1, NULL, 16);
+ }
+ }
+ kfree(buf);
+ size += probe_size;
+ }
+ return 0;
+}
+
+int get_predef_uprobes_size(int *size)
+{
+ int i, k;
+
+ inst_us_proc_t *my_uprobes_info = (inst_us_proc_t *)lookup_name("my_uprobes_info");
+ inst_us_proc_t empty_uprobes_info =
+ {
+ .libs_count = 0,
+ .p_libs = NULL,
+ };
+
+ if (my_uprobes_info == 0)
+ my_uprobes_info = &empty_uprobes_info;
+
+ *size = 0;
+ for(i = 0; i < my_uprobes_info->libs_count; i++){
+ int lib_size = strlen(my_uprobes_info->p_libs[i].path);
+ for(k = 0; k < my_uprobes_info->p_libs[i].ips_count; k++){
+ // libc.so.6:printf:
+ *size += lib_size + 1 + strlen(my_uprobes_info->p_libs[i].p_ips[k].name) + 2;
+ }
+ }
+ return 0;
+}
+
+int get_predef_uprobes(ioctl_predef_uprobes_info_t *udata)
+{
+ ioctl_predef_uprobes_info_t data;
+ int i, k, size, lib_size, func_size, result;
+ unsigned count = 0;
+ char sep[] = ":";
+
+ inst_us_proc_t *my_uprobes_info = (inst_us_proc_t *)lookup_name("my_uprobes_info");
+ inst_us_proc_t empty_uprobes_info =
+ {
+ .libs_count = 0,
+ .p_libs = NULL,
+ };
+ if (my_uprobes_info == 0)
+ my_uprobes_info = &empty_uprobes_info;
+
+ // get addr of array
+ if (copy_from_user ((void *)&data, udata, sizeof (data)))
+ {
+ EPRINTF("failed to copy from user!");
+ return -EFAULT;
+ }
+
+ size = 0;
+ for(i = 0; i < my_uprobes_info->libs_count; i++){
+ lib_size = strlen(my_uprobes_info->p_libs[i].path);
+ for(k = 0; k < my_uprobes_info->p_libs[i].ips_count; k++){
+ // libname
+ result = copy_to_user ((void *)(data.p_probes+size), my_uprobes_info->p_libs[i].path, lib_size);
+ if (result)
+ {
+ EPRINTF("failed to copy to user!");
+ return -EFAULT;
+ }
+ size += lib_size;
+ // ":"
+ result = copy_to_user ((void *)(data.p_probes+size), sep, 1);
+ if (result)
+ {
+ EPRINTF("failed to copy to user!");
+ return -EFAULT;
+ }
+ size++;
+ // probename
+ //DPRINTF("'%s'", my_uprobes_info->p_libs[i].p_ips[k].name);
+ func_size = strlen(my_uprobes_info->p_libs[i].p_ips[k].name);
+ result = copy_to_user ((void *)(data.p_probes+size), my_uprobes_info->p_libs[i].p_ips[k].name, func_size);
+ if (result)
+ {
+ EPRINTF("failed to copy to user!");
+ return -EFAULT;
+ }
+ size += func_size;
+ // ":\0"
+ result = copy_to_user ((void *)(data.p_probes+size), sep, 2);
+ if (result)
+ {
+ EPRINTF("failed to copy to user!");
+ return -EFAULT;
+ }
+ size += 2;
+ count++;
+ }
+ }
+
+ // set probes_count
+ result = copy_to_user ((void *)&(udata->probes_count), &count, sizeof(count));
+ if (result)
+ {
+ EPRINTF("failed to copy to user!");
+ return -EFAULT;
+ }
+ return 0;
+}
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: storage.h
+//
+// DESCRIPTION:
+// This file is C source for SWAP.
+//
+// SEE ALSO: storage.c
+// AUTHOR: L.Komkov, S.Dianov, A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.02.15
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#if !defined(__STORAGE_H__)
+#define __STORAGE_H__
+
+#include "picl.h"
+#include "ec_ioctl.h"
+#include "ec_probe.h"
+#include "ec_module.h"
+#include "probes_manager.h"
+#include "probes.h"
+#include "event_tmpl.h"
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+#ifndef __DISABLE_RELAYFS
+extern struct rchan* GetRelayChannel(void);
+extern struct dentry* GetRelayDir(void);
+#endif //__DISABLE_RELAYFS
+
+extern int EnableMultipleBuffer(void);
+extern int DisableMultipleBuffer(void);
+extern int EnableContinuousRetrieval(void);
+extern int DisableContinuousRetrieval(void);
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+extern unsigned int GetBufferSize(void);
+extern int SetBufferSize(unsigned int nSize);
+extern int ResetBuffer(void);
+
+//extern spinlock_t buffer_spinlock;
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+/*
+ Functions "storage_init()" and "storage_down()" are for initialization and
+ shutdown respectively.
+*/
+extern int storage_init (void);
+extern void storage_down (void);
+
+/*
+ Function "pack_event_info()" saves information about event into buffer. It
+ is used in 'probes' to pack and save event data.
+*/
+extern void pack_event_info (probe_id_t probe_id, record_type_t record_type, const char *fmt, ...);
+
+/*
+ Function "set_us_proc_inst()" saves instrumentation info for user space process.
+*/
+extern int set_us_proc_inst_info (ioctl_inst_usr_space_proc_t * inst_info);
+
+/*
+ Function "release_us_proc_inst_info()" destroys instrumentation info for user space process.
+*/
+extern void release_us_proc_inst_info (void);
+
+/*
+ Adds non-predefined kernel probe to the list.
+*/
+extern int add_probe_to_list (unsigned long addr, kernel_probe_t ** pprobe);
+
+/*
+ Removes non-predefined kernel probe from the list.
+*/
+extern int remove_probe_from_list (unsigned long addr);
+
+/*
+ Searches non-predefined kernel probe in the list.
+*/
+extern kernel_probe_t *find_probe (unsigned long addr);
+
+/*
+ Copies event from user space to buffer and updates its pid/tid/cpu/time.
+*/
+extern int put_us_event (char *data, unsigned long len);
+
+/*
+ Sets event mask.
+*/
+extern int set_event_mask (int new_mask);
+
+/*
+ Gets event mask.
+*/
+extern int get_event_mask (int *mask);
+
+/*
+ Sets predefined user space probes info.
+*/
+extern int set_predef_uprobes (ioctl_predef_uprobes_info_t *data);
+/*
+ Gets predefined user space probes info length.
+*/
+extern int get_predef_uprobes_size(int *size);
+/*
+ Gets predefined user space probes info.
+*/
+extern int get_predef_uprobes(ioctl_predef_uprobes_info_t *data);
+
+#ifdef MEMORY_CHECKER
+extern void mec_remove_objects(void);
+extern void *mec_find_in_object_range(void *addr, int *rz);
+extern void mec_change_active_object(void *obj);
+extern unsigned long pf_addr;
+// maximum overflow/underflow size
+// access into unallocated process space near active object,
+// but within this gap will be treated as overflow/underflow
+#define MEC_MAX_OVERFLOW_SIZE (1024)
+#endif
+
+// internal bookkeeping of storage
+extern char *p_buffer;
+
+// list of selected non-predefined kernel probes
+extern struct hlist_head kernel_probes;
+
+// event mask
+extern int event_mask;
+
+typedef struct
+{
+ char *name;
+ int installed;
+ struct jprobe jprobe;
+ struct kretprobe retprobe;
+ unsigned long offset;
+} us_proc_ip_t;
+
+typedef struct
+{
+ int installed;
+ struct jprobe jprobe;
+ unsigned long addr;
+ struct list_head list;
+} us_proc_vtp_t;
+
+typedef struct
+{
+ //char *name;
+ char type;
+ unsigned long size;
+ signed char reg; // -1 - memory, 0..127 - register number
+ long off;
+ struct list_head list;
+} us_proc_vtp_data_t;
+
+typedef struct dentry *STRUCT_DENTRY_PTR;
+
+typedef struct
+{
+ char *path;
+ STRUCT_DENTRY_PTR m_f_dentry;
+ unsigned ips_count;
+ us_proc_ip_t *p_ips;
+ unsigned vtps_count;
+ us_proc_vtp_t *p_vtps;
+ int loaded;
+} us_proc_lib_t;
+
+typedef struct
+{
+ char *path;
+ STRUCT_DENTRY_PTR m_f_dentry;
+ pid_t tgid;
+ unsigned unres_ips_count;
+ unsigned unres_vtps_count;
+ //kprobe_opcode_t *mapped_codelets;
+ unsigned libs_count;
+ us_proc_lib_t *p_libs;
+} inst_us_proc_t;
+
+struct cond {
+ /* cond data itself */
+ struct event_tmpl tmpl;
+ /* linked list */
+ struct list_head list;
+ /* has been applied (for start and stop conditions) */
+ int applied;
+};
+
+extern struct cond cond_list;
+
+/* macros for testing flags */
+#define ET_FIELD_CLR(flags, field) (flags &= ~field)
+#define ET_FIELD_SET(flags, field) (flags |= field)
+#define ET_FIELD_ISSET(flags, field) ((flags & field) != 0)
+
+extern inst_us_proc_t us_proc_info;
+
+
+#endif /* !defined(__STORAGE_H__) */
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: us_proc_inst.c
+//
+// DESCRIPTION:
+// This file is C source for SWAP driver.
+//
+// SEE ALSO: us_proc_inst.h
+// AUTHOR: A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.06.02
+// VERSION: 1.0
+// REVISION DATE: 2008.12.02
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+#include "module.h"
+#include "us_proc_inst.h"
+
+static int register_usprobe (struct task_struct *task, struct mm_struct *mm, us_proc_ip_t * ip, int atomic, kprobe_opcode_t * islot);
+static int unregister_usprobe (struct task_struct *task, us_proc_ip_t * ip, int atomic);
+
+int us_proc_probes;
+
+static int
+find_task_by_path (const char *path, struct task_struct **p_task, struct list_head *tids)
+{
+ int found = 0;
+ struct task_struct *task;
+ struct vm_area_struct *vma;
+ struct mm_struct *mm;
+ struct nameidata nd;
+ //fp_kallsyms_lookup_name_t fp;
+
+ *p_task = 0;
+
+ /* find corresponding dir entry, this is also check for valid path */
+ // TODO: test - try to instrument process with non-existing path
+ // TODO: test - try to instrument process with existing path and delete file just after start
+ if (path_lookup (path, LOOKUP_FOLLOW, &nd) != 0)
+ {
+ EPRINTF ("failed to lookup dentry for path %s!", path);
+ return -EINVAL;
+ }
+
+ rcu_read_lock ();
+ for_each_process (task) {
+ mm = get_task_mm (task);
+ if (!mm)
+ continue;
+ down_read (&mm->mmap_sem);
+ vma = mm->mmap;
+ while (vma) {
+ if ((vma->vm_flags & VM_EXECUTABLE) && vma->vm_file) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)
+ if (vma->vm_file->f_dentry == nd.dentry) {
+#else
+ if (vma->vm_file->f_dentry == nd.path.dentry) {
+#endif
+ if (!*p_task) {
+ *p_task = task;
+ get_task_struct (task);
+ }
+ //break;
+ }
+ }
+ vma = vma->vm_next;
+ }
+ up_read (&mm->mmap_sem);
+ mmput (mm);
+ if (found)
+ break;
+ }
+ rcu_read_unlock ();
+
+ if (*p_task)
+ DPRINTF ("found pid %d for %s.", (*p_task)->pid, path);
+ else
+ DPRINTF ("pid for %s not found!", path);
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)
+ path_release (&nd);
+#else
+ path_put (&nd.path);
+#endif
+ return 0;
+}
+
+#if defined(CONFIG_MIPS)
+# define ARCH_REG_VAL(regs, idx) regs->regs[idx]
+#elif defined(CONFIG_ARM)
+# define ARCH_REG_VAL(regs, idx) regs->uregs[idx]
+#else
+# define ARCH_REG_VAL(regs, idx) 0
+# warning ARCH_REG_VAL is not implemented for this architecture. FBI will work improperly or even crash!!!
+#endif // ARCH
+
+DEFINE_PER_CPU (us_proc_vtp_t *, gpVtp) = NULL;
+//EXPORT_PER_CPU_SYMBOL_GPL(gpVtp);
+DEFINE_PER_CPU (struct pt_regs *, gpCurVtpRegs) = NULL;
+//EXPORT_PER_CPU_SYMBOL_GPL(gpCurVtpRegs);
+
+static void
+us_vtp_event_pre_handler (us_proc_vtp_t * vtp, struct pt_regs *regs)
+{
+ __get_cpu_var(gpVtp) = vtp;
+ __get_cpu_var(gpCurVtpRegs) = regs;
+}
+
+static void
+us_vtp_event_handler (unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5, unsigned long arg6)
+{
+ //static int nCount;
+ us_proc_vtp_t *vtp = __get_cpu_var(gpVtp);
+#if !defined(CONFIG_X86)
+ struct pt_regs *regs = __get_cpu_var(gpCurVtpRegs);
+#endif
+ char fmt[3];
+ unsigned long vaddr;
+ long ival;
+ char cval, *sval;
+ //float fval;
+ //struct list_head *pos, *tmp;
+ us_proc_vtp_data_t *vtp_data;
+
+ fmt[0] = 'p';
+ fmt[2] = 0;
+
+ list_for_each_entry_rcu (vtp_data, &vtp->list, list)
+ {
+// DPRINTF ("[%d]proc %s(%d): %lx", nCount++, current->comm, current->pid, vtp->addr);
+ fmt[1] = vtp_data->type;
+ if (vtp_data->reg == -1)
+ vaddr = vtp_data->off;
+ else
+ vaddr = ARCH_REG_VAL (regs, vtp_data->reg) + vtp_data->off;
+// DPRINTF ("VTP type '%c'", vtp_data->type);
+ switch (vtp_data->type)
+ {
+ case 'd':
+ case 'x':
+ case 'p':
+ if (read_proc_vm_atomic (current, vaddr, &ival, sizeof (ival)) < sizeof (ival))
+ EPRINTF ("failed to read vm of proc %s/%u addr %lu!", current->comm, current->pid, vaddr);
+ else
+ pack_event_info (VTP_PROBE_ID, RECORD_ENTRY, fmt, vtp->jprobe.kp.addr, ival);
+ break;
+ case 'f':
+ if (read_proc_vm_atomic (current, vaddr, &ival, sizeof (ival)) < sizeof (ival))
+ EPRINTF ("failed to read vm of proc %s/%u addr %lu!", current->comm, current->pid, vaddr);
+ else
+ pack_event_info (VTP_PROBE_ID, RECORD_ENTRY, fmt, vtp->jprobe.kp.addr, ival);
+ break;
+ case 'c':
+ if (read_proc_vm_atomic (current, vaddr, &cval, sizeof (cval)) < sizeof (cval))
+ EPRINTF ("failed to read vm of proc %s/%u addr %lu!", current->comm, current->pid, vaddr);
+ else
+ pack_event_info (VTP_PROBE_ID, RECORD_ENTRY, fmt, vtp->jprobe.kp.addr, cval);
+ break;
+ case 's':
+ if (current->active_mm)
+ {
+ struct page *page;
+ struct vm_area_struct *vma;
+ void *maddr;
+ int len;
+ if (get_user_pages_atomic (current, current->active_mm, vaddr, 1, 0, 1, &page, &vma) <= 0)
+ {
+ EPRINTF ("get_user_pages_atomic failed for proc %s/%u addr %lu!", current->comm, current->pid, vaddr);
+ break;
+ }
+ maddr = kmap_atomic (page, KM_USER0);
+ len = strlen (maddr + (vaddr & ~PAGE_MASK));
+ sval = kmalloc (len + 1, GFP_KERNEL);
+ if (!sval)
+ EPRINTF ("failed to alloc memory for string in proc %s/%u addr %lu!", current->comm, current->pid, vaddr);
+ else
+ {
+ copy_from_user_page (vma, page, vaddr, sval, maddr + (vaddr & ~PAGE_MASK), len + 1);
+ pack_event_info (VTP_PROBE_ID, RECORD_ENTRY, fmt, vtp->jprobe.kp.addr, sval);
+ kfree (sval);
+ }
+ kunmap_atomic (maddr, KM_USER0);
+ page_cache_release (page);
+ }
+ else
+ EPRINTF ("task %s/%u has no mm!", current->comm, current->pid);
+ break;
+ default:
+ EPRINTF ("unknown variable type '%c'", vtp_data->type);
+ }
+ }
+ uprobe_return ();
+}
+
+static int
+install_mapped_ips (struct task_struct *task, int atomic)
+{
+ struct vm_area_struct *vma;
+ int i, k, err, retry;
+ unsigned long addr;//, slot_idx;
+ //char lib_path[256];
+ //static unsigned npcdep;
+ unsigned int old_ips_count, old_vtps_count;
+ struct mm_struct *mm;
+ struct ip_node {
+ struct list_head plist;
+ us_proc_ip_t * ip;
+ } * nip, *tnip;
+ LIST_HEAD(iplist);
+ struct vtp_node {
+ struct list_head plist;
+ us_proc_vtp_t * vtp;
+ } * nvtp, *tnvtp;
+ LIST_HEAD(vtplist);
+
+_restart:
+ mm = atomic ? task->active_mm : get_task_mm (task);
+ if (!mm){
+// DPRINTF ("proc %d has no mm", task->pid);
+ return us_proc_info.unres_ips_count + us_proc_info.unres_vtps_count;
+ }
+
+ old_ips_count = us_proc_info.unres_ips_count;
+ old_vtps_count = us_proc_info.unres_vtps_count;
+ if(!atomic)
+ down_read (&mm->mmap_sem);
+ vma = mm->mmap;
+ while (vma)
+ {
+ // skip non-text section
+ if (!(vma->vm_flags & VM_EXEC) || !vma->vm_file ||
+ !(vma->vm_flags & (VM_WRITE | VM_MAYWRITE)) ||
+ !(vma->vm_flags & (VM_READ | VM_MAYREAD)))
+ {
+ vma = vma->vm_next;
+ continue;
+ }
+ //DPRINTF("check mapped lib %s at %lx.",
+ // d_path(vma->vm_file->f_dentry, vma->vm_file->f_vfsmnt, lib_path, sizeof(lib_path)), vma->vm_start);
+ //slot_idx = 0;
+ for (i = 0; i < us_proc_info.libs_count; i++)
+ {
+ //DPRINTF("lookup lib %s.", us_proc_info.p_libs[i].path);
+ //TODO: test - try to instrument non-existing libs
+ if (vma->vm_file->f_dentry == us_proc_info.p_libs[i].m_f_dentry)
+ {
+ //if (strcmp(d_path(vma->vm_file->m_f_dentry, vma->vm_file->f_vfsmnt, lib_path, sizeof(lib_path)),
+ // us_proc_info.p_libs[i].path) == 0){
+ //DPRINTF("found lib %s.", us_proc_info.p_libs[i].path);
+ //unsigned oi_count = us_proc_info.unres_ips_count;
+ //unsigned ov_count = us_proc_info.unres_vtps_count;
+ for (k = 0; k < us_proc_info.p_libs[i].ips_count; k++/*, slot_idx++*/)
+ {
+ if (!us_proc_info.p_libs[i].p_ips[k].installed)
+ {
+ addr = us_proc_info.p_libs[i].p_ips[k].offset;
+ if (!(vma->vm_flags & VM_EXECUTABLE))
+ addr += vma->vm_start;
+ //DPRINTF("check sym %s at %lx.", us_proc_info.p_libs[i].p_ips[k].name, addr);
+ if (page_present (mm, addr))
+ {
+ //DPRINTF ("pid %d, %s sym is present at %lx/%lx.", task->pid, us_proc_info.p_libs[i].path, us_proc_info.p_libs[i].p_ips[k].offset, addr);
+ //if (!us_proc_info.p_libs[i].p_ips[k].installed)
+ {
+ us_proc_info.unres_ips_count--;
+ us_proc_info.p_libs[i].p_ips[k].installed = 1;
+ DPRINTF ("pid %d, %s sym is loaded at %lx/%lx.", task->pid, us_proc_info.p_libs[i].path, us_proc_info.p_libs[i].p_ips[k].offset, addr);
+ nip = kmalloc(sizeof(struct ip_node), GFP_KERNEL);
+ if(!nip){
+ EPRINTF ("failed to allocate list item for IP!");
+ continue;
+ }
+ us_proc_info.p_libs[i].p_ips[k].jprobe.kp.addr = (kprobe_opcode_t *) addr;
+ us_proc_info.p_libs[i].p_ips[k].retprobe.kp.addr = (kprobe_opcode_t *) addr;
+ INIT_LIST_HEAD (&nip->plist);
+ nip->ip = &us_proc_info.p_libs[i].p_ips[k];
+ list_add_tail (&nip->plist, &iplist);
+ }
+ }
+ }
+ }
+ for (k = 0; k < us_proc_info.p_libs[i].vtps_count; k++/*, slot_idx++*/)
+ {
+ if (us_proc_info.p_libs[i].p_vtps[k].installed)
+ {
+ /*DPRINTF("VTPs at %lx are already installed.",
+ us_proc_info.p_libs[i].p_ips[k].offset); */
+ }
+ else
+ {
+ addr = us_proc_info.p_libs[i].p_vtps[k].addr;
+ if (!(vma->vm_flags & VM_EXECUTABLE))
+ addr += vma->vm_start;
+ //DPRINTF("check VTPs at %lx.", addr);
+ if (page_present (mm, addr))
+ {
+ //us_proc_vtp_data_t *vtp_data;
+ //DPRINTF("VTPs is loaded at %lx.", addr);
+ us_proc_info.unres_vtps_count--;
+ us_proc_info.p_libs[i].p_vtps[k].installed = 1;
+ /*list_for_each_entry_rcu (vtp_data, &us_proc_info.p_libs[i].p_vtps[k].list, list)
+ {
+ DPRINTF ("VTP %s is loaded at %lx.", vtp_data->name, addr);
+ }*/
+ us_proc_info.p_libs[i].p_vtps[k].jprobe.kp.tgid = us_proc_info.tgid;
+ us_proc_info.p_libs[i].p_vtps[k].jprobe.kp.addr = (kprobe_opcode_t *) addr;
+ us_proc_info.p_libs[i].p_vtps[k].jprobe.entry = (kprobe_opcode_t *) us_vtp_event_handler;
+ us_proc_info.p_libs[i].p_vtps[k].jprobe.pre_entry = (kprobe_pre_entry_handler_t) us_vtp_event_pre_handler;
+ us_proc_info.p_libs[i].p_vtps[k].jprobe.priv_arg = &us_proc_info.p_libs[i].p_vtps[k];
+ nvtp = kmalloc(sizeof(struct vtp_node), GFP_KERNEL);
+ if(!nvtp){
+ EPRINTF ("failed to allocate list item for VTP!");
+ continue;
+ }
+ INIT_LIST_HEAD (&nvtp->plist);
+ nvtp->vtp = &us_proc_info.p_libs[i].p_vtps[k];
+ list_add_tail (&nvtp->plist, &vtplist);
+ }
+ }
+ }
+ if(!(vma->vm_flags & VM_EXECUTABLE) && !us_proc_info.p_libs[i].loaded
+ /*((oi_count != us_proc_info.unres_ips_count) || (ov_count != us_proc_info.unres_vtps_count))*/){
+ char *p;
+// DPRINTF ("post dyn lib event %s", us_proc_info.p_libs[i].path);
+ // if we installed something, post library info for those IPs
+ p = strrchr(us_proc_info.p_libs[i].path, '/');
+ if(!p)
+ p = us_proc_info.p_libs[i].path;
+ else
+ p++;
+ us_proc_info.p_libs[i].loaded = 1;
+ pack_event_info (DYN_LIB_PROBE_ID, RECORD_ENTRY, "spd",
+ p, vma->vm_start, vma->vm_end-vma->vm_start);
+ }
+ }
+ }
+ vma = vma->vm_next;
+ }
+ if(!atomic){
+ up_read (&mm->mmap_sem);
+ mmput (mm);
+ }
+
+ if(!list_empty(&iplist) || !list_empty(&vtplist)){
+// DPRINTF ("Unres IPs/VTPs %d/%d -> %d/%d.", old_ips_count, old_vtps_count,
+// us_proc_info.unres_ips_count, us_proc_info.unres_vtps_count);
+ }
+
+ retry = 0;
+ list_for_each_entry_safe(nip, tnip, &iplist, plist) {
+// DPRINTF ("Install %p/%d IP at %lx.", task, task->pid, nip->ip->offset);
+ if((PAGE_SIZE-(nip->ip->offset % PAGE_SIZE)) < MAX_INSN_SIZE){
+ retry = 1;
+// DPRINTF ("Possibly 1st insn of IP at %lx lies on 2 pages.", nip->ip->offset);
+ }
+ err = register_usprobe (task, mm, nip->ip, atomic, 0);
+ if (err != 0)
+ EPRINTF ("failed to install IP at %lx/%p. Error %d!", nip->ip->offset, nip->ip->jprobe.kp.addr, err);
+ list_del(&nip->plist);
+ kfree(nip);
+ }
+ list_for_each_entry_safe(nvtp, tnvtp, &vtplist, plist) {
+// DPRINTF ("Install VTP at %p.", nvtp->vtp->jprobe.kp.addr);
+ if((PAGE_SIZE-(nvtp->vtp->addr % PAGE_SIZE)) < MAX_INSN_SIZE){
+ retry = 1;
+// DPRINTF ("Possibly 1st insn of VTP %lx lies on 2 pages.", nvtp->vtp->addr);
+ }
+ err = register_ujprobe (task, mm, &nvtp->vtp->jprobe, atomic);
+ if (err)
+ EPRINTF ("failed to install VTP at %p. Error %d!", nvtp->vtp->jprobe.kp.addr, err);
+ list_del(&nvtp->plist);
+ kfree(nvtp);
+ }
+
+ if(retry) goto _restart;
+
+ return us_proc_info.unres_ips_count + us_proc_info.unres_vtps_count;
+}
+
+static int
+uninstall_mapped_ips (struct task_struct *task, int atomic)
+{
+ int i, k, err;
+
+ for (i = 0; i < us_proc_info.libs_count; i++)
+ {
+// DPRINTF ("clear lib %s.", us_proc_info.p_libs[i].path);
+ for (k = 0; k < us_proc_info.p_libs[i].ips_count; k++)
+ {
+ if (us_proc_info.p_libs[i].p_ips[k].installed)
+ {
+// DPRINTF ("remove IP at %p.", us_proc_info.p_libs[i].p_ips[k].jprobe.kp.addr);
+ err = unregister_usprobe (task, &us_proc_info.p_libs[i].p_ips[k], atomic);
+ if (err != 0)
+ {
+ EPRINTF ("failed to uninstall IP at %p. Error %d!", us_proc_info.p_libs[i].p_ips[k].jprobe.kp.addr, err);
+ continue;
+ }
+ us_proc_info.unres_ips_count++;
+ us_proc_info.p_libs[i].p_ips[k].installed = 0;
+ }
+ }
+ for (k = 0; k < us_proc_info.p_libs[i].vtps_count; k++)
+ {
+ if (us_proc_info.p_libs[i].p_vtps[k].installed)
+ {
+ //us_proc_vtp_data_t *vtp_data;
+ /*list_for_each_entry_rcu (vtp_data, &us_proc_info.p_libs[i].p_vtps[k].list, list)
+ {
+ DPRINTF ("remove VTP %s.", vtp_data->name);
+ }*/
+ unregister_ujprobe (task, &us_proc_info.p_libs[i].p_vtps[k].jprobe, atomic);
+ us_proc_info.unres_vtps_count++;
+ us_proc_info.p_libs[i].p_vtps[k].installed = 0;
+ }
+ }
+ }
+// DPRINTF ("Ures IPs %d.", us_proc_info.unres_ips_count);
+// DPRINTF ("Ures VTPs %d.", us_proc_info.unres_vtps_count);
+
+ return 0;
+}
+
+void
+send_sig_jprobe_event_handler (int sig, struct siginfo *info, struct task_struct *t, struct sigpending *signals)
+{
+ int iRet, del = 0;
+ struct task_struct *task;
+
+ //DPRINTF("send_signal[%d] %d proc %d", nCount++, sig, t->pid);
+
+ //DPRINTF("%s(%d) send_signal %d for target proc %s(%d)", current->comm, current->pid, sig, t->comm, t->pid);
+ if (sig != SIGKILL)
+ return;
+
+ if (t->tgid != us_proc_info.tgid)
+ return;
+
+ del = 1;
+ // look for another process with the same tgid
+ rcu_read_lock ();
+ for_each_process (task)
+ {
+ if ((task->pid != t->pid) && (task->tgid == us_proc_info.tgid))
+ {
+ del = 0;
+ break;
+ }
+ }
+ rcu_read_unlock ();
+ if (del)
+ {
+// DPRINTF ("%s(%d) send_signal SIGKILL for the last target proc %s(%d)", current->comm, current->pid, t->comm, t->pid);
+ iRet = uninstall_mapped_ips (t, 1);
+ if (iRet != 0)
+ EPRINTF ("failed to uninstall IPs (%d)!", iRet);
+ }
+}
+
+static int
+uninstall_kernel_probe (unsigned long addr, int uflag, int kflag, kernel_probe_t ** pprobe)
+{
+ kernel_probe_t *probe = NULL;
+ int iRet = 0;
+
+ if (probes_flags & kflag) {
+ probe = find_probe(addr);
+ if (probe) {
+ iRet = remove_probe_from_list (addr);
+ if (iRet)
+ EPRINTF ("remove_probe_from_list(0x%lx) result=%d!", addr, iRet);
+ if (pprobe)
+ *pprobe = NULL;
+ }
+ probes_flags &= ~kflag;
+ }
+
+ if (us_proc_probes & uflag) {
+ if (!(probes_flags & uflag)) {
+ if (probe) {
+ iRet = unregister_kernel_probe(probe);
+ if (iRet) {
+ EPRINTF ("unregister_kernel_probe(0x%lx) result=%d!",
+ addr, iRet);
+ return iRet;
+ }
+ }
+ }
+ us_proc_probes &= ~uflag;
+ }
+
+ return iRet;
+}
+
+int
+deinst_usr_space_proc (void)
+{
+ int iRet = 0, found = 0;
+ struct task_struct *task = 0;
+
+ iRet = uninstall_kernel_probe (pf_addr, US_PROC_PF_INSTLD,
+ 0, &pf_probe);
+ if (iRet)
+ EPRINTF ("uninstall_kernel_probe(do_page_fault) result=%d!", iRet);
+
+ iRet = uninstall_kernel_probe (exit_addr, US_PROC_EXIT_INSTLD,
+ 0, &exit_probe);
+ if (iRet)
+ EPRINTF ("uninstall_kernel_probe(do_exit) result=%d!", iRet);
+
+ if (us_proc_info.tgid == 0)
+ return 0;
+
+ rcu_read_lock ();
+ for_each_process (task)
+ {
+ if (task->tgid == us_proc_info.tgid)
+ {
+ found = 1;
+ get_task_struct (task);
+ break;
+ }
+ }
+ rcu_read_unlock ();
+ if (found)
+ {
+ // uninstall IPs
+ iRet = uninstall_mapped_ips (task, 0);
+ if (iRet != 0)
+ EPRINTF ("failed to uninstall IPs %d!", iRet);
+ put_task_struct (task);
+ //unregister_all_uprobes(task, 1);
+ us_proc_info.tgid = 0;
+ }
+
+ return iRet;
+}
+
+static int
+install_kernel_probe (unsigned long addr, int uflag, int kflag, kernel_probe_t ** pprobe)
+{
+ kernel_probe_t *probe = NULL;
+ int iRet = 0;
+
+ DPRINTF("us_proc_probes = 0x%x, uflag = 0x%x, "
+ "probes_flags = 0x%x, kflag = 0x%x",
+ us_proc_probes, uflag, probes_flags, kflag);
+
+ if (!(probes_flags & kflag)) {
+ iRet = add_probe_to_list (addr, &probe);
+ if (iRet) {
+ EPRINTF ("add_probe_to_list(0x%lx) result=%d!", addr, iRet);
+ return iRet;
+ }
+ probes_flags |= kflag;
+ }
+
+ if (!(us_proc_probes & uflag)) {
+ if (!(probes_flags & uflag)) {
+ iRet = register_kernel_probe (probe);
+ if (iRet) {
+ EPRINTF ("register_kernel_probe(0x%lx) result=%d!", addr, iRet);
+ return iRet;
+ }
+ }
+ us_proc_probes |= uflag;
+ }
+
+ if (pprobe)
+ *pprobe = probe;
+
+ return 0;
+}
+
+int
+inst_usr_space_proc (void)
+{
+ int iRet, i;
+ struct task_struct *task = 0;
+ //struct mm_struct *mm;
+
+ if (!us_proc_info.path)
+ return 0;
+
+ for (i = 0; i < us_proc_info.libs_count; i++)
+ us_proc_info.p_libs[i].loaded = 0;
+ /* check whether process is already running
+ * 1) if process is running - look for the libraries in the process maps
+ * 1.1) check if page for symbol does exist
+ * 1.1.1) if page exists - instrument it
+ * 1.1.2) if page does not exist - make sure that do_page_fault handler is installed
+ * 2) if process is not running - make sure that do_page_fault handler is installed
+ * */
+ //iRet =
+ find_task_by_path (us_proc_info.path, &task, NULL);
+ //if (iRet != 0)
+ // return iRet;
+
+ if (task)
+ {
+ us_proc_info.tgid = task->pid;
+ /*mm = get_task_mm (task);
+ if (!mm)
+ {
+ EPRINTF ("task %d has no mm!", task->pid);
+ return -1;
+ }
+ down_read (&mm->mmap_sem);*/
+ install_mapped_ips (task, 0);
+ //up_read (&mm->mmap_sem);
+ //mmput (mm);
+ put_task_struct (task);
+ }
+
+ // enable 'do_page_fault' probe to detect when they will be loaded
+ iRet = install_kernel_probe (pf_addr, US_PROC_PF_INSTLD, 0, &pf_probe);
+ if (iRet)
+ {
+ EPRINTF ("install_kernel_probe(do_page_fault) result=%d!", iRet);
+ return iRet;
+ }
+ // enable 'do_exit' probe to detect when user proc exits in order to remove user space probes
+ iRet = install_kernel_probe (exit_addr, US_PROC_EXIT_INSTLD, 0, &exit_probe);
+ if (iRet)
+ {
+ EPRINTF ("install_kernel_probe(do_exit) result=%d!", iRet);
+ return iRet;
+ }
+
+ return 0;
+}
+
+void
+do_page_fault_ret_pre_code (void)
+{
+ struct mm_struct *mm;
+ struct vm_area_struct *vma = 0;
+
+ if (!us_proc_info.path)
+ return;
+
+ //DPRINTF("do_page_fault from proc %d-%d-%d", current->pid, us_proc_info.tgid, us_proc_info.unres_ips_count);
+ if ((us_proc_info.unres_ips_count + us_proc_info.unres_vtps_count) == 0)
+ {
+ //DPRINTF("do_page_fault: there no unresolved IPs");
+ return;
+ }
+
+ if (us_proc_info.tgid == 0)
+ {
+ //DPRINTF("do_page_fault check proc %d", current->pid);
+ // check whether
+ /*if( path_lookup(us_proc_info.path, LOOKUP_FOLLOW, &nd) != 0 ) {
+ EPRINTF("failed to lookup dentry for path %s!", us_proc_info.path);
+ return;
+ } */
+ mm = get_task_mm (current);//current->active_mm;
+ if (mm)
+ {
+ down_read (&mm->mmap_sem);
+ //BUG_ON(down_read_trylock(&mm->mmap_sem) == 0);
+ vma = mm->mmap;
+ while (vma)
+ {
+ if ((vma->vm_flags & VM_EXECUTABLE) && vma->vm_file)
+ {
+ //DPRINTF("do_page_fault: dentry %p-%p.", vma->vm_file->m_f_dentry, nd.dentry);
+ //DPRINTF("do_page_fault: expath %s.",
+ // d_path(vma->vm_file->m_f_dentry, vma->vm_file->f_vfsmnt, expath, sizeof(expath)));
+ if (vma->vm_file->f_dentry == us_proc_info.m_f_dentry)
+ {
+ //if (strcmp(d_path(vma->vm_file->m_f_dentry, vma->vm_file->f_vfsmnt, expath, sizeof(expath)),
+ // us_proc_info.path) == 0){
+ //DPRINTF("do_page_fault: found!");
+ //orig_mm = mm;
+ break;
+ }
+ }
+ vma = vma->vm_next;
+ }
+ up_read (&mm->mmap_sem);
+ mmput (mm);
+ } else {
+// DPRINTF ("proc %s/%d has no mm", current->comm, current->pid);
+ }
+ //path_release(&nd);
+ if (vma)
+ {
+ DPRINTF ("do_page_fault found target proc %s(%d)", current->comm, current->pid);
+ us_proc_info.tgid = current->pid;
+ }
+ }
+ if (us_proc_info.tgid == current->tgid)
+ {
+ //DPRINTF("do_page_fault from target proc %d", us_proc_info.tgid);
+ if (install_mapped_ips (current, 1) == 0)
+ {
+ /*iRet = unregister_one_probe(pf_probe_id);
+ if (iRet)
+ EPRINTF("unregister_one_probe(do_page_fault) result=%d!", iRet); */
+ }
+ }
+
+ //DPRINTF("do_page_fault from proc %d-%d exit", current->pid, us_proc_info.pid);
+}
+EXPORT_SYMBOL_GPL(do_page_fault_ret_pre_code);
+
+void
+do_exit_probe_pre_code (void)
+{
+ int iRet, del = 0;
+ struct task_struct *task;
+
+ //DPRINTF("do_exit from proc %s-%d", current->comm, current->pid);
+
+ if (current->tgid != us_proc_info.tgid)
+ return;
+
+ //DPRINTF("exit target proc %s-%d", current->comm, current->pid);
+ del = 1;
+ // look for another process with the same tgid
+ rcu_read_lock ();
+ for_each_process (task)
+ {
+ //if(task->tgid == us_proc_info.tgid)
+ // DPRINTF("check proc %s-%d", task->comm, task->pid);
+ if ((task->pid != current->pid) && (task->tgid == us_proc_info.tgid))
+ {
+ del = 0;
+ break;
+ }
+ }
+ rcu_read_unlock ();
+ if (del)
+ {
+// DPRINTF ("do_exit from the last target proc %s-%d", current->comm, current->pid);
+ iRet = uninstall_mapped_ips (current, 1);
+ if (iRet != 0)
+ EPRINTF ("failed to uninstall IPs (%d)!", iRet);
+ unregister_all_uprobes(current, 1);
+ us_proc_info.tgid = 0;
+ }
+
+#ifdef MEMORY_CHECKER
+ mec_remove_objects();
+#endif
+}
+EXPORT_SYMBOL_GPL(do_exit_probe_pre_code);
+
+DEFINE_PER_CPU (us_proc_ip_t *, gpCurIp) = NULL;
+EXPORT_PER_CPU_SYMBOL_GPL(gpCurIp);
+DEFINE_PER_CPU(struct pt_regs *, gpUserRegs) = NULL;
+EXPORT_PER_CPU_SYMBOL_GPL(gpUserRegs);
+
+static unsigned long
+ujprobe_event_pre_handler (us_proc_ip_t * ip, struct pt_regs *regs)
+{
+ __get_cpu_var (gpCurIp) = ip;
+ __get_cpu_var (gpUserRegs) = regs;
+ return 0;
+}
+
+void
+ujprobe_event_handler (unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5, unsigned long arg6)
+{
+ us_proc_ip_t *ip = __get_cpu_var (gpCurIp);
+
+ //static int nCount;
+
+ //DPRINTF("[%d]proc %s(%d): ENTRY %s(%#lx)", nCount++, current->comm, current->pid, ip->name, arg1);
+#ifdef MEMORY_CHECKER
+ struct pt_regs *regs = __get_cpu_var(gpUserRegs);
+#if defined(CONFIG_ARM)
+ pack_event_info (US_PROBE_ID, RECORD_ENTRY, "ppppppp", ip->jprobe.kp.addr, regs->ARM_lr, regs->ARM_fp, arg1, arg2, arg3, arg4, arg5, arg6);
+#else
+#warning not implemented for this arch!
+#endif
+#else
+ pack_event_info (US_PROBE_ID, RECORD_ENTRY, "ppppppp", ip->jprobe.kp.addr, arg1, arg2, arg3, arg4, arg5, arg6);
+#endif
+ uprobe_return ();
+}
+
+int
+uretprobe_event_handler (struct kretprobe_instance *probe, struct pt_regs *regs, us_proc_ip_t * ip)
+{
+ //static int nCount;
+
+ int retval = regs_return_value(regs);
+ //DPRINTF("[%d]proc %s(%d): RETURN %s(%d)", nCount++, current->comm, current->pid, ip->name, retval);
+ pack_event_info (US_PROBE_ID, RECORD_RET, "pd", ip->retprobe.kp.addr, retval);
+ return 0;
+}
+
+static int
+register_usprobe (struct task_struct *task, struct mm_struct *mm, us_proc_ip_t * ip, int atomic, kprobe_opcode_t * islot)
+{
+ int ret = 0;
+
+ ip->jprobe.kp.tgid = task->tgid;
+ //ip->jprobe.kp.addr = (kprobe_opcode_t *) addr;
+ if(!ip->jprobe.entry)
+ ip->jprobe.entry = (kprobe_opcode_t *) ujprobe_event_handler;
+ if(!ip->jprobe.pre_entry)
+ ip->jprobe.pre_entry = (kprobe_pre_entry_handler_t) ujprobe_event_pre_handler;
+ ip->jprobe.priv_arg = ip;
+ ret = register_ujprobe (task, mm, &ip->jprobe, atomic);
+ if (ret)
+ {
+ EPRINTF ("register_ujprobe() failure %d", ret);
+ return ret;
+ }
+
+ //DPRINTF("IP %s boostable = %d", ip->name, ip->jprobe.kp.ainsn.boostable);
+#if 0//defined(CONFIG_X86)
+ // _start is an entry point of the program, so when control reaches it
+ // return address is not saved on stack and can not be hijacked,
+ // so we can not set retprobe on it
+ if(strcmp(ip->name, "_start")!=0)
+#endif
+ {
+ ip->retprobe.kp.tgid = task->tgid;
+ //ip->retprobe.kp.addr = (kprobe_opcode_t *) addr;
+ if(!ip->retprobe.handler)
+ ip->retprobe.handler = (kretprobe_handler_t) uretprobe_event_handler;
+ ip->retprobe.priv_arg = ip;
+ ret = register_uretprobe (task, mm, &ip->retprobe, atomic);
+ if (ret)
+ {
+ EPRINTF ("register_uretprobe() failure %d", ret);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+static int
+unregister_usprobe (struct task_struct *task, us_proc_ip_t * ip, int atomic)
+{
+ unregister_ujprobe (task, &ip->jprobe, atomic);
+#if 0//defined(CONFIG_X86)
+ // _start is an entry point of the program, so when control reaches it
+ // return address is not saved on stack and can not be hijacked,
+ // so we can not set retprobe on it
+ if(strcmp(ip->name, "_start")!=0)
+#endif
+ unregister_uretprobe (task, &ip->retprobe, atomic);
+
+ return 0;
+}
--- /dev/null
+////////////////////////////////////////////////////////////////////////////////////
+//
+// FILE: us_proc_inst.h
+//
+// DESCRIPTION:
+//
+// SEE ALSO: us_proc_inst.c
+// AUTHOR: A.Gerenkov
+// COMPANY NAME: Samsung Research Center in Moscow
+// DEPT NAME: Advanced Software Group
+// CREATED: 2008.06.02
+// VERSION: 1.0
+// REVISION DATE: 2008.12.03
+//
+////////////////////////////////////////////////////////////////////////////////////
+
+/*
+ Functions in "us_proc_inst.h" file .
+*/
+
+#if !defined(__US_PROC_INST_H__)
+#define __US_PROC_INST_H__
+
+#include <linux/signal.h> // struct sigpending
+
+/*
+ Instruments or schedules pending instrumentation of user space process.
+*/
+extern int inst_usr_space_proc (void);
+extern int deinst_usr_space_proc (void);
+
+/*
+ Detects when IPs are really loaded into phy mem and installs probes.
+*/
+extern void do_page_fault_ret_pre_code (void);
+
+/*
+ Detects when target process exits and removes IPs.
+*/
+extern void do_exit_probe_pre_code (void);
+
+/*
+ Detects when target process is killed and removes IPs.
+*/
+//extern void send_sig_jprobe_event_handler (int sig, struct siginfo *info, struct task_struct *t, struct sigpending *signals);
+
+extern int us_proc_probes;
+
+#define US_PROC_PF_INSTLD 0x1
+#define US_PROC_EXIT_INSTLD 0x2
+//#define US_PROC_SS_INSTLD 0x4
+//#define US_PROC_FORK_INSTLD 0x8
+
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+#endif /* !defined(__US_PROC_INST_H__) */
--- /dev/null
+include $(project_pwd)/common.mk
+
+program_name := $(kprobe_name)
+source_dirs := $(project_pwd)/$(src_path)/$(kprobe_path)
+include_dirs :=
+include_libs :=
+special_defines := CONFIG_KPROBES
+ifeq ($(specific_target), chelsea)
+ special_defines += KERNEL_HAS_ISPAGEPRESENT
+endif
+special_compiling_flags :=
+special_linking_flags :=
+
+include $(project_pwd)/rules-m.mk
--- /dev/null
+include $(project_pwd)/common.mk
+
+help:
+ @$(MAKE) ARCH=$(target_cpu) CROSS_COMPILE=$(target_compiler_prefix) -C $(target_kernel_src) \
+ M=$(project_pwd)/$(src_path)/$(kprobe_path) \
+ project_pwd=$(project_pwd) help
+
+build:
+ @$(MAKE) ARCH=$(target_cpu) CROSS_COMPILE=$(target_compiler_prefix) -C $(target_kernel_src) \
+ M=$(project_pwd)/$(src_path)/$(kprobe_path) \
+ project_pwd=$(project_pwd) modules
+
+clean:
+ @$(MAKE) ARCH=$(target_cpu) CROSS_COMPILE=$(target_compiler_prefix) -C $(target_kernel_src) \
+ M=$(project_pwd)/$(src_path)/$(kprobe_path) \
+ project_pwd=$(project_pwd) clean
--- /dev/null
+########################################
+## ELP2 KProbe Dynamically Loaded Module
+########################################
+
+1. Overview
+ ELP2 KProbe Module provides dynamic enabling of KProbe facilities on the current platform,
+ eliminating the necessity of according Linux kernel patches.
+ Supports the following architectures:
+ a) ARM (OMAP 5912 OSK, MV320-LCD)
+ b) MIPS (DTV x260, DMB2 AMD)
+ Runs on Linux kernel 2.6.10 - 2.6.21.
+
+2. Installation
+ a) Extract files from installation archive "elp2-kprobe-dynamic.tgz"
+ into desired location.
+ b) Find the following files:
+ asm/kprobe.h - architecture specific header file
+ kprobes_arch.c - architecture specific source code
+ kprobe.h - header file
+ kprobes.c - source code
+ build_arm - example of how to compile the module for OMAP 5912 OSK
+ build_cosmos - example of how to compile the module for MV320-LCD
+ build_dtv_x260 - example of how to compile the module for DTV x260
+ build_mips - example of how to compile the module for DMB2 AMD
+ README - this file
+ c) Execute the following command to compile the module:
+ make ARCH=[arm, mips] CROSS_COMPILE=[/opt/...] KERNELDIR=[/opt/...]
+ d) Copy the output file "elp_kprobes.ko" to the target filesystem.
+
+3. Usage
+ a) Find out the address [XXX] of function "kallsyms_lookup_name":
+ cat /proc/kallsyms > kallsyms.txt
+ b) Execute the pointed command to load the module into memory:
+ insmod elp_kprobes.ko ksyms=0x[XXX]
+
+ Now, KProbe facilities are enabled to requestors.
--- /dev/null
+// src_asm_kprobes.h
+#ifndef _SRC_ASM_KPROBES_H
+#define _SRC_ASM_KPROBES_H
+
+#include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
+#include <linux/config.h>
+#endif
+
+//#define _DEBUG
+//#define KPROBES_RET_PROBE_TRAMP
+//#define KPROBES_PROFILE
+
+
+#ifdef _DEBUG
+extern int gSilent;
+#define DBPRINTF(format, args...) do { \
+ if( !gSilent ){ \
+ char *f = __FILE__; \
+ char *n = strrchr(f, '/'); \
+ printk("%s : %u : %s : " format "\n" , (n) ? n+1 : f, __LINE__, __FUNCTION__, ##args); \
+ } \
+ } while(0)
+#else
+#define DBPRINTF(format, args...)
+#endif
+
+#if defined(CONFIG_MIPS)
+typedef unsigned long kprobe_opcode_t;
+# define BREAKPOINT_INSTRUCTION 0x0000000d
+# ifndef KPROBES_RET_PROBE_TRAMP
+# define UNDEF_INSTRUCTION 0x0000004d
+# endif
+#elif defined(CONFIG_ARM)
+typedef unsigned long kprobe_opcode_t;
+# ifdef CONFIG_CPU_S3C2443
+# define BREAKPOINT_INSTRUCTION 0xe1200070
+# else
+# define BREAKPOINT_INSTRUCTION 0xffffffff
+# endif
+# ifndef KPROBES_RET_PROBE_TRAMP
+# ifdef CONFIG_CPU_S3C2443
+# define UNDEF_INSTRUCTION 0xe1200071
+# else
+# define UNDEF_INSTRUCTION 0xfffffffe
+# endif
+# endif
+#elif defined(CONFIG_X86)
+typedef u8 kprobe_opcode_t;
+# define BREAKPOINT_INSTRUCTION 0xcc
+# define RELATIVEJUMP_INSTRUCTION 0xe9
+/*# define UNDEF_INSTRUCTION 0xff
+# warning UNDEF_INSTRUCTION is not defined for x86 arch!!!*/
+#else
+# error BREAKPOINT_INSTRUCTION is not defined for this arch!!!
+# error UNDEF_INSTRUCTION is not defined for this arch!!!
+#endif // ARCH
+
+#if defined(CONFIG_X86)
+# define MAX_INSN_SIZE 16
+# define MAX_STACK_SIZE 64
+# define MIN_STACK_SIZE(ADDR) (((MAX_STACK_SIZE) < \
+ (((unsigned long)current_thread_info()) + THREAD_SIZE - (ADDR))) \
+ ? (MAX_STACK_SIZE) \
+ : (((unsigned long)current_thread_info()) + THREAD_SIZE - (ADDR)))
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 25)
+# define EREG(rg) e##rg
+# define XREG(rg) x##rg
+# define ORIG_EAX_REG orig_eax
+#else
+# define EREG(rg) rg
+# define XREG(rg) rg
+# define ORIG_EAX_REG orig_ax
+#endif
+#else//non x86
+# define MAX_INSN_SIZE 1
+#endif
+
+#define flush_insn_slot(p) do { } while (0)
+
+#define kprobe_lookup_name(name, addr)
+
+/* Architecture specific copy of original instruction */
+struct arch_specific_insn {
+ /* copy of the original instruction */
+ kprobe_opcode_t *insn;
+ /*
+ * If this flag is not 0, this kprobe can be boost when its
+ * post_handler and break_handler is not set.
+ */
+ int boostable;
+};
+
+#define JPROBE_ENTRY(pentry) (kprobe_opcode_t *)pentry
+
+struct prev_kprobe {
+ struct kprobe *kp;
+ unsigned long status;
+#if defined(CONFIG_X86)
+ unsigned long old_eflags;
+ unsigned long saved_eflags;
+#endif
+};
+
+/* per-cpu kprobe control block */
+struct kprobe_ctlblk {
+ unsigned long kprobe_status;
+ struct prev_kprobe prev_kprobe;
+#if defined(CONFIG_X86)
+ struct pt_regs jprobe_saved_regs;
+ unsigned long kprobe_old_eflags;
+ unsigned long kprobe_saved_eflags;
+ unsigned long *jprobe_saved_esp;
+ kprobe_opcode_t jprobes_stack[MAX_STACK_SIZE];
+#endif
+};
+
+void kretprobe_trampoline (void);
+
+extern int kprobe_handler (struct pt_regs *regs);
+extern int page_present (struct mm_struct *mm, unsigned long addr);
+
+#if defined(CONFIG_X86)
+extern int kprobe_exceptions_notify (struct notifier_block *self, unsigned long val, void *data);
+#endif // CONFIG_X86
+
+#endif /* _SRC_ASM_KPROBES_H */
--- /dev/null
+#!/bin/sh
+
+SWAP_KPROBE=inperfa_kprobe
+KSYMS=kallsyms_lookup_name
+
+# ADDRESS for "kallsyms_lookup_name" function taken from /proc/kallsyms
+ADDRESS=0x`sed "/ kallsyms_lookup_name/ ! d" /proc/kallsyms | sed "s/ T kallsyms_lookup_name//"`
+
+if [ "${ADDRESS}" = "0x" ]; then
+ if [ "$1" = "" ]; then
+ echo "Enter kallsyms_lookup_name as parameter:"
+ echo "inperfa_kprobe.sh <kallsyms_lookup_name address>"
+ exit 1
+ else
+ ADDRESS=$1
+ echo "kallsyms_lookup_name address is ${ADDRESS}"
+ fi
+fi
+
+# Check for running module in /proc/modules
+RUNNING=`sed "/${SWAP_KPROBE}/ ! d" /proc/modules`
+
+if [ "${RUNNING}" = "" ]; then
+ insmod ${SWAP_KPROBE}.ko ksyms=${ADDRESS}
+ if [ $? -ne 0 ]; then
+ echo "Error: unable to load Inperfa KProbe!"
+ fi
+else
+ echo "Inperfa Kprobe is already running!"
+ exit 1
+fi
--- /dev/null
+// src_kprobes.c
+
+
+#include <linux/version.h>
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
+#include <linux/config.h>
+#endif
+
+#include <asm/types.h>
+
+#include <linux/hash.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/moduleloader.h>
+#include <linux/kallsyms.h>
+//#include <linux/freezer.h>
+#include <linux/seq_file.h>
+#ifdef CONFIG_DEBUG_FS
+#include <linux/debugfs.h>
+#endif
+#include <asm-generic/sections.h>
+#include <asm/cacheflush.h>
+#include <asm/errno.h>
+#include <linux/spinlock.h>
+#include <linux/version.h>
+#include <linux/highmem.h> // kmap_atomic, kunmap_atomic, copy_from_user_page, copy_to_user_page
+#include <linux/pagemap.h> // page_cache_release
+#include <linux/vmalloc.h> // vmalloc, vfree
+#if defined(CONFIG_X86)
+#include <linux/kdebug.h> // register_die_notifier, unregister_die_notifier
+#endif
+#include <linux/hugetlb.h> // follow_hugetlb_page, is_vm_hugetlb_page
+
+#include "kprobes.h"
+
+//#define arch_remove_kprobe(p) do { } while (0)
+
+#ifdef _DEBUG
+extern int nCount;
+#endif
+
+/*
+static spinlock_t die_notifier_lock = SPIN_LOCK_UNLOCKED;
+
+int src_register_die_notifier(struct notifier_block *nb)
+{
+ int err = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&die_notifier_lock, flags);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,17)
+ err = atomic_notifier_chain_register(&panic_notifier_list, nb);
+#else
+ err = notifier_chain_register(&panic_notifier_list, nb);
+#endif
+ spin_unlock_irqrestore(&die_notifier_lock, flags);
+
+ return err;
+}
+*/
+/**
+ * hlist_replace_rcu - replace old entry by new one
+ * @old : the element to be replaced
+ * @new : the new element to insert
+ *
+ * The @old entry will be replaced with the @new entry atomically.
+ */
+static inline void
+src_hlist_replace_rcu (struct hlist_node *old, struct hlist_node *new)
+{
+ struct hlist_node *next = old->next;
+
+ new->next = next;
+ new->pprev = old->pprev;
+ smp_wmb ();
+ if (next)
+ new->next->pprev = &new->next;
+ if (new->pprev)
+ *new->pprev = new;
+ old->pprev = LIST_POISON2;
+}
+
+#define KPROBE_HASH_BITS 6
+#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
+
+
+/*
+ * Some oddball architectures like 64bit powerpc have function descriptors
+ * so this must be overridable.
+ */
+#ifndef kprobe_lookup_name
+#define kprobe_lookup_name(name, addr) \
+ addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
+#endif
+
+static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
+static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
+static struct hlist_head uprobe_insn_slot_table[KPROBE_TABLE_SIZE];
+static atomic_t kprobe_count;
+
+//DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */
+DEFINE_SPINLOCK (kretprobe_lock); /* Protects kretprobe_inst_table */
+static DEFINE_PER_CPU (struct kprobe *, kprobe_instance) = NULL;
+unsigned long handled_exceptions;
+
+/* We have preemption disabled.. so it is safe to use __ versions */
+static inline void
+set_kprobe_instance (struct kprobe *kp)
+{
+ __get_cpu_var (kprobe_instance) = kp;
+}
+
+static inline void
+reset_kprobe_instance (void)
+{
+ __get_cpu_var (kprobe_instance) = NULL;
+}
+
+/*
+ * This routine is called either:
+ * - under the kprobe_mutex - during kprobe_[un]register()
+ * OR
+ * - with preemption disabled - from arch/xxx/kernel/kprobes.c
+ */
+struct kprobe __kprobes *
+get_kprobe (void *addr, int tgid, struct task_struct *ctask)
+{
+ struct hlist_head *head;
+ struct hlist_node *node;
+ struct kprobe *p, *retVal = NULL;
+ int ret = 0, uprobe_found;
+ struct page *page = 0, *tpage = 0;
+ struct vm_area_struct *vma = 0;
+ struct task_struct *task = 0;
+ void *paddr = 0;
+
+
+ if (ctask && ctask->active_mm)
+ {
+ ret = get_user_pages_atomic (ctask, ctask->active_mm, (unsigned long) addr, 1, 0, 0, &tpage, NULL);
+ if (ret <= 0)
+ DBPRINTF ("get_user_pages for task %d at %p failed!", current->pid, addr);
+ else
+ {
+ paddr = page_address (tpage);
+ page_cache_release (tpage);
+ }
+ }
+ //else
+ // DBPRINTF("task %d has no mm!", ctask->pid);
+
+ //TODO: test - two processes invokes instrumented function
+ head = &kprobe_table[hash_ptr (addr, KPROBE_HASH_BITS)];
+ hlist_for_each_entry_rcu (p, node, head, hlist)
+ {
+ //if looking for kernel probe and this is kernel probe with the same addr OR
+ //if looking for the user space probe and this is user space probe probe with the same addr and pid
+ DBPRINTF ("get_kprobe[%d]: check probe at %p/%p, task %d/%d", nCount, addr, p->addr, tgid, p->tgid);
+ if (p->addr == addr)
+ {
+ uprobe_found = 0;
+ if (tgid == p->tgid)
+ uprobe_found = 1;
+ if (!tgid || uprobe_found)
+ {
+ retVal = p;
+ if (tgid)
+ DBPRINTF ("get_kprobe[%d]: found user space probe at %p for task %d", nCount, p->addr, p->tgid);
+ else
+ DBPRINTF ("get_kprobe[%d]: found kernel probe at %p", nCount, p->addr);
+ break;
+ }
+ }
+ else if (tgid != p->tgid)
+ {
+ // if looking for the user space probe and this is user space probe
+ // with another addr and pid but with the same offset whithin the page
+ // it could be that it is the same probe (with address from other user space)
+ // we should handle it as usual probe but without notification to user
+ if (paddr && tgid && (((unsigned long) addr & ~PAGE_MASK) == ((unsigned long) p->addr & ~PAGE_MASK))
+ && p->tgid)
+ {
+ DBPRINTF ("get_kprobe[%d]: found user space probe at %p in task %d. possibly for addr %p in task %d", nCount, p->addr, p->tgid, addr, tgid);
+ // this probe has the same offset in the page
+ // look in the probes for the other pids
+ // get page for user space probe addr
+ rcu_read_lock ();
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
+ task = find_task_by_pid (p->tgid);
+#else //lif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 31)
+ task = pid_task(find_pid_ns(p->tgid, &init_pid_ns), PIDTYPE_PID);
+#endif
+ if (task)
+ get_task_struct (task);
+ rcu_read_unlock ();
+ if (!task)
+ {
+ DBPRINTF ("task for pid %d not found! Dead probe?", p->tgid);
+ continue;
+ }
+ if (task->active_mm)
+ {
+ if (page_present (task->active_mm, (unsigned long) p->addr))
+ {
+ ret = get_user_pages_atomic (task, task->active_mm, (unsigned long) p->addr, 1, 0, 0, &page, &vma);
+ if (ret <= 0)
+ DBPRINTF ("get_user_pages for task %d at %p failed!", p->tgid, p->addr);
+ }
+ else
+ ret = -1;
+ }
+ else
+ {
+ DBPRINTF ("task %d has no mm!", task->pid);
+ ret = -1;
+ }
+ put_task_struct (task);
+ if (ret <= 0)
+ continue;
+ if (paddr == page_address (page))
+ {
+ retVal = p; // we found the probe in other process address space
+ DBPRINTF ("get_kprobe[%d]: found user space probe at %p in task %d for addr %p in task %d", nCount, p->addr, p->tgid, addr, tgid);
+ panic ("user space probe from another process");
+ }
+ page_cache_release (page);
+ if (retVal)
+ break;
+ }
+ }
+ }
+
+ DBPRINTF ("get_kprobe[%d]: probe %p", nCount, retVal);
+ return retVal;
+}
+
+struct kprobe __kprobes *
+get_kprobe_by_insn_slot (void *addr, int tgid, struct task_struct *ctask)
+{
+ struct hlist_head *head;
+ struct hlist_node *node;
+ struct kprobe *p, *retVal = NULL;
+ int uprobe_found;
+
+ //TODO: test - two processes invokes instrumented function
+ head = &uprobe_insn_slot_table[hash_ptr (addr, KPROBE_HASH_BITS)];
+ hlist_for_each_entry_rcu (p, node, head, is_hlist)
+ {
+ //if looking for kernel probe and this is kernel probe with the same addr OR
+ //if looking for the user space probe and this is user space probe probe with the same addr and pid
+ DBPRINTF ("get_kprobe[%d]: check probe at %p/%p, task %d/%d", nCount, addr, p->ainsn.insn, tgid, p->tgid);
+ if (p->ainsn.insn == addr)
+ {
+ uprobe_found = 0;
+ if (tgid == p->tgid)
+ uprobe_found = 1;
+ if (!tgid || uprobe_found)
+ {
+ retVal = p;
+ if (tgid)
+ DBPRINTF ("get_kprobe[%d]: found user space probe at %p for task %d", nCount, p->addr, p->tgid);
+ else
+ DBPRINTF ("get_kprobe[%d]: found kernel probe at %p", nCount, p->addr);
+ break;
+ }
+ }
+ }
+
+ DBPRINTF ("get_kprobe[%d]: probe %p", nCount, retVal);
+ return retVal;
+}
+
+/*
+ * Aggregate handlers for multiple kprobes support - these handlers
+ * take care of invoking the individual kprobe handlers on p->list
+ */
+static int __kprobes
+aggr_pre_handler (struct kprobe *p, struct pt_regs *regs /*,
+ struct vm_area_struct **vma,
+ struct page **page, unsigned long **kaddr */ )
+{
+ struct kprobe *kp;
+ int ret;
+
+ list_for_each_entry_rcu (kp, &p->list, list)
+ {
+ if (kp->pre_handler)
+ {
+ set_kprobe_instance (kp);
+ ret = kp->pre_handler (kp, regs);
+ if (ret)
+ return ret;
+ }
+ reset_kprobe_instance ();
+ }
+ return 0;
+}
+
+static void __kprobes
+aggr_post_handler (struct kprobe *p, struct pt_regs *regs, unsigned long flags)
+{
+ struct kprobe *kp;
+
+ list_for_each_entry_rcu (kp, &p->list, list)
+ {
+ if (kp->post_handler)
+ {
+ set_kprobe_instance (kp);
+ kp->post_handler (kp, regs, flags);
+ reset_kprobe_instance ();
+ }
+ }
+ return;
+}
+
+#if 1
+static int __kprobes
+aggr_fault_handler (struct kprobe *p, struct pt_regs *regs, int trapnr)
+{
+ struct kprobe *cur = __get_cpu_var (kprobe_instance);
+
+ /*
+ * if we faulted "during" the execution of a user specified
+ * probe handler, invoke just that probe's fault handler
+ */
+ if (cur && cur->fault_handler)
+ {
+ if (cur->fault_handler (cur, regs, trapnr))
+ return 1;
+ }
+ return 0;
+}
+#endif
+
+static int __kprobes
+aggr_break_handler (struct kprobe *p, struct pt_regs *regs /*,
+ struct vm_area_struct **vma,
+ struct page **page, unsigned long **kaddr */ )
+{
+ struct kprobe *cur = __get_cpu_var (kprobe_instance);
+ int ret = 0;
+ DBPRINTF ("cur = 0x%p\n", cur);
+ if (cur)
+ DBPRINTF ("cur = 0x%p cur->break_handler = 0x%p\n", cur, cur->break_handler);
+
+ if (cur && cur->break_handler)
+ {
+ if (cur->break_handler (cur, regs /*, vma, page, kaddr */ ))
+ ret = 1;
+ }
+ reset_kprobe_instance ();
+ return ret;
+}
+
+/* Walks the list and increments nmissed count for multiprobe case */
+void __kprobes
+kprobes_inc_nmissed_count (struct kprobe *p)
+{
+ struct kprobe *kp;
+ if (p->pre_handler != aggr_pre_handler)
+ {
+ p->nmissed++;
+ }
+ else
+ {
+ list_for_each_entry_rcu (kp, &p->list, list) kp->nmissed++;
+ }
+ return;
+}
+
+/* Called with kretprobe_lock held */
+struct kretprobe_instance __kprobes *
+get_free_rp_inst (struct kretprobe *rp)
+{
+ struct hlist_node *node;
+ struct kretprobe_instance *ri;
+ hlist_for_each_entry (ri, node, &rp->free_instances, uflist)
+ return ri;
+ return NULL;
+}
+
+/* Called with kretprobe_lock held */
+static struct kretprobe_instance __kprobes *
+get_used_rp_inst (struct kretprobe *rp)
+{
+ struct hlist_node *node;
+ struct kretprobe_instance *ri;
+ hlist_for_each_entry (ri, node, &rp->used_instances, uflist) return ri;
+ return NULL;
+}
+
+/* Called with kretprobe_lock held */
+void __kprobes
+add_rp_inst (struct kretprobe_instance *ri)
+{
+ /*
+ * Remove rp inst off the free list -
+ * Add it back when probed function returns
+ */
+ hlist_del (&ri->uflist);
+
+ /* Add rp inst onto table */
+ INIT_HLIST_NODE (&ri->hlist);
+ hlist_add_head (&ri->hlist, &kretprobe_inst_table[hash_ptr (ri->task, KPROBE_HASH_BITS)]);
+
+ /* Also add this rp inst to the used list. */
+ INIT_HLIST_NODE (&ri->uflist);
+ hlist_add_head (&ri->uflist, &ri->rp->used_instances);
+}
+
+/* Called with kretprobe_lock held */
+void __kprobes
+recycle_rp_inst (struct kretprobe_instance *ri, struct hlist_head *head)
+{
+ /* remove rp inst off the rprobe_inst_table */
+ hlist_del (&ri->hlist);
+ if (ri->rp)
+ {
+ /* remove rp inst off the used list */
+ hlist_del (&ri->uflist);
+ /* put rp inst back onto the free list */
+ INIT_HLIST_NODE (&ri->uflist);
+ hlist_add_head (&ri->uflist, &ri->rp->free_instances);
+ }
+ else
+ /* Unregistering */
+ hlist_add_head (&ri->hlist, head);
+}
+
+struct hlist_head __kprobes *
+kretprobe_inst_table_head (struct task_struct *tsk)
+{
+ return &kretprobe_inst_table[hash_ptr (tsk, KPROBE_HASH_BITS)];
+}
+
+/*
+ * This function is called from finish_task_switch when task tk becomes dead,
+ * so that we can recycle any function-return probe instances associated
+ * with this task. These left over instances represent probed functions
+ * that have been called but will never return.
+ */
+/*void __kprobes kprobe_flush_task(struct task_struct *tk)
+{
+ struct kretprobe_instance *ri;
+ struct hlist_head *head, empty_rp;
+ struct hlist_node *node, *tmp;
+ unsigned long flags = 0;
+
+ INIT_HLIST_HEAD(&empty_rp);
+ spin_lock_irqsave(&kretprobe_lock, flags);
+ head = kretprobe_inst_table_head(tk);
+ hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
+ if (ri->task == tk)
+ recycle_rp_inst(ri, &empty_rp);
+ }
+ spin_unlock_irqrestore(&kretprobe_lock, flags);
+
+ hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
+ hlist_del(&ri->hlist);
+ kfree(ri);
+ }
+}*/
+
+static inline void
+free_rp_inst (struct kretprobe *rp)
+{
+ struct kretprobe_instance *ri;
+ while ((ri = get_free_rp_inst (rp)) != NULL)
+ {
+ hlist_del (&ri->uflist);
+ kfree (ri);
+ }
+}
+
+/*
+ * Keep all fields in the kprobe consistent
+ */
+static inline void
+copy_kprobe (struct kprobe *old_p, struct kprobe *p)
+{
+ memcpy (&p->opcode, &old_p->opcode, sizeof (kprobe_opcode_t));
+ memcpy (&p->ainsn, &old_p->ainsn, sizeof (struct arch_specific_insn));
+ p->tgid = old_p->tgid;
+ p->ss_addr = old_p->ss_addr;
+ //p->spid = old_p->spid;
+}
+
+/*
+* Add the new probe to old_p->list. Fail if this is the
+* second jprobe at the address - two jprobes can't coexist
+*/
+static int __kprobes
+add_new_kprobe (struct kprobe *old_p, struct kprobe *p)
+{
+ if (p->break_handler)
+ {
+ if (old_p->break_handler)
+ return -EEXIST;
+ list_add_tail_rcu (&p->list, &old_p->list);
+ old_p->break_handler = aggr_break_handler;
+ }
+ else
+ list_add_rcu (&p->list, &old_p->list);
+ if (p->post_handler && !old_p->post_handler)
+ old_p->post_handler = aggr_post_handler;
+ return 0;
+}
+
+/*
+ * Fill in the required fields of the "manager kprobe". Replace the
+ * earlier kprobe in the hlist with the manager kprobe
+ */
+static inline void
+add_aggr_kprobe (struct kprobe *ap, struct kprobe *p)
+{
+ copy_kprobe (p, ap);
+ flush_insn_slot (ap);
+ ap->addr = p->addr;
+ ap->pre_handler = aggr_pre_handler;
+ ap->fault_handler = aggr_fault_handler;
+ if (p->post_handler)
+ ap->post_handler = aggr_post_handler;
+ if (p->break_handler)
+ ap->break_handler = aggr_break_handler;
+
+ INIT_LIST_HEAD (&ap->list);
+ list_add_rcu (&p->list, &ap->list);
+
+ src_hlist_replace_rcu (&p->hlist, &ap->hlist);
+}
+
+/*
+ * This is the second or subsequent kprobe at the address - handle
+ * the intricacies
+ */
+static int __kprobes
+register_aggr_kprobe (struct kprobe *old_p, struct kprobe *p)
+{
+ int ret = 0;
+ struct kprobe *ap;
+ DBPRINTF ("start\n");
+
+ DBPRINTF ("p = %p old_p = %p \n", p, old_p);
+ if (old_p->pre_handler == aggr_pre_handler)
+ {
+ DBPRINTF ("aggr_pre_handler \n");
+
+ copy_kprobe (old_p, p);
+ ret = add_new_kprobe (old_p, p);
+ }
+ else
+ {
+ DBPRINTF ("kzalloc\n");
+
+#ifdef kzalloc
+ ap = kzalloc (sizeof (struct kprobe), GFP_KERNEL);
+#else
+ ap = kmalloc (sizeof (struct kprobe), GFP_KERNEL);
+ if (ap)
+ memset (ap, 0, sizeof (struct kprobe));
+#endif
+ if (!ap)
+ return -ENOMEM;
+ add_aggr_kprobe (ap, old_p);
+ copy_kprobe (ap, p);
+ DBPRINTF ("ap = %p p = %p old_p = %p \n", ap, p, old_p);
+ ret = add_new_kprobe (ap, p);
+ }
+ return ret;
+}
+
+static int __kprobes
+__register_kprobe (struct kprobe *p, unsigned long called_from, int atomic)
+{
+ struct kprobe *old_p;
+// struct module *probed_mod;
+ int ret = 0;
+ /*
+ * If we have a symbol_name argument look it up,
+ * and add it to the address. That way the addr
+ * field can either be global or relative to a symbol.
+ */
+ if (p->symbol_name)
+ {
+ if (p->addr)
+ return -EINVAL;
+ kprobe_lookup_name (p->symbol_name, p->addr);
+ }
+
+ if (!p->addr)
+ return -EINVAL;
+ DBPRINTF ("p->addr = 0x%p\n", p->addr);
+ p->addr = (kprobe_opcode_t *) (((char *) p->addr) + p->offset);
+ DBPRINTF ("p->addr = 0x%p p = 0x%p\n", p->addr, p);
+
+/* if ((!kernel_text_address((unsigned long) p->addr)) ||
+ in_kprobes_functions((unsigned long) p->addr))
+ return -EINVAL;*/
+
+#ifdef KPROBES_PROFILE
+ p->start_tm.tv_sec = p->start_tm.tv_usec = 0;
+ p->hnd_tm_sum.tv_sec = p->hnd_tm_sum.tv_usec = 0;
+ p->count = 0;
+#endif
+ p->mod_refcounted = 0;
+ //p->proc_prio = 0;
+ //p->proc_sched = 0;
+ //p->spid = -1;
+ //p->irq = 0;
+ //p->task_flags = 0;
+/*
+ // Check are we probing a module
+ if ((probed_mod = module_text_address((unsigned long) p->addr))) {
+ struct module *calling_mod = module_text_address(called_from);
+ // We must allow modules to probe themself and
+ // in this case avoid incrementing the module refcount,
+ // so as to allow unloading of self probing modules.
+ //
+ if (calling_mod && (calling_mod != probed_mod)) {
+ if (unlikely(!try_module_get(probed_mod)))
+ return -EINVAL;
+ p->mod_refcounted = 1;
+ } else
+ probed_mod = NULL;
+ }
+*/
+ p->nmissed = 0;
+// mutex_lock(&kprobe_mutex);
+ old_p = get_kprobe (p->addr, 0, NULL);
+ if (old_p)
+ {
+ ret = register_aggr_kprobe (old_p, p);
+ if (!ret)
+ atomic_inc (&kprobe_count);
+ goto out;
+ }
+
+ if ((ret = arch_prepare_kprobe (p)) != 0)
+ goto out;
+
+ DBPRINTF ("before out ret = 0x%x\n", ret);
+
+ INIT_HLIST_NODE (&p->hlist);
+ hlist_add_head_rcu (&p->hlist, &kprobe_table[hash_ptr (p->addr, KPROBE_HASH_BITS)]);
+
+/* if (atomic_add_return(1, &kprobe_count) == \
+ (ARCH_INACTIVE_KPROBE_COUNT + 1))
+ register_page_fault_notifier(&kprobe_page_fault_nb);*/
+
+ arch_arm_kprobe (p);
+
+ out:
+// mutex_unlock(&kprobe_mutex);
+/*
+ if (ret && probed_mod)
+ module_put(probed_mod);
+*/
+ DBPRINTF ("out ret = 0x%x\n", ret);
+
+ return ret;
+}
+
+static int __kprobes
+__register_uprobe (struct kprobe *p, struct task_struct *task, int atomic, unsigned long called_from)
+{
+ int ret = 0;
+ struct kprobe *old_p;
+
+ if (!p->addr)
+ return -EINVAL;
+
+ DBPRINTF ("p->addr = 0x%p p = 0x%p\n", p->addr, p);
+
+ p->mod_refcounted = 0;
+ p->nmissed = 0;
+#ifdef KPROBES_PROFILE
+ p->start_tm.tv_sec = p->start_tm.tv_usec = 0;
+ p->hnd_tm_sum.tv_sec = p->hnd_tm_sum.tv_usec = 0;
+ p->count = 0;
+#endif
+
+ // get the first item
+ old_p = get_kprobe (p->addr, p->tgid, NULL);
+ if (old_p)
+ {
+ ret = register_aggr_kprobe (old_p, p);
+ if (!ret)
+ atomic_inc (&kprobe_count);
+ goto out;
+ }
+ if ((ret = arch_prepare_uprobe (p, task, atomic)) != 0)
+ {
+ goto out;
+ }
+
+ DBPRINTF ("before out ret = 0x%x\n", ret);
+
+ INIT_HLIST_NODE (&p->hlist);
+ hlist_add_head_rcu (&p->hlist, &kprobe_table[hash_ptr (p->addr, KPROBE_HASH_BITS)]);
+
+ INIT_HLIST_NODE (&p->is_hlist);
+ hlist_add_head_rcu (&p->is_hlist, &uprobe_insn_slot_table[hash_ptr (p->ainsn.insn, KPROBE_HASH_BITS)]);
+
+ arch_arm_uprobe (p, task);
+out:
+ DBPRINTF ("out ret = 0x%x\n", ret);
+
+ return ret;
+}
+
+void __kprobes
+unregister_uprobe (struct kprobe *p, struct task_struct *task, int atomic)
+{
+ unregister_kprobe (p, task, atomic);
+}
+
+int __kprobes
+register_kprobe (struct kprobe *p, int atomic)
+{
+ return __register_kprobe (p, (unsigned long) __builtin_return_address (0), atomic);
+}
+
+void __kprobes
+unregister_kprobe (struct kprobe *p, struct task_struct *task, int atomic)
+{
+// struct module *mod;
+ struct kprobe *old_p, *list_p;
+ int cleanup_p, pid = 0;
+
+// mutex_lock(&kprobe_mutex);
+
+ pid = p->tgid;
+
+ old_p = get_kprobe (p->addr, pid, NULL);
+ DBPRINTF ("unregister_kprobe p=%p old_p=%p", p, old_p);
+ if (unlikely (!old_p))
+ {
+// mutex_unlock(&kprobe_mutex);
+ return;
+ }
+ if (p != old_p)
+ {
+ list_for_each_entry_rcu (list_p, &old_p->list, list)
+ if (list_p == p)
+ /* kprobe p is a valid probe */
+ goto valid_p;
+// mutex_unlock(&kprobe_mutex);
+ return;
+ }
+valid_p:
+ DBPRINTF ("unregister_kprobe valid_p");
+ if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) &&
+ (p->list.next == &old_p->list) && (p->list.prev == &old_p->list)))
+ {
+ /* Only probe on the hash list */
+ DBPRINTF ("unregister_kprobe disarm pid=%d", pid);
+ if (pid)
+ arch_disarm_uprobe (p, task);//vma, page, kaddr);
+ else
+ arch_disarm_kprobe (p);
+ hlist_del_rcu (&old_p->hlist);
+ cleanup_p = 1;
+ }
+ else
+ {
+ list_del_rcu (&p->list);
+ cleanup_p = 0;
+ }
+ DBPRINTF ("unregister_kprobe cleanup_p=%d", cleanup_p);
+// mutex_unlock(&kprobe_mutex);
+
+// synchronize_sched();
+/*
+ if (p->mod_refcounted &&
+ (mod = module_text_address((unsigned long)p->addr)))
+ module_put(mod);
+*/
+ if (cleanup_p)
+ {
+ if (p != old_p)
+ {
+ list_del_rcu (&p->list);
+ kfree (old_p);
+ }
+ arch_remove_kprobe (p, task);
+ }
+ else
+ {
+/// mutex_lock(&kprobe_mutex);
+ if (p->break_handler)
+ old_p->break_handler = NULL;
+ if (p->post_handler)
+ {
+ list_for_each_entry_rcu (list_p, &old_p->list, list)
+ {
+ if (list_p->post_handler)
+ {
+ cleanup_p = 2;
+ break;
+ }
+ }
+ if (cleanup_p == 0)
+ old_p->post_handler = NULL;
+ }
+// mutex_unlock(&kprobe_mutex);
+ }
+
+ /* Call unregister_page_fault_notifier()
+ * if no probes are active
+ */
+// mutex_lock(&kprobe_mutex);
+/* if (atomic_add_return(-1, &kprobe_count) == \
+ ARCH_INACTIVE_KPROBE_COUNT)
+ unregister_page_fault_notifier(&kprobe_page_fault_nb);*/
+// mutex_unlock(&kprobe_mutex);
+ return;
+}
+
+int __kprobes
+register_ujprobe (struct task_struct *task, struct mm_struct *mm, struct jprobe *jp, int atomic)
+{
+#ifdef _DEBUG
+ gSilent = 0;
+#endif
+ /* Todo: Verify probepoint is a function entry point */
+ jp->kp.pre_handler = setjmp_pre_handler;
+ jp->kp.break_handler = longjmp_break_handler;
+
+ int ret = __register_uprobe (&jp->kp, task, atomic,
+ (unsigned long) __builtin_return_address (0));
+
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+ return ret;
+}
+
+void __kprobes
+unregister_ujprobe (struct task_struct *task, struct jprobe *jp, int atomic)
+{
+ unregister_uprobe (&jp->kp, task, atomic);
+}
+
+int __kprobes
+register_jprobe (struct jprobe *jp, int atomic)
+{
+ /* Todo: Verify probepoint is a function entry point */
+ jp->kp.pre_handler = setjmp_pre_handler;
+ jp->kp.break_handler = longjmp_break_handler;
+
+ return __register_kprobe (&jp->kp, (unsigned long) __builtin_return_address (0), atomic);
+}
+
+void __kprobes
+unregister_jprobe (struct jprobe *jp, int atomic)
+{
+ unregister_kprobe (&jp->kp, 0, atomic);
+}
+
+/*
+ * This kprobe pre_handler is registered with every kretprobe. When probe
+ * hits it will set up the return probe.
+ */
+static int __kprobes
+pre_handler_kretprobe (struct kprobe *p, struct pt_regs *regs /*, struct vm_area_struct **vma,
+ struct page **page, unsigned long **kaddr */ )
+{
+ struct kretprobe *rp = container_of (p, struct kretprobe, kp);
+ unsigned long flags = 0;
+ DBPRINTF ("START\n");
+
+ /*TODO: consider to only swap the RA after the last pre_handler fired */
+ spin_lock_irqsave (&kretprobe_lock, flags);
+ if (!rp->disarm)
+ __arch_prepare_kretprobe (rp, regs);
+ spin_unlock_irqrestore (&kretprobe_lock, flags);
+ DBPRINTF ("END\n");
+ return 0;
+}
+
+struct kretprobe *sched_rp;
+
+int __kprobes
+register_kretprobe (struct kretprobe *rp, int atomic)
+{
+ int ret = 0;
+ struct kretprobe_instance *inst;
+ int i;
+ DBPRINTF ("START");
+
+ rp->kp.pre_handler = pre_handler_kretprobe;
+ rp->kp.post_handler = NULL;
+ rp->kp.fault_handler = NULL;
+ rp->kp.break_handler = NULL;
+
+ rp->disarm = 0;
+
+ /* Pre-allocate memory for max kretprobe instances */
+ if(rp->kp.addr == sched_addr)
+ rp->maxactive = 1000;//max (100, 2 * NR_CPUS);
+ else if (rp->maxactive <= 0)
+ {
+#if 1//def CONFIG_PREEMPT
+ rp->maxactive = max (10, 2 * NR_CPUS);
+#else
+ rp->maxactive = NR_CPUS;
+#endif
+ }
+ INIT_HLIST_HEAD (&rp->used_instances);
+ INIT_HLIST_HEAD (&rp->free_instances);
+ for (i = 0; i < rp->maxactive; i++)
+ {
+ inst = kmalloc (sizeof (struct kretprobe_instance), GFP_KERNEL);
+ if (inst == NULL)
+ {
+ free_rp_inst (rp);
+ return -ENOMEM;
+ }
+ INIT_HLIST_NODE (&inst->uflist);
+ hlist_add_head (&inst->uflist, &rp->free_instances);
+ }
+
+ DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
+ rp->nmissed = 0;
+ /* Establish function entry probe point */
+ if ((ret = __register_kprobe (&rp->kp, (unsigned long) __builtin_return_address (0), atomic)) != 0)
+ free_rp_inst (rp);
+
+ DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
+ if(rp->kp.addr == sched_addr)
+ sched_rp = rp;
+
+ return ret;
+}
+
+void __kprobes
+unregister_kretprobe (struct kretprobe *rp, int atomic)
+{
+ unsigned long flags;
+ struct kretprobe_instance *ri;
+
+ //printk("addr=%p, *addr=[%lx %lx %lx]\n", rp->kp.addr,
+ // *(rp->kp.addr), *(rp->kp.addr+1), *(rp->kp.addr+2));
+ unregister_kprobe (&rp->kp, 0, atomic);
+
+ if(rp->kp.addr == sched_addr)
+ sched_rp = NULL;
+
+ //printk("addr=%p, *addr=[%lx %lx %lx]\n", rp->kp.addr,
+ // *(rp->kp.addr), *(rp->kp.addr+1), *(rp->kp.addr+2));
+ /* No race here */
+ spin_lock_irqsave (&kretprobe_lock, flags);
+ while ((ri = get_used_rp_inst (rp)) != NULL)
+ {
+ ri->rp = NULL;
+ hlist_del (&ri->uflist);
+ }
+ spin_unlock_irqrestore (&kretprobe_lock, flags);
+ free_rp_inst (rp);
+}
+
+int __kprobes
+register_uretprobe (struct task_struct *task, struct mm_struct *mm, struct kretprobe *rp, int atomic)
+{
+ int ret = 0;
+ struct kretprobe_instance *inst;
+ /*struct page *pages[2] = {0, 0};
+ struct vm_area_struct *vmas[2] = {0, 0};
+ unsigned long *kaddrs[2] = {0, 0}; */
+ int i;
+#ifdef _DEBUG
+ gSilent = 0;
+#endif
+
+ DBPRINTF ("START\n");
+
+ rp->kp.pre_handler = pre_handler_kretprobe;
+ rp->kp.post_handler = NULL;
+ rp->kp.fault_handler = NULL;
+ rp->kp.break_handler = NULL;
+
+ rp->disarm = 0;
+
+ /* Pre-allocate memory for max kretprobe instances */
+ if (rp->maxactive <= 0)
+ {
+#if 1//def CONFIG_PREEMPT
+ rp->maxactive = max (10, 2 * NR_CPUS);
+#else
+ rp->maxactive = NR_CPUS;
+#endif
+ }
+ INIT_HLIST_HEAD (&rp->used_instances);
+ INIT_HLIST_HEAD (&rp->free_instances);
+ for (i = 0; i < rp->maxactive; i++)
+ {
+ inst = kmalloc (sizeof (struct kretprobe_instance), GFP_KERNEL);
+ if (inst == NULL)
+ {
+ free_rp_inst (rp);
+ ret = -ENOMEM;
+ goto out;
+ }
+ INIT_HLIST_NODE (&inst->uflist);
+ hlist_add_head (&inst->uflist, &rp->free_instances);
+ }
+
+ rp->nmissed = 0;
+#if 0
+ if (atomic)
+ ret = get_user_pages_atomic (task, mm, (unsigned long) rp->kp.addr, 1, 1, 1, &pages[0], &vmas[0]);
+ else
+ ret = get_user_pages (task, mm, (unsigned long) rp->kp.addr, 1, 1, 1, &pages[0], &vmas[0]);
+ if (ret <= 0)
+ {
+ DBPRINTF ("get_user_pages for %p failed!", rp->kp.addr);
+ ret = -EFAULT;
+ goto out;
+ }
+ if (atomic)
+ kaddrs[0] = kmap_atomic (pages[0], KM_USER0) + ((unsigned long) rp->kp.addr & ~PAGE_MASK);
+ else
+ kaddrs[0] = kmap (pages[0]) + ((unsigned long) rp->kp.addr & ~PAGE_MASK);
+ // if 2nd instruction is on the 2nd page
+ if ((((unsigned long) (rp->kp.addr + 1)) & ~PAGE_MASK) == 0)
+ {
+ if (atomic)
+ ret = get_user_pages_atomic (task, mm, (unsigned long) (rp->kp.addr + 1), 1, 1, 1, &pages[1], &vmas[1]);
+ else
+ ret = get_user_pages (task, mm, (unsigned long) (rp->kp.addr + 1), 1, 1, 1, &pages[1], &vmas[1]);
+ if (ret <= 0)
+ {
+ DBPRINTF ("get_user_pages for %p failed!", rp->kp.addr + 1);
+ ret = -EFAULT;
+ goto out;
+ }
+ if (atomic)
+ kaddrs[1] = kmap_atomic (pages[1], KM_USER1) + ((unsigned long) (rp->kp.addr + 1) & ~PAGE_MASK);
+ else
+ kaddrs[1] = kmap (pages[1]) + ((unsigned long) (rp->kp.addr + 1) & ~PAGE_MASK);
+ }
+ else
+ {
+ // 2nd instruction is on the 1st page too
+ vmas[1] = vmas[0];
+ pages[1] = pages[0];
+ kaddrs[1] = kaddrs[0] + 1;
+ }
+#endif
+ /* Establish function exit probe point */
+ if ((ret = arch_prepare_uretprobe (rp, task/*vmas, pages, kaddrs */ )) != 0)
+ goto out;
+ /* Establish function entry probe point */
+ if ((ret = __register_uprobe (&rp->kp, task, atomic,
+ (unsigned long) __builtin_return_address (0))) != 0)
+ {
+ free_rp_inst (rp);
+ goto out;
+ }
+
+ arch_arm_uretprobe (rp, task);//vmas[1], pages[1], kaddrs[1]);
+#if 0
+ if (atomic)
+ set_page_dirty (pages[1]);
+ else
+ set_page_dirty_lock (pages[1]);
+#endif
+ out:
+#if 0
+ if (pages[0])
+ {
+ if (kaddrs[0])
+ {
+ if (atomic)
+ kunmap_atomic (kaddrs[0] - ((unsigned long) rp->kp.addr & ~PAGE_MASK), KM_USER0);
+ else
+ kunmap (pages[0]);
+ }
+ page_cache_release (pages[0]);
+ }
+ if ((pages[0] != pages[1]))
+ {
+ if (pages[1])
+ {
+ if (kaddrs[1])
+ {
+ if (atomic)
+ kunmap_atomic (kaddrs[1] - ((unsigned long) (rp->kp.addr + 1) & ~PAGE_MASK), KM_USER1);
+ else
+ kunmap (pages[1]);
+ }
+ page_cache_release (pages[1]);
+ }
+ }
+ /*else if( (pages[0] != pages[2]) ){
+ if(pages[2]){
+ if(kaddrs[2]) {
+ if (atomic) kunmap_atomic(kaddrs[2], KM_USER1);
+ else kunmap(pages[2]);
+ }
+ page_cache_release(pages[2]);
+ }
+ } */
+#endif
+
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+ return ret;
+}
+
+static struct kretprobe *__kprobes
+clone_kretprobe (struct kretprobe *rp)
+{
+ struct kprobe *old_p;
+ struct kretprobe *clone = NULL;
+ int ret;
+
+ clone = kmalloc (sizeof (struct kretprobe), GFP_KERNEL);
+ if (!clone)
+ {
+ DBPRINTF ("failed to alloc memory for clone probe %p!", rp->kp.addr);
+ return NULL;
+ }
+ memcpy (clone, rp, sizeof (struct kretprobe));
+ clone->kp.pre_handler = pre_handler_kretprobe;
+ clone->kp.post_handler = NULL;
+ clone->kp.fault_handler = NULL;
+ clone->kp.break_handler = NULL;
+ old_p = get_kprobe (rp->kp.addr, rp->kp.tgid, NULL);
+ if (old_p)
+ {
+ ret = register_aggr_kprobe (old_p, &clone->kp);
+ if (ret)
+ {
+ kfree (clone);
+ return NULL;
+ }
+ atomic_inc (&kprobe_count);
+ }
+
+ return clone;
+}
+
+void __kprobes
+unregister_uretprobe (struct task_struct *task, struct kretprobe *rp, int atomic)
+{
+ //int ret = 0;
+ unsigned long flags;
+ struct kretprobe_instance *ri;
+ struct kretprobe *rp2 = NULL;
+ /*struct mm_struct *mm;
+ struct page *pages[2] = {0, 0};
+ struct vm_area_struct *vmas[2] = {0, 0};
+ unsigned long *kaddrs[2] = {0, 0}; */
+
+#ifdef _DEBUG
+ gSilent = 0;
+#endif
+#if 0
+ mm = atomic ? task->active_mm : get_task_mm (task);
+ if (!mm)
+ {
+ DBPRINTF ("task %u has no mm!", task->pid);
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+ return;
+ }
+ if (atomic)
+ ret = get_user_pages_atomic (task, mm, (unsigned long) rp->kp.addr, 1, 1, 1, &pages[0], &vmas[0]);
+ else
+ {
+ down_read (&mm->mmap_sem);
+ ret = get_user_pages (task, mm, (unsigned long) rp->kp.addr, 1, 1, 1, &pages[0], &vmas[0]);
+ }
+ if (ret <= 0)
+ {
+ DBPRINTF ("get_user_pages for %p failed!", rp->kp.addr);
+ goto out;
+ }
+ if (atomic)
+ kaddrs[0] = kmap_atomic (pages[0], KM_USER0) + ((unsigned long) rp->kp.addr & ~PAGE_MASK);
+ else
+ kaddrs[0] = kmap (pages[0]) + ((unsigned long) rp->kp.addr & ~PAGE_MASK);
+ if ((((unsigned long) (rp->kp.addr + 1)) & ~PAGE_MASK) == 0)
+ {
+ if (atomic)
+ ret = get_user_pages_atomic (task, mm, (unsigned long) (rp->kp.addr + 1), 1, 1, 1, &pages[1], &vmas[1]);
+ else
+ ret = get_user_pages (task, mm, (unsigned long) (rp->kp.addr + 1), 1, 1, 1, &pages[1], &vmas[1]);
+ if (ret <= 0)
+ {
+ DBPRINTF ("get_user_pages for %p failed!", rp->kp.addr + 1);
+ goto out;
+ }
+ if (atomic)
+ kaddrs[1] = kmap_atomic (pages[1], KM_USER1) + ((unsigned long) (rp->kp.addr + 1) & ~PAGE_MASK);
+ else
+ kaddrs[1] = kmap (pages[1]) + ((unsigned long) (rp->kp.addr + 1) & ~PAGE_MASK);
+ }
+ else
+ {
+ vmas[1] = vmas[0];
+ pages[1] = pages[0];
+ kaddrs[1] = kaddrs[0] + 1;
+ }
+
+ /* No race here */
+ DBPRINTF ("unregister_uretprobe1 addr %p [%lx %lx]", rp->kp.addr, *kaddrs[0], *kaddrs[1]);
+#endif
+ spin_lock_irqsave (&kretprobe_lock, flags);
+ if (hlist_empty (&rp->used_instances))
+ {
+ // if there are no used retprobe instances (i.e. function is not entered) - disarm retprobe
+ arch_disarm_uretprobe (rp, task);//vmas[1], pages[1], kaddrs[1]);
+#if 0
+ if (atomic)
+ set_page_dirty (pages[1]);
+ else
+ set_page_dirty_lock (pages[1]);
+#endif
+ }
+ else
+ {
+ rp2 = clone_kretprobe (rp);
+ if (!rp2)
+ DBPRINTF ("unregister_uretprobe addr %p: failed to clone retprobe!", rp->kp.addr);
+ else
+ {
+ DBPRINTF ("initiating deferred retprobe deletion addr %p", rp->kp.addr);
+ printk ("initiating deferred retprobe deletion addr %p\n", rp->kp.addr);
+ rp2->disarm = 1;
+ }
+ }
+
+ while ((ri = get_used_rp_inst (rp)) != NULL)
+ {
+ ri->rp = NULL;
+ ri->rp2 = rp2;
+ hlist_del (&ri->uflist);
+ }
+ spin_unlock_irqrestore (&kretprobe_lock, flags);
+ free_rp_inst (rp);
+
+ unregister_uprobe (&rp->kp, task, atomic);
+ //DBPRINTF("unregister_uretprobe3 addr %p [%lx %lx]",
+ // rp->kp.addr, *kaddrs[0], *kaddrs[1]);
+#if 0
+ out:
+ if (pages[0])
+ {
+ if (kaddrs[0])
+ {
+ if (atomic)
+ kunmap_atomic (kaddrs[0] - ((unsigned long) rp->kp.addr & ~PAGE_MASK), KM_USER0);
+ else
+ kunmap (pages[0]);
+ }
+ page_cache_release (pages[0]);
+ }
+ if (pages[1] && (pages[0] != pages[1]))
+ {
+ if (kaddrs[1])
+ {
+ if (atomic)
+ kunmap_atomic (kaddrs[1] - ((unsigned long) (rp->kp.addr + 1) & ~PAGE_MASK), KM_USER1);
+ else
+ kunmap (pages[1]);
+ }
+ page_cache_release (pages[1]);
+ }
+ if (!atomic)
+ {
+ up_read (&mm->mmap_sem);
+ mmput (mm);
+ }
+#endif
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+}
+
+void __kprobes
+unregister_all_uprobes (struct task_struct *task, int atomic)
+{
+ struct hlist_head *head;
+ struct hlist_node *node, *tnode;
+ struct kprobe *p;
+ int i;
+
+ for(i = 0; i < KPROBE_TABLE_SIZE; i++){
+ head = &kprobe_table[i];
+ hlist_for_each_entry_safe (p, node, tnode, head, hlist){
+ if(p->tgid == task->tgid){
+ printk("unregister_all_uprobes: delete uprobe at %pf for %s/%d\n", p->addr, task->comm, task->pid);
+ unregister_uprobe (p, task, atomic);
+ }
+ }
+ }
+ purge_garbage_uslots(task, atomic);
+}
+
+#if 0
+int
+access_process_vm (struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
+{
+ struct mm_struct *mm;
+ struct vm_area_struct *vma;
+ struct page *page;
+ void *old_buf = buf;
+
+ mm = get_task_mm (tsk);
+ if (!mm)
+ return 0;
+
+ down_read (&mm->mmap_sem);
+ /* ignore errors, just check how much was sucessfully transfered */
+ while (len)
+ {
+ int bytes, ret, offset;
+ void *maddr;
+
+ ret = get_user_pages (tsk, mm, addr, 1, write, 1, &page, &vma);
+ if (ret <= 0)
+ break;
+
+ bytes = len;
+ offset = addr & (PAGE_SIZE - 1);
+ if (bytes > PAGE_SIZE - offset)
+ bytes = PAGE_SIZE - offset;
+
+ maddr = kmap (page); //, KM_USER0);
+ if (write)
+ {
+ copy_to_user_page (vma, page, addr, maddr + offset, buf, bytes);
+ set_page_dirty_lock (page);
+ }
+ else
+ {
+ copy_from_user_page (vma, page, addr, buf, maddr + offset, bytes);
+ }
+ kunmap (page); //, KM_USER0);
+ page_cache_release (page);
+ len -= bytes;
+ buf += bytes;
+ addr += bytes;
+ }
+ up_read (&mm->mmap_sem);
+ mmput (mm);
+
+ return buf - old_buf;
+}
+#endif
+
+#ifdef CONFIG_DEBUG_FS
+const char *(*__real_kallsyms_lookup) (unsigned long addr, unsigned long *symbolsize, unsigned long *offset, char **modname, char *namebuf);
+const char *
+kallsyms_lookup (unsigned long addr, unsigned long *symbolsize, unsigned long *offset, char **modname, char *namebuf)
+{
+ return __real_kallsyms_lookup (addr, symbolsize, offset, modname, namebuf);
+}
+
+static void __kprobes
+report_probe (struct seq_file *pi, struct kprobe *p, const char *sym, int offset, char *modname)
+{
+ char *kprobe_type;
+
+ if (p->pre_handler == pre_handler_kretprobe)
+ if (p->tgid)
+ kprobe_type = "ur";
+ else
+ kprobe_type = "r";
+ else if (p->pre_handler == setjmp_pre_handler)
+ if (p->tgid)
+ kprobe_type = "uj";
+ else
+ kprobe_type = "j";
+ else if (p->tgid)
+ kprobe_type = "u";
+ else
+ kprobe_type = "k";
+ if (sym)
+ seq_printf (pi, "%p %s %s+0x%x %s\n", p->addr, kprobe_type, sym, offset, (modname ? modname : " "));
+ else
+ seq_printf (pi, "%p %s %p\n", p->addr, kprobe_type, p->addr);
+}
+
+static void __kprobes *
+kprobe_seq_start (struct seq_file *f, loff_t * pos)
+{
+ return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
+}
+
+static void __kprobes *
+kprobe_seq_next (struct seq_file *f, void *v, loff_t * pos)
+{
+ (*pos)++;
+ if (*pos >= KPROBE_TABLE_SIZE)
+ return NULL;
+ return pos;
+}
+
+static void __kprobes
+kprobe_seq_stop (struct seq_file *f, void *v)
+{
+ /* Nothing to do */
+}
+
+struct us_proc_ip
+{
+ char *name;
+ int installed;
+ struct jprobe jprobe;
+ struct kretprobe retprobe;
+ unsigned long offset;
+};
+
+static int __kprobes
+show_kprobe_addr (struct seq_file *pi, void *v)
+{
+ struct hlist_head *head;
+ struct hlist_node *node;
+ struct kprobe *p, *kp;
+ const char *sym = NULL;
+ unsigned int i = *(loff_t *) v;
+ unsigned long size, offset = 0;
+ char *modname, namebuf[128];
+
+ head = &kprobe_table[i];
+ preempt_disable ();
+ hlist_for_each_entry_rcu (p, node, head, hlist)
+ {
+ /*if(p->pid){
+ struct us_proc_ip *up = NULL;
+ if (p->pre_handler == pre_handler_kretprobe){
+ struct kretprobe *rp = container_of(p, struct kretprobe, kp);
+ up = container_of(rp, struct us_proc_ip, retprobe);
+ }
+ else {//if (p->pre_handler == setjmp_pre_handler){
+ struct jprobe *jp = container_of(p, struct jprobe, kp);
+ up = container_of(jp, struct us_proc_ip, jprobe);
+ }
+ if(up){
+ sym = up->name;
+ printk("show_kprobe_addr: %s\n", sym);
+ }
+ }
+ else */
+ sym = kallsyms_lookup ((unsigned long) p->addr, &size, &offset, &modname, namebuf);
+ if (p->pre_handler == aggr_pre_handler)
+ {
+ list_for_each_entry_rcu (kp, &p->list, list) report_probe (pi, kp, sym, offset, modname);
+ }
+ else
+ report_probe (pi, p, sym, offset, modname);
+ }
+ //seq_printf (pi, "handled exceptions %lu\n", handled_exceptions);
+ preempt_enable ();
+ return 0;
+}
+
+static struct seq_operations kprobes_seq_ops = {
+ .start = kprobe_seq_start,
+ .next = kprobe_seq_next,
+ .stop = kprobe_seq_stop,
+ .show = show_kprobe_addr
+};
+
+static int __kprobes
+kprobes_open (struct inode *inode, struct file *filp)
+{
+ return seq_open (filp, &kprobes_seq_ops);
+}
+
+static struct file_operations debugfs_kprobes_operations = {
+ .open = kprobes_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release,
+};
+
+#ifdef KPROBES_PROFILE
+extern unsigned long nCount;
+extern struct timeval probe_enter_diff_sum;
+static void __kprobes *
+kprobe_prof_seq_start (struct seq_file *f, loff_t * pos)
+{
+ return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
+}
+
+static void __kprobes *
+kprobe_prof_seq_next (struct seq_file *f, void *v, loff_t * pos)
+{
+ (*pos)++;
+ if (*pos >= KPROBE_TABLE_SIZE)
+ return NULL;
+ return pos;
+}
+
+static void __kprobes
+kprobe_prof_seq_stop (struct seq_file *f, void *v)
+{
+}
+
+static void __kprobes
+report_probe_prof (struct seq_file *pi, struct kprobe *p, const char *sym, int offset, char *modname)
+{
+ char *kprobe_type;
+
+ if (p->pre_handler == pre_handler_kretprobe)
+ if (p->pid)
+ kprobe_type = "ur";
+ else
+ kprobe_type = "r";
+ else if (p->pre_handler == setjmp_pre_handler)
+ if (p->pid)
+ kprobe_type = "uj";
+ else
+ kprobe_type = "j";
+ else if (p->pid)
+ kprobe_type = "u";
+ else
+ kprobe_type = "k";
+
+ if (sym)
+ seq_printf (pi, "%p %s %s+0x%x %s %lu.%06ld\n", p->addr, kprobe_type,
+ sym, offset, (modname ? modname : " "), p->count ? p->hnd_tm_sum.tv_sec / p->count : 0, p->count ? p->hnd_tm_sum.tv_usec / p->count : 0);
+ else
+
+ seq_printf (pi, "%p %s %p %lu.%06ld\n", p->addr, kprobe_type, p->addr, p->count ? p->hnd_tm_sum.tv_sec / p->count : 0, p->count ? p->hnd_tm_sum.tv_usec / p->count : 0);
+}
+
+static int __kprobes
+show_kprobe_prof (struct seq_file *pi, void *v)
+{
+ struct hlist_head *head;
+ struct hlist_node *node;
+ struct kprobe *p; //, *kp;
+ const char *sym = NULL;
+ unsigned int i = *(loff_t *) v;
+ unsigned long size, offset = 0;
+ char *modname, namebuf[128];
+ static struct timeval utv, ktv;
+ static unsigned long ucount, kcount;
+
+ head = &kprobe_table[i];
+ preempt_disable ();
+ hlist_for_each_entry_rcu (p, node, head, hlist)
+ {
+ sym = kallsyms_lookup ((unsigned long) p->addr, &size, &offset, &modname, namebuf);
+ /*if (p->pre_handler == aggr_pre_handler) {
+ list_for_each_entry_rcu(kp, &p->list, list)
+ report_probe_prof(pi, kp, sym, offset, modname);
+ } else */
+ report_probe_prof (pi, p, sym, offset, modname);
+ if (p->count)
+ {
+ if (p->pid)
+ {
+ set_normalized_timeval (&utv, utv.tv_sec + p->hnd_tm_sum.tv_sec, utv.tv_usec + p->hnd_tm_sum.tv_usec);
+ ucount += p->count;
+ }
+ else
+ {
+ //seq_printf(pi, "kernel probe handling %lu %lu.%06ld\n",
+ // p->count, p->hnd_tm_sum.tv_sec, p->hnd_tm_sum.tv_usec);
+ //seq_printf(pi, "kernel probe handling2 %lu %lu.%06ld\n",
+ // kcount, ktv.tv_sec, ktv.tv_usec);
+ set_normalized_timeval (&ktv, ktv.tv_sec + p->hnd_tm_sum.tv_sec, ktv.tv_usec + p->hnd_tm_sum.tv_usec);
+ kcount += p->count;
+ //seq_printf(pi, "kernel probe handling3 %lu %lu.%06ld\n",
+ // kcount, ktv.tv_sec, ktv.tv_usec);
+ }
+ }
+ }
+ if (i == (KPROBE_TABLE_SIZE - 1))
+ {
+ seq_printf (pi, "Average kernel probe handling %lu.%06ld\n", kcount ? ktv.tv_sec / kcount : 0, kcount ? ktv.tv_usec / kcount : 0);
+ seq_printf (pi, "Average user probe handling %lu.%06ld\n", ucount ? utv.tv_sec / ucount : 0, ucount ? utv.tv_usec / ucount : 0);
+ seq_printf (pi, "Average probe period %lu.%06ld\n", nCount ? probe_enter_diff_sum.tv_sec / nCount : 0, nCount ? probe_enter_diff_sum.tv_usec / nCount : 0);
+ utv.tv_sec = utv.tv_usec = ktv.tv_sec = ktv.tv_usec = 0;
+ ucount = kcount = 0;
+ }
+ preempt_enable ();
+ return 0;
+}
+
+static struct seq_operations kprobes_prof_seq_ops = {
+ .start = kprobe_prof_seq_start,
+ .next = kprobe_prof_seq_next,
+ .stop = kprobe_prof_seq_stop,
+ .show = show_kprobe_prof
+};
+
+static int __kprobes
+kprobes_prof_open (struct inode *inode, struct file *filp)
+{
+ return seq_open (filp, &kprobes_prof_seq_ops);
+}
+
+static struct file_operations debugfs_kprobes_prof_operations = {
+ .open = kprobes_prof_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release,
+};
+#endif
+
+int __kprobes debugfs_kprobe_init (void);
+static struct dentry *dbg_dir, *dbg_file;
+#ifdef KPROBES_PROFILE
+static struct dentry *dbg_file_prof;
+#endif
+
+int __kprobes
+debugfs_kprobe_init (void)
+{
+ //struct dentry *dir, *file;
+
+ dbg_dir = debugfs_create_dir ("kprobes", NULL);
+ if (!dbg_dir)
+ return -ENOMEM;
+
+ dbg_file = debugfs_create_file ("list", 0444, dbg_dir, 0, &debugfs_kprobes_operations);
+ if (!dbg_file)
+ {
+ debugfs_remove (dbg_dir);
+ dbg_dir = NULL;
+ return -ENOMEM;
+ }
+
+#ifdef KPROBES_PROFILE
+ dbg_file_prof = debugfs_create_file ("prof", 0444, dbg_dir, 0, &debugfs_kprobes_prof_operations);
+ if (!dbg_file_prof)
+ {
+ debugfs_remove (dbg_file);
+ debugfs_remove (dbg_dir);
+ dbg_dir = NULL;
+ return -ENOMEM;
+ }
+#endif
+ return 0;
+}
+
+//late_initcall(debugfs_kprobe_init);
+extern unsigned long (*kallsyms_search) (const char *name);
+#endif /* CONFIG_DEBUG_FS */
+
+#if defined(CONFIG_X86)
+static struct notifier_block kprobe_exceptions_nb = {
+ .notifier_call = kprobe_exceptions_notify,
+ .priority = INT_MAX
+};
+#endif
+
+static int __init
+init_kprobes (void)
+{
+ int i, err = 0;
+
+ /* FIXME allocate the probe table, currently defined statically */
+ /* initialize all list heads */
+ for (i = 0; i < KPROBE_TABLE_SIZE; i++)
+ {
+ INIT_HLIST_HEAD (&kprobe_table[i]);
+ INIT_HLIST_HEAD (&kretprobe_inst_table[i]);
+ INIT_HLIST_HEAD (&uprobe_insn_slot_table[i]);
+ }
+ atomic_set (&kprobe_count, 0);
+
+ err = arch_init_kprobes ();
+
+ DBPRINTF ("init_kprobes: arch_init_kprobes - %d", err);
+#if defined(CONFIG_X86)
+ if (!err)
+ err = register_die_notifier (&kprobe_exceptions_nb);
+ DBPRINTF ("init_kprobes: register_die_notifier - %d", err);
+#endif // CONFIG_X86
+
+#ifdef CONFIG_DEBUG_FS
+ if (!err)
+ {
+ __real_kallsyms_lookup = (void *) kallsyms_search ("kallsyms_lookup");
+ if (!__real_kallsyms_lookup)
+ {
+ DBPRINTF ("kallsyms_lookup is not found! Oops. Where is the kernel?");
+ return -ESRCH;
+ }
+ err = debugfs_kprobe_init ();
+ DBPRINTF ("init_kprobes: debugfs_kprobe_init - %d", err);
+ }
+#endif /* CONFIG_DEBUG_FS */
+
+ return err;
+}
+
+static void __exit
+exit_kprobes (void)
+{
+#ifdef CONFIG_DEBUG_FS
+#ifdef KPROBES_PROFILE
+ if (dbg_file_prof)
+ debugfs_remove (dbg_file_prof);
+#endif
+ if (dbg_file)
+ debugfs_remove (dbg_file);
+ if (dbg_dir)
+ debugfs_remove (dbg_dir);
+#endif /* CONFIG_DEBUG_FS */
+
+#if defined(CONFIG_X86)
+ unregister_die_notifier (&kprobe_exceptions_nb);
+#endif // CONFIG_X86
+ arch_exit_kprobes ();
+}
+
+module_init (init_kprobes);
+module_exit (exit_kprobes);
+
+EXPORT_SYMBOL_GPL (register_kprobe);
+EXPORT_SYMBOL_GPL (unregister_kprobe);
+EXPORT_SYMBOL_GPL (register_jprobe);
+EXPORT_SYMBOL_GPL (unregister_jprobe);
+EXPORT_SYMBOL_GPL (register_ujprobe);
+EXPORT_SYMBOL_GPL (unregister_ujprobe);
+EXPORT_SYMBOL_GPL (jprobe_return);
+EXPORT_SYMBOL_GPL (uprobe_return);
+EXPORT_SYMBOL_GPL (register_kretprobe);
+EXPORT_SYMBOL_GPL (unregister_kretprobe);
+EXPORT_SYMBOL_GPL (register_uretprobe);
+EXPORT_SYMBOL_GPL (unregister_uretprobe);
+EXPORT_SYMBOL_GPL (unregister_all_uprobes);
+//EXPORT_SYMBOL_GPL (access_process_vm_atomic);
+#if LINUX_VERSION_CODE != KERNEL_VERSION(2,6,23)
+EXPORT_SYMBOL_GPL (access_process_vm);
+#endif
+#ifdef KERNEL_HAS_ISPAGEPRESENT
+EXPORT_SYMBOL_GPL (is_page_present);
+#else
+EXPORT_SYMBOL_GPL (page_present);
+#endif
+//EXPORT_SYMBOL_GPL(get_user_pages_atomic);
--- /dev/null
+// src_kprobes.h
+#ifndef _SRC_KPROBES_H
+#define _SRC_KPROBES_H
+
+#include <linux/list.h>
+#include <linux/notifier.h>
+#include <linux/smp.h>
+#include <linux/percpu.h>
+#include <linux/spinlock.h>
+#include <linux/rcupdate.h>
+//#include <linux/mutex.h>
+#include <linux/sched.h>
+
+#include "asm/kprobes.h"
+
+/* kprobe_status settings */
+#define KPROBE_HIT_ACTIVE 0x00000001
+#define KPROBE_HIT_SS 0x00000002
+#define KPROBE_REENTER 0x00000004
+#define KPROBE_HIT_SSDONE 0x00000008
+
+/* Attach to insert probes on any functions which should be ignored*/
+#define __kprobes __attribute__((__section__(".kprobes.text")))
+
+struct kprobe;
+struct pt_regs;
+struct kretprobe;
+struct kretprobe_instance;
+typedef int (*kprobe_pre_handler_t) (struct kprobe *, struct pt_regs * /*, struct vm_area_struct **,
+ struct page **, unsigned long ** */ );
+typedef int (*kprobe_break_handler_t) (struct kprobe *, struct pt_regs * /*, struct vm_area_struct **,
+ struct page **, unsigned long ** */ );
+typedef void (*kprobe_post_handler_t) (struct kprobe *, struct pt_regs *, unsigned long flags);
+typedef int (*kprobe_fault_handler_t) (struct kprobe *, struct pt_regs *, int trapnr);
+typedef int (*kretprobe_handler_t) (struct kretprobe_instance *, struct pt_regs *, void *);
+
+struct kprobe
+{
+ struct hlist_node hlist;
+
+ /*list of probes to search by instruction slot*/
+ struct hlist_node is_hlist;
+
+ /* list of kprobes for multi-handler support */
+ struct list_head list;
+
+ /* Indicates that the corresponding module has been ref counted */
+ unsigned int mod_refcounted;
+
+ /*count the number of times this probe was temporarily disarmed */
+ unsigned long nmissed;
+
+ /* location of the probe point */
+ kprobe_opcode_t *addr;
+
+ /* Allow user to indicate symbol name of the probe point */
+ char *symbol_name;
+
+ /* Offset into the symbol */
+ unsigned int offset;
+
+ /* Called before addr is executed. */
+ kprobe_pre_handler_t pre_handler;
+
+ /* Called after addr is executed, unless... */
+ kprobe_post_handler_t post_handler;
+
+ /* ... called if executing addr causes a fault (eg. page fault).
+ * Return 1 if it handled fault, otherwise kernel will see it. */
+ kprobe_fault_handler_t fault_handler;
+
+ /* ... called if breakpoint trap occurs in probe handler.
+ * Return 1 if it handled break, otherwise kernel will see it. */
+ kprobe_break_handler_t break_handler;
+
+ /* Saved opcode (which has been replaced with breakpoint) */
+ kprobe_opcode_t opcode;
+
+ /* copy of the original instruction */
+ struct arch_specific_insn ainsn;
+ // TGID to which probe belongs
+ pid_t tgid;
+ // override single-step target address,
+ // may be used to redirect control-flow to arbitrary address after probe point
+ // without invocation of original instruction;
+ // useful for functions replacement
+ // if jprobe.entry should return address of function or NULL
+ // if original function should be called
+ // not supported for X86, not tested for MIPS
+ kprobe_opcode_t *ss_addr;
+#ifdef _DEBUG
+ unsigned long entry_count;
+ unsigned long step_count;
+ unsigned long exit_count;
+ unsigned long lr;
+#endif
+#ifdef KPROBES_PROFILE
+ struct timeval start_tm;
+ struct timeval hnd_tm_sum;
+ unsigned long count;
+#endif
+};
+
+typedef unsigned long (*kprobe_pre_entry_handler_t) (void *priv_arg, struct pt_regs * regs);
+
+/*
+ * Special probe type that uses setjmp-longjmp type tricks to resume
+ * execution at a specified entry with a matching prototype corresponding
+ * to the probed function - a trick to enable arguments to become
+ * accessible seamlessly by probe handling logic.
+ * Note:
+ * Because of the way compilers allocate stack space for local variables
+ * etc upfront, regardless of sub-scopes within a function, this mirroring
+ * principle currently works only for probes placed on function entry points.
+ */
+struct jprobe
+{
+ struct kprobe kp;
+ kprobe_opcode_t *entry; /* probe handling code to jump to */
+ kprobe_pre_entry_handler_t pre_entry; /*handler whichw willb bec called before 'entry' */
+ void *priv_arg;
+};
+
+struct jprobe_instance
+{
+ struct hlist_node uflist; /* either on free list or used list */
+ struct hlist_node hlist;
+ struct jprobe *jp;
+ struct task_struct *task;
+};
+
+DECLARE_PER_CPU (struct kprobe *, current_kprobe);
+DECLARE_PER_CPU (struct kprobe_ctlblk, kprobe_ctlblk);
+
+extern void __arch_prepare_kretprobe (struct kretprobe *rp, struct pt_regs *regs);
+
+/*
+ * Function-return probe -
+ * Note:
+ * User needs to provide a handler function, and initialize maxactive.
+ * maxactive - The maximum number of instances of the probed function that
+ * can be active concurrently.
+ * nmissed - tracks the number of times the probed function's return was
+ * ignored, due to maxactive being too low.
+ *
+ */
+struct kretprobe
+{
+ struct kprobe kp;
+ kretprobe_handler_t handler;
+ void *priv_arg;
+ int maxactive;
+ int nmissed;
+ int disarm;
+ struct hlist_head free_instances;
+ struct hlist_head used_instances;
+};
+
+struct kretprobe_instance
+{
+ struct hlist_node uflist; /* either on free list or used list */
+ struct hlist_node hlist;
+ struct kretprobe *rp;
+ kprobe_opcode_t *ret_addr;
+ struct kretprobe *rp2;
+ struct task_struct *task;
+};
+
+extern spinlock_t kretprobe_lock;
+extern struct mutex kprobe_mutex;
+extern int arch_prepare_kprobe (struct kprobe *p);
+extern int arch_prepare_uprobe (struct kprobe *p, struct task_struct *task, int atomic);
+extern int arch_prepare_kretprobe (struct kretprobe *p);
+extern int arch_prepare_uretprobe (struct kretprobe *p, struct task_struct *task);
+extern void arch_arm_kprobe (struct kprobe *p);
+extern void arch_arm_kretprobe (struct kretprobe *p);
+extern void arch_arm_uprobe (struct kprobe *p, struct task_struct *tsk);
+extern void arch_arm_uretprobe (struct kretprobe *p, struct task_struct *tsk);
+extern void arch_disarm_kprobe (struct kprobe *p);
+extern void arch_disarm_kretprobe (struct kretprobe *p);
+extern void arch_disarm_uprobe (struct kprobe *p, struct task_struct *tsk);
+extern void arch_disarm_uretprobe (struct kretprobe *p, struct task_struct *tsk);
+extern int arch_init_kprobes (void);
+extern void arch_exit_kprobes (void);
+extern void show_registers (struct pt_regs *regs);
+extern void kprobes_inc_nmissed_count (struct kprobe *p);
+
+/* Get the kprobe at this addr (if any) - called with preemption disabled */
+struct kprobe *get_kprobe (void *addr, int pid, struct task_struct *ctask);
+struct kprobe *get_kprobe_by_insn_slot (void *addr, int tgid, struct task_struct *ctask);
+struct hlist_head *kretprobe_inst_table_head (struct task_struct *tsk);
+
+/* kprobe_running() will just return the current_kprobe on this CPU */
+static inline struct kprobe *
+kprobe_running (void)
+{
+ return (__get_cpu_var (current_kprobe));
+}
+
+static inline void
+reset_current_kprobe (void)
+{
+ //__get_cpu_var (current_kprobe)->spid = -1;
+ __get_cpu_var (current_kprobe) = NULL;
+}
+
+static inline struct kprobe_ctlblk *
+get_kprobe_ctlblk (void)
+{
+ return (&__get_cpu_var (kprobe_ctlblk));
+}
+
+int register_kprobe (struct kprobe *p, int atomic);
+void unregister_kprobe (struct kprobe *p, struct task_struct *task, int atomic);
+int setjmp_pre_handler (struct kprobe *, struct pt_regs *);
+int longjmp_break_handler (struct kprobe *, struct pt_regs *);
+int register_jprobe (struct jprobe *p, int atomic);
+void unregister_jprobe (struct jprobe *p, int atomic);
+int register_ujprobe (struct task_struct *task, struct mm_struct *mm, struct jprobe *jp, int atomic);
+void unregister_ujprobe (struct task_struct *task, struct jprobe *jp, int atomic);
+void unregister_uprobe (struct kprobe *p, struct task_struct *task, int atomic);
+void jprobe_return (void);
+void uprobe_return (void);
+
+int register_kretprobe (struct kretprobe *rp, int atomic);
+void unregister_kretprobe (struct kretprobe *rp, int atomic);
+int register_uretprobe (struct task_struct *task, struct mm_struct *mm, struct kretprobe *rp, int atomic);
+void unregister_uretprobe (struct task_struct *task, struct kretprobe *rp, int atomic);
+
+void unregister_all_uprobes (struct task_struct *task, int atomic);
+
+struct kretprobe_instance *get_free_rp_inst (struct kretprobe *rp);
+void add_rp_inst (struct kretprobe_instance *ri);
+//void kprobe_flush_task(struct task_struct *tk);
+void recycle_rp_inst (struct kretprobe_instance *ri, struct hlist_head *head);
+
+//void arch_copy_kprobe(struct kprobe *p);
+void arch_remove_kprobe (struct kprobe *p, struct task_struct *task);
+void kretprobe_trampoline_holder (void);
+int __kprobes trampoline_probe_handler (struct kprobe *p, struct pt_regs *regs);
+#ifdef KPROBES_PROFILE
+int __kprobes pre_handler_kretprobe (struct kprobe *p, struct pt_regs *regs, struct vm_area_struct **vma, struct page **page, unsigned long **kaddr);
+void set_normalized_timeval (struct timeval *tv, time_t sec, suseconds_t usec);
+#endif
+
+kprobe_opcode_t *get_insn_slot (struct task_struct *task, int atomic);
+void free_insn_slot (struct hlist_head *page_list, struct task_struct *task, kprobe_opcode_t *slot, int dirty);
+
+//int access_process_vm_atomic(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
+#define access_process_vm_atomic access_process_vm
+#define read_proc_vm_atomic(tsk, addr, buf, len) access_process_vm_atomic(tsk, addr, buf, len, 0)
+#define write_proc_vm_atomic(tsk, addr, buf, len) access_process_vm_atomic(tsk, addr, buf, len, 1)
+int page_present (struct mm_struct *mm, unsigned long addr);
+/*int get_user_pages_atomic(struct task_struct *tsk, struct mm_struct *mm,
+ unsigned long start, int len, int write, int force,
+ struct page **pages, struct vm_area_struct **vmas);*/
+#define get_user_pages_atomic get_user_pages
+#ifdef KERNEL_HAS_ISPAGEPRESENT
+#define page_present is_page_present
+#else
+int page_present (struct mm_struct *mm, unsigned long addr);
+#endif
+void purge_garbage_uslots(struct task_struct *task, int atomic);
+#endif /* _SRC_KPROBES_H */
+
+extern kprobe_opcode_t *sched_addr;
+extern kprobe_opcode_t *fork_addr;
--- /dev/null
+#include <linux/version.h> // LINUX_VERSION_CODE, KERNEL_VERSION()
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
+#include <linux/config.h>
+#endif
+#include <linux/ptrace.h>
+#include <linux/spinlock.h>
+#include <linux/preempt.h>
+#include <linux/module.h>
+#include <linux/highmem.h> // kmap_atomic, kunmap_atomic, copy_from_user_page, copy_to_user_page
+#include <linux/pagemap.h> // page_cache_release
+#include <asm/system.h>
+#include <asm/cacheflush.h>
+#include <linux/kallsyms.h>
+#include <linux/vmalloc.h>
+#include <linux/syscalls.h>
+#include <linux/security.h>
+#include <linux/mount.h>
+#include <linux/mman.h>
+#include <linux/personality.h>
+#include <linux/hugetlb.h>
+#include <linux/file.h>
+#include <linux/mempolicy.h>
+#if defined(CONFIG_X86)
+#include <linux/kdebug.h>
+#include <linux/moduleloader.h>
+#include <linux/freezer.h>
+#include <linux/hardirq.h>
+#endif
+#if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 19))
+#include <linux/freezer.h>
+#endif
+
+#include "kprobes.h"
+
+#if defined(CONFIG_X86)
+# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
+# define TF_MASK X86_EFLAGS_TF
+# define IF_MASK X86_EFLAGS_IF
+# endif
+# define UPROBES_TRAMP_LEN (MAX_INSN_SIZE+sizeof(kprobe_opcode_t))
+# define UPROBES_TRAMP_INSN_IDX 0
+# define UPROBES_TRAMP_RET_BREAK_IDX MAX_INSN_SIZE
+# define KPROBES_TRAMP_LEN MAX_INSN_SIZE
+# define KPROBES_TRAMP_INSN_IDX 0
+#elif defined(CONFIG_ARM)
+# define UPROBES_TRAMP_LEN 8
+# define UPROBES_TRAMP_INSN_IDX 2
+# define UPROBES_TRAMP_SS_BREAK_IDX 4
+# define UPROBES_TRAMP_RET_BREAK_IDX 5
+# define KPROBES_TRAMP_LEN 8
+# define KPROBES_TRAMP_INSN_IDX UPROBES_TRAMP_INSN_IDX
+# define KPROBES_TRAMP_SS_BREAK_IDX UPROBES_TRAMP_SS_BREAK_IDX
+# define KPROBES_TRAMP_RET_BREAK_IDX UPROBES_TRAMP_RET_BREAK_IDX
+#elif defined(CONFIG_MIPS)
+# define UPROBES_TRAMP_LEN 3
+# define UPROBES_TRAMP_INSN_IDX 0
+# define UPROBES_TRAMP_SS_BREAK_IDX 1
+# define UPROBES_TRAMP_RET_BREAK_IDX 2
+# define KPROBES_TRAMP_LEN UPROBES_TRAMP_LEN
+# define KPROBES_TRAMP_INSN_IDX UPROBES_TRAMP_INSN_IDX
+# define KPROBES_TRAMP_SS_BREAK_IDX UPROBES_TRAMP_SS_BREAK_IDX
+# define KPROBES_TRAMP_RET_BREAK_IDX UPROBES_TRAMP_RET_BREAK_IDX
+#endif //CONFIG_MIPS
+
+DEFINE_PER_CPU (struct kprobe *, current_kprobe) = NULL;
+DEFINE_PER_CPU (struct kprobe_ctlblk, kprobe_ctlblk);
+
+/* kprobe_status settings */
+#define KPROBE_HIT_ACTIVE 0x00000001
+#define KPROBE_HIT_SS 0x00000002
+
+#define INVALID_VALUE 0xFFFFFFFF
+#define INVALID_POINTER (void*)INVALID_VALUE
+
+static int ksyms = INVALID_VALUE;
+module_param (ksyms, int, 0);
+
+extern unsigned long handled_exceptions;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 12))
+#define synchronize_sched synchronize_kernel
+#endif
+
+void jprobe_return_end (void);
+void uprobe_return_end (void);
+
+#if defined(CONFIG_X86)
+/*fastcall*/ void *__kprobes trampoline_probe_handler_x86 (struct pt_regs *regs);
+#endif
+
+static inline void
+kretprobe_assert (struct kretprobe_instance *ri, unsigned long orig_ret_address, unsigned long trampoline_address)
+{
+ if (!orig_ret_address || (orig_ret_address == trampoline_address))
+ panic ("kretprobe BUG!: Processing kretprobe %p @ %p\n", ri->rp, ri->rp->kp.addr);
+}
+
+#define HIWORD(x) (((x) & 0xFFFF0000) >> 16)
+#define LOWORD(x) ((x) & 0x0000FFFF)
+
+unsigned int gl_nNumberOfInstructions = 0;
+unsigned int gl_nCodeSize = 0;
+
+unsigned int arrTrapsTemplate[] = {
+#if defined(CONFIG_MIPS)
+ 0x3c010000, // lui a1 [0]
+ 0x24210000, // addiu a1, a1 [1]
+ 0x00200008, // jr a1 [2]
+ 0x00000000, // nop
+ 0xffffffff // end
+#elif defined(CONFIG_ARM)
+ 0xe1a0c00d, // mov ip, sp
+ 0xe92dd800, // stmdb sp!, {fp, ip, lr, pc}
+ 0xe24cb004, // sub fp, ip, #4 ; 0x4
+ 0x00000000, // b [3]
+ 0xe3500000, // cmp r0, #0 ; 0x0
+ 0xe89da800, // ldmia sp, {fp, sp, pc}
+ 0x00000000, // nop
+ 0xffffffff // end
+#endif // ARCH
+};
+
+unsigned long nCount;
+
+kprobe_opcode_t *sched_addr;
+kprobe_opcode_t *fork_addr;
+
+#if defined(CONFIG_MIPS)
+#define REG_HI_INDEX 0
+#define REG_LO_INDEX 1
+#define NOTIFIER_CALL_CHAIN_INDEX 0
+
+#elif defined(CONFIG_ARM)
+#define NOTIFIER_CALL_CHAIN_INDEX 3
+//#define NOTIFIER_CALL_CHAIN_INDEX1 6
+//#define NOTIFIER_CALL_CHAIN_INDEX2 11
+
+static unsigned int
+arch_construct_brunch (unsigned int base, unsigned int addr, int link)
+{
+ kprobe_opcode_t insn;
+ unsigned int bpi = (unsigned int) base - (unsigned int) addr - 8;
+ insn = bpi >> 2;
+ DBPRINTF ("base=%x addr=%x base-addr-8=%x\n", base, addr, bpi);
+ if (abs (insn & 0xffffff) > 0xffffff)
+ {
+ DBPRINTF ("ERROR: kprobe address out of range\n");
+ BUG ();
+ }
+ insn = insn & 0xffffff;
+ insn = insn | ((link != 0) ? 0xeb000000 : 0xea000000);
+ DBPRINTF ("insn=%lX\n", insn);
+ return (unsigned int) insn;
+}
+#endif // ARCH
+
+unsigned int *arrTrapsOriginal = NULL;
+
+#ifndef KERNEL_HAS_ISPAGEPRESENT
+int
+page_present (struct mm_struct *mm, unsigned long addr)
+{
+ pgd_t *pgd;
+ pmd_t *pmd;
+ pte_t *pte;
+ int ret = 0;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
+ pud_t *pud;
+#endif
+
+ //printk("page_present\n");
+ //BUG_ON(down_read_trylock(&mm->mmap_sem) == 0);
+ down_read (&mm->mmap_sem);
+ spin_lock (&(mm->page_table_lock));
+ pgd = pgd_offset (mm, addr);
+ //printk("pgd %p\n", pgd);
+ if ((pgd != NULL) && pgd_present (*pgd))
+ {
+ //printk("pgd_present\n");
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
+ pud = pud_offset (pgd, addr);
+ //printk("pud %p\n", pud);
+ if ((pud != NULL) && pud_present (*pud))
+ {
+ pmd = pmd_offset (pud, addr);
+#else
+ {
+ pmd = pmd_offset (pgd, addr);
+#endif
+ //printk("pmd %p\n", pmd);
+ if ((pmd != NULL) && pmd_present (*pmd))
+ {
+ //spinlock_t *ptl;
+ //printk("pmd_present\n");
+ pte = pte_offset_map (pmd, addr);
+ //pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
+ //printk("pte %p/%lx\n", pte, addr);
+ if ((pte != NULL) && pte_present (*pte))
+ {
+ ret = 1;
+ //printk("pte_present\n");
+ }
+ pte_unmap (pte);
+ //pte_unmap_unlock(pte, ptl);
+ }
+ }
+ }
+ spin_unlock (&(mm->page_table_lock));
+ up_read (&mm->mmap_sem);
+ //printk("page_present %d\n", ret);
+ return ret;
+}
+#endif
+
+#if defined(CONFIG_MIPS)
+#define MIPS_INSN_OPCODE_MASK 0xFC000000
+#define MIPS_INSN_RS_MASK 0x03E00000
+#define MIPS_INSN_RT_MASK 0x001F0000
+//#define MIPS_INSN_UN_MASK 0x0000FFC0
+#define MIPS_INSN_FUNC_MASK 0x0000003F
+#define MIPS_INSN_OPCODE(insn) (insn & MIPS_INSN_OPCODE_MASK)
+#define MIPS_INSN_RS(insn) (insn & MIPS_INSN_RS_MASK)
+#define MIPS_INSN_RT(insn) (insn & MIPS_INSN_RT_MASK)
+#define MIPS_INSN_FUNC(insn) (insn & MIPS_INSN_FUNC_MASK)
+// opcodes 31..26
+#define MIPS_BEQ_OPCODE 0x10000000
+#define MIPS_BNE_OPCODE 0x14000000
+#define MIPS_BLEZ_OPCODE 0x18000000
+#define MIPS_BGTZ_OPCODE 0x1C000000
+#define MIPS_BEQL_OPCODE 0x50000000
+#define MIPS_BNEL_OPCODE 0x54000000
+#define MIPS_BLEZL_OPCODE 0x58000000
+#define MIPS_BGTZL_OPCODE 0x5C000000
+#define MIPS_REGIMM_OPCODE 0x04000000
+#define MIPS_SPECIAL_OPCODE 0x00000000
+#define MIPS_COP1_OPCODE 0x44000000
+#define MIPS_COP2_OPCODE 0x48000000
+#define MIPS_J_OPCODE 0x08000000
+#define MIPS_JAL_OPCODE 0x0C000000
+#define MIPS_JALX_OPCODE 0x74000000
+// rs 25..21
+#define MIPS_BC_RS 0x01000000
+// rt 20..16
+#define MIPS_BLTZ_RT 0x00000000
+#define MIPS_BGEZ_RT 0x00010000
+#define MIPS_BLTZL_RT 0x00020000
+#define MIPS_BGEZL_RT 0x00030000
+#define MIPS_BLTZAL_RT 0x00100000
+#define MIPS_BGEZAL_RT 0x00110000
+#define MIPS_BLTZALL_RT 0x00120000
+#define MIPS_BGEZALL_RT 0x00130000
+// unnamed 15..6
+// function 5..0
+#define MIPS_JR_FUNC 0x00000008
+#define MIPS_JALR_FUNC 0x00000009
+#define MIPS_BREAK_FUNC 0x0000000D
+#define MIPS_SYSCALL_FUNC 0x0000000C
+
+#elif defined(CONFIG_ARM)
+// undefined
+#define MASK_ARM_INSN_UNDEF 0x0FF00000
+#define PTRN_ARM_INSN_UNDEF 0x03000000
+// architecturally undefined
+#define MASK_ARM_INSN_AUNDEF 0x0FF000F0
+#define PTRN_ARM_INSN_AUNDEF 0x07F000F0
+// branches
+#define MASK_ARM_INSN_B 0x0E000000
+#define PTRN_ARM_INSN_B 0x0A000000
+#define MASK_ARM_INSN_BL 0x0E000000
+#define PTRN_ARM_INSN_BL 0x0B000000
+#define MASK_ARM_INSN_BLX1 0xFF000000
+#define PTRN_ARM_INSN_BLX1 0xFA000000
+#define MASK_ARM_INSN_BLX2 0x0FF000F0
+#define PTRN_ARM_INSN_BLX2 0x01200030
+#define MASK_ARM_INSN_BX 0x0FF000F0
+#define PTRN_ARM_INSN_BX 0x01200010
+#define MASK_ARM_INSN_BXJ 0x0FF000F0
+#define PTRN_ARM_INSN_BXJ 0x01200020
+// software interrupts
+#define MASK_ARM_INSN_SWI 0x0F000000
+#define PTRN_ARM_INSN_SWI 0x0F000000
+// break
+#define MASK_ARM_INSN_BREAK 0xFFF000F0
+#define PTRN_ARM_INSN_BREAK 0xE1200070
+// Data processing immediate shift
+#define MASK_ARM_INSN_DPIS 0x0E000010
+#define PTRN_ARM_INSN_DPIS 0x00000000
+// Data processing register shift
+#define MASK_ARM_INSN_DPRS 0x0E000090
+#define PTRN_ARM_INSN_DPRS 0x00000010
+// Data processing immediate
+#define MASK_ARM_INSN_DPI 0x0E000000
+#define PTRN_ARM_INSN_DPI 0x02000000
+// Load immediate offset
+#define MASK_ARM_INSN_LIO 0x0E100000
+#define PTRN_ARM_INSN_LIO 0x04100000
+// Store immediate offset
+#define MASK_ARM_INSN_SIO MASK_ARM_INSN_LIO
+#define PTRN_ARM_INSN_SIO 0x04000000
+// Load register offset
+#define MASK_ARM_INSN_LRO 0x0E100010
+#define PTRN_ARM_INSN_LRO 0x06100000
+// Store register offset
+#define MASK_ARM_INSN_SRO MASK_ARM_INSN_LRO
+#define PTRN_ARM_INSN_SRO 0x06000000
+// Load multiple
+#define MASK_ARM_INSN_LM 0x0E100000
+#define PTRN_ARM_INSN_LM 0x08100000
+// Store multiple
+#define MASK_ARM_INSN_SM MASK_ARM_INSN_LM
+#define PTRN_ARM_INSN_SM 0x08000000
+// Coprocessor load/store and double register transfers
+#define MASK_ARM_INSN_CLS 0x0E000000
+#define PTRN_ARM_INSN_CLS 0x0C000000
+// Coprocessor register transfers
+#define MASK_ARM_INSN_CRT 0x0F000010
+#define PTRN_ARM_INSN_CRT 0x0E000010
+
+#define ARM_INSN_MATCH(name, insn) ((insn & MASK_ARM_INSN_##name) == PTRN_ARM_INSN_##name)
+
+#define ARM_INSN_REG_RN(insn) ((insn & 0x000F0000)>>16)
+#define ARM_INSN_REG_SET_RN(insn, nreg) {insn &= ~0x000F0000; insn |= nreg<<16;}
+#define ARM_INSN_REG_RD(insn) ((insn & 0x0000F000)>>12)
+#define ARM_INSN_REG_SET_RD(insn, nreg) {insn &= ~0x0000F000; insn |= nreg<<12;}
+#define ARM_INSN_REG_RS(insn) ((insn & 0x00000F00)>>8)
+#define ARM_INSN_REG_SET_RS(insn, nreg) {insn &= ~0x00000F00; insn |= nreg<<8;}
+#define ARM_INSN_REG_RM(insn) (insn & 0x0000000F)
+#define ARM_INSN_REG_SET_RM(insn, nreg) {insn &= ~0x0000000F; insn |= nreg;}
+#define ARM_INSN_REG_MR(insn, nreg) (insn & (1 << nreg))
+#define ARM_INSN_REG_SET_MR(insn, nreg) {insn |= (1 << nreg);}
+#define ARM_INSN_REG_CLEAR_MR(insn, nreg) {insn &= ~(1 << nreg);}
+
+#elif defined(CONFIG_X86)
+//# warning Branch instruction patterns are not defined for x86 arch!!!
+#endif
+
+#if defined(CONFIG_X86)
+/* insert a jmp code */
+static __always_inline void
+set_jmp_op (void *from, void *to)
+{
+ struct __arch_jmp_op
+ {
+ char op;
+ long raddr;
+ } __attribute__ ((packed)) * jop;
+ jop = (struct __arch_jmp_op *) from;
+ jop->raddr = (long) (to) - ((long) (from) + 5);
+ jop->op = RELATIVEJUMP_INSTRUCTION;
+}
+
+static void
+set_user_jmp_op (void *from, void *to)
+{
+ struct __arch_jmp_op
+ {
+ char op;
+ long raddr;
+ } __attribute__ ((packed)) jop;
+ //jop = (struct __arch_jmp_op *) from;
+ jop.raddr = (long) (to) - ((long) (from) + 5);
+ jop.op = RELATIVEJUMP_INSTRUCTION;
+ if (!write_proc_vm_atomic (current, (unsigned long)from, &jop, sizeof(jop)))
+ panic ("failed to write jump opcode to user space %p!\n", from);
+}
+
+/*
+ * returns non-zero if opcodes can be boosted.
+ */
+static __always_inline int
+can_boost (kprobe_opcode_t * opcodes)
+{
+#define W(row,b0,b1,b2,b3,b4,b5,b6,b7,b8,b9,ba,bb,bc,bd,be,bf) \
+ (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
+ (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \
+ (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \
+ (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \
+ << (row % 32))
+ /*
+ * Undefined/reserved opcodes, conditional jump, Opcode Extension
+ * Groups, and some special opcodes can not be boost.
+ */
+ static const unsigned long twobyte_is_boostable[256 / 32] = {
+ /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
+ /* ------------------------------- */
+ W (0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */
+ W (0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), /* 10 */
+ W (0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */
+ W (0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), /* 30 */
+ W (0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
+ W (0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), /* 50 */
+ W (0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */
+ W (0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1), /* 70 */
+ W (0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */
+ W (0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), /* 90 */
+ W (0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */
+ W (0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1), /* b0 */
+ W (0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
+ W (0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1), /* d0 */
+ W (0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */
+ W (0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0) /* f0 */
+ /* ------------------------------- */
+ /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
+ };
+#undef W
+ kprobe_opcode_t opcode;
+ kprobe_opcode_t *orig_opcodes = opcodes;
+ retry:
+ if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
+ return 0;
+ opcode = *(opcodes++);
+
+ /* 2nd-byte opcode */
+ if (opcode == 0x0f)
+ {
+ if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
+ return 0;
+ return test_bit (*opcodes, twobyte_is_boostable);
+ }
+
+ switch (opcode & 0xf0)
+ {
+ case 0x60:
+ if (0x63 < opcode && opcode < 0x67)
+ goto retry; /* prefixes */
+ /* can't boost Address-size override and bound */
+ return (opcode != 0x62 && opcode != 0x67);
+ case 0x70:
+ return 0; /* can't boost conditional jump */
+ case 0xc0:
+ /* can't boost software-interruptions */
+ return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf;
+ case 0xd0:
+ /* can boost AA* and XLAT */
+ return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7);
+ case 0xe0:
+ /* can boost in/out and absolute jmps */
+ return ((opcode & 0x04) || opcode == 0xea);
+ case 0xf0:
+ if ((opcode & 0x0c) == 0 && opcode != 0xf1)
+ goto retry; /* lock/rep(ne) prefix */
+ /* clear and set flags can be boost */
+ return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe));
+ default:
+ if (opcode == 0x26 || opcode == 0x36 || opcode == 0x3e)
+ goto retry; /* prefixes */
+ /* can't boost CS override and call */
+ return (opcode != 0x2e && opcode != 0x9a);
+ }
+}
+
+/*
+ * returns non-zero if opcode modifies the interrupt flag.
+ */
+static int __kprobes
+is_IF_modifier (kprobe_opcode_t opcode)
+{
+ switch (opcode)
+ {
+ case 0xfa: /* cli */
+ case 0xfb: /* sti */
+ case 0xcf: /* iret/iretd */
+ case 0x9d: /* popf/popfd */
+ return 1;
+ }
+ return 0;
+}
+#endif
+
+static int
+arch_check_insn (struct arch_specific_insn *ainsn)
+{
+ int ret = 0;
+
+#if defined(CONFIG_MIPS)
+ switch (MIPS_INSN_OPCODE (ainsn->insn[0]))
+ {
+ case MIPS_BEQ_OPCODE: //B, BEQ
+ case MIPS_BEQL_OPCODE: //BEQL
+ case MIPS_BNE_OPCODE: //BNE
+ case MIPS_BNEL_OPCODE: //BNEL
+ case MIPS_BGTZ_OPCODE: //BGTZ
+ case MIPS_BGTZL_OPCODE: //BGTZL
+ case MIPS_BLEZ_OPCODE: //BLEZ
+ case MIPS_BLEZL_OPCODE: //BLEZL
+ case MIPS_J_OPCODE: //J
+ case MIPS_JAL_OPCODE: //JAL
+ DBPRINTF ("arch_check_insn: opcode");
+ ret = -EFAULT;
+ break;
+ case MIPS_REGIMM_OPCODE:
+ //BAL, BGEZ, BGEZAL, BGEZALL, BGEZL, BLTZ, BLTZAL, BLTZALL, BLTZL
+ switch (MIPS_INSN_RT (ainsn->insn[0]))
+ {
+ case MIPS_BLTZ_RT:
+ case MIPS_BGEZ_RT:
+ case MIPS_BLTZL_RT:
+ case MIPS_BGEZL_RT:
+ case MIPS_BLTZAL_RT:
+ case MIPS_BGEZAL_RT:
+ case MIPS_BLTZALL_RT:
+ case MIPS_BGEZALL_RT:
+ DBPRINTF ("arch_check_insn: REGIMM opcode\n");
+ ret = -EFAULT;
+ break;
+ }
+ break;
+ //BC1F, BC1FL, BC1T, BC1TL
+ case MIPS_COP1_OPCODE:
+ //BC2F, BC2FL, BC2T, BC2TL
+ case MIPS_COP2_OPCODE:
+ if (MIPS_INSN_RS (ainsn->insn[0]) == MIPS_BC_RS)
+ {
+ DBPRINTF ("arch_check_insn: COP1 opcode\n");
+ ret = -EFAULT;
+ }
+ break;
+ case MIPS_SPECIAL_OPCODE:
+ //BREAK, JALR, JALR.HB, JR, JR.HB
+ switch (MIPS_INSN_FUNC (ainsn->insn[0]))
+ {
+ case MIPS_JR_FUNC:
+ case MIPS_JALR_FUNC:
+ case MIPS_BREAK_FUNC:
+ case MIPS_SYSCALL_FUNC:
+ DBPRINTF ("arch_check_insn: SPECIAL opcode\n");
+ ret = -EFAULT;
+ break;
+ }
+ break;
+ }
+#elif defined(CONFIG_ARM)
+ // check instructions that can change PC by nature
+ if (ARM_INSN_MATCH (UNDEF, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (AUNDEF, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (SWI, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (BREAK, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (B, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (BL, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (BLX1, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (BLX2, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (BX, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (BXJ, ainsn->insn[0]))
+ {
+ DBPRINTF ("arch_check_insn: %lx\n", ainsn->insn[0]);
+ ret = -EFAULT;
+ }
+#ifndef CONFIG_CPU_V7
+ // check instructions that can write result to PC
+ else if ((ARM_INSN_MATCH (DPIS, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (DPRS, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (DPI, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (LIO, ainsn->insn[0]) ||
+ ARM_INSN_MATCH (LRO, ainsn->insn[0])) &&
+ (ARM_INSN_REG_RD (ainsn->insn[0]) == 15))
+ {
+ DBPRINTF ("arch_check_insn: %lx\n", ainsn->insn[0]);
+ ret = -EFAULT;
+ }
+#endif // CONFIG_CPU_V7
+ // check special instruction loads store multiple registers
+ else if ((ARM_INSN_MATCH (LM, ainsn->insn[0]) || ARM_INSN_MATCH (SM, ainsn->insn[0])) &&
+ // store pc or load to pc
+ (ARM_INSN_REG_MR (ainsn->insn[0], 15) ||
+ // store/load with pc update
+ ((ARM_INSN_REG_RN (ainsn->insn[0]) == 15) && (ainsn->insn[0] & 0x200000))))
+ {
+ DBPRINTF ("arch_check_insn: %lx\n", ainsn->insn[0]);
+ ret = -EFAULT;
+ }
+#elif defined(CONFIG_X86)
+//# warning arch_check_insn is not implemented for x86 arch!!!
+#endif
+
+ return ret;
+}
+
+/*
+ * kprobe->ainsn.insn points to the copy of the instruction to be
+ * single-stepped. x86_64, POWER4 and above have no-exec support and
+ * stepping on the instruction on a vmalloced/kmalloced/data page
+ * is a recipe for disaster
+ */
+#define INSNS_PER_PAGE (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
+
+struct kprobe_insn_page
+{
+ struct hlist_node hlist;
+ kprobe_opcode_t *insns; /* Page of instruction slots */
+ char *slot_used;//[INSNS_PER_PAGE];
+ int nused;
+ int ngarbage;
+ int tgid;
+};
+
+enum kprobe_slot_state
+{
+ SLOT_CLEAN = 0,
+ SLOT_DIRTY = 1,
+ SLOT_USED = 2,
+};
+
+static struct hlist_head kprobe_insn_pages;
+static int kprobe_garbage_slots;
+static struct hlist_head uprobe_insn_pages;
+static int uprobe_garbage_slots;
+static int collect_garbage_slots (struct hlist_head *page_list, struct task_struct *task);
+
+void gen_insn_execbuf (void);
+void pc_dep_insn_execbuf (void);
+void gen_insn_execbuf_holder (void);
+void pc_dep_insn_execbuf_holder (void);
+
+void
+gen_insn_execbuf_holder (void)
+{
+ asm volatile (".global gen_insn_execbuf\n"
+ "gen_insn_execbuf:\n"
+#if defined(CONFIG_ARM)
+ "nop\n"
+ "nop\n"
+ "nop\n" // original instruction
+ "nop\n"
+ "ldr pc, [pc, #4]\n" //ssbreak
+ "nop\n" //retbreak
+ "nop\n"
+ "nop\n"); //stored PC-4(next insn addr)
+#elif defined(CONFIG_MIPS)
+ "nop\n" // original instruction
+ "nop\n" //ssbreak
+ "nop\n");//retbreak
+#else
+ "nop\n");//retbreak
+#endif
+}
+
+#if defined(CONFIG_ARM)
+/*void
+pc_dep_uinsn_execbuf_holder (void)
+{
+ asm volatile (".global pc_dep_uinsn_execbuf\n"
+ "pc_dep_uinsn_execbuf:\n"
+ "str r0, [pc, #20]\n"
+ "ldr r0, [pc, #12]\n"
+ "nop\n" // instruction with replaced PC
+ "ldr r0, [pc, #8]\n"
+ "nop\n" // ssbreak
+ "nop\n" // retbreak
+ "nop\n" // stored PC
+ "nop\n");// stored Rx
+}*/
+/*
+ * 0. push Rx on stack
+ * 1. load address to Rx
+ * 2. do insn using Rx
+ * 3. pop Rx from stack
+ * 4. BREAK1
+ * 5. BREAK2
+ * 6. stored PC
+ * 7. stored PC-4(next insn addr)
+ */
+void
+pc_dep_insn_execbuf_holder (void)
+{
+ asm volatile (".global pc_dep_insn_execbuf\n"
+ "pc_dep_insn_execbuf:\n"
+ "str r0, [sp, #-4]\n"
+ "ldr r0, [pc, #12]\n"
+ "nop\n" // instruction with replaced PC
+ "ldr r0, [sp, #-4]\n"
+ "ldr pc, [pc, #4]\n" //ssbreak
+ "nop\n" // retbreak
+ "nop\n" // stored PC
+ "nop\n");// stored PC-4 (next insn addr)
+}
+
+static int
+prep_pc_dep_insn_execbuf (kprobe_opcode_t * insns, kprobe_opcode_t insn, int uregs)
+{
+ int i;
+
+ if (uregs & 0x10)
+ {
+ int reg_mask = 0x1;
+ //search in reg list
+ for (i = 0; i < 13; i++, reg_mask <<= 1)
+ {
+ if (!(insn & reg_mask))
+ break;
+ }
+ }
+ else
+ {
+ for (i = 0; i < 13; i++)
+ {
+ // DBPRINTF("prep_pc_dep_insn_execbuf: check R%d/%d, changing regs %x in %x",
+ // i, ARM_INSN_REG_RN(insn), uregs, insn);
+ if ((uregs & 0x1) && (ARM_INSN_REG_RN (insn) == i))
+ continue;
+ if ((uregs & 0x2) && (ARM_INSN_REG_RD (insn) == i))
+ continue;
+ if ((uregs & 0x4) && (ARM_INSN_REG_RS (insn) == i))
+ continue;
+ if ((uregs & 0x8) && (ARM_INSN_REG_RM (insn) == i))
+ continue;
+ break;
+ }
+ }
+ if (i == 13)
+ {
+ DBPRINTF ("there are no free register %x in insn %lx!", uregs, insn);
+ return -EINVAL;
+ }
+ DBPRINTF ("prep_pc_dep_insn_execbuf: using R%d, changing regs %x", i, uregs);
+
+ // set register to save
+ ARM_INSN_REG_SET_RD (insns[0], i);
+ // set register to load address to
+ ARM_INSN_REG_SET_RD (insns[1], i);
+ // set instruction to execute and patch it
+ if (uregs & 0x10)
+ {
+ ARM_INSN_REG_CLEAR_MR (insn, 15);
+ ARM_INSN_REG_SET_MR (insn, i);
+ }
+ else
+ {
+ if ((uregs & 0x1) && (ARM_INSN_REG_RN (insn) == 15))
+ ARM_INSN_REG_SET_RN (insn, i);
+ if ((uregs & 0x2) && (ARM_INSN_REG_RD (insn) == 15))
+ ARM_INSN_REG_SET_RD (insn, i);
+ if ((uregs & 0x4) && (ARM_INSN_REG_RS (insn) == 15))
+ ARM_INSN_REG_SET_RS (insn, i);
+ if ((uregs & 0x8) && (ARM_INSN_REG_RM (insn) == 15))
+ ARM_INSN_REG_SET_RM (insn, i);
+ }
+ insns[UPROBES_TRAMP_INSN_IDX] = insn;
+ // set register to restore
+ ARM_INSN_REG_SET_RD (insns[3], i);
+ return 0;
+}
+#endif//ARM
+
+int
+arch_prepare_kprobe (struct kprobe *p)
+{
+#if !defined(CONFIG_X86)
+ kprobe_opcode_t insns[KPROBES_TRAMP_LEN];
+#endif
+#if defined(CONFIG_ARM)
+ int uregs, pc_dep;
+#endif
+
+ int ret = 0;
+#if !defined(CONFIG_X86)
+ if ((unsigned long) p->addr & 0x01)
+ {
+ DBPRINTF ("Attempt to register kprobe at an unaligned address\n");
+ ret = -EINVAL;
+ }
+#endif
+ /* XXX: Might be a good idea to check if p->addr is a valid
+ * kernel address as well... */
+
+ if (!ret)
+ {
+ kprobe_opcode_t insn[MAX_INSN_SIZE];
+ struct arch_specific_insn ainsn;
+ /* insn: must be on special executable page on i386. */
+ p->ainsn.insn = get_insn_slot (NULL, 0);
+ if (!p->ainsn.insn)
+ return -ENOMEM;
+ memcpy (insn, p->addr, MAX_INSN_SIZE * sizeof (kprobe_opcode_t));
+ ainsn.insn = insn;
+ ret = arch_check_insn (&ainsn);
+ if (!ret)
+ {
+ p->opcode = *p->addr;
+#if defined(CONFIG_ARM)
+ p->ainsn.boostable = 1;
+ uregs = pc_dep = 0;
+ // Rn, Rm ,Rd
+ if (ARM_INSN_MATCH (DPIS, insn[0]) || ARM_INSN_MATCH (LRO, insn[0]) ||
+ ARM_INSN_MATCH (SRO, insn[0]))
+ {
+
+ uregs = 0xb;
+ if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_REG_RM (insn[0]) == 15) ||
+ (ARM_INSN_MATCH (SRO, insn[0]) && (ARM_INSN_REG_RD (insn[0]) == 15)))
+ {
+
+ DBPRINTF ("Unboostable insn %lx, DPIS/LRO/SRO\n", insn[0]);
+ pc_dep = 1;
+ }
+ }
+ // Rn ,Rd
+ else if (ARM_INSN_MATCH (DPI, insn[0]) || ARM_INSN_MATCH (LIO, insn[0]) ||
+ ARM_INSN_MATCH (SIO, insn[0]))
+ {
+
+ uregs = 0x3;
+ if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_MATCH (SIO, insn[0]) &&
+ (ARM_INSN_REG_RD (insn[0]) == 15)))
+ {
+
+ pc_dep = 1;
+ DBPRINTF ("Unboostable insn %lx/%p/%d, DPI/LIO/SIO\n", insn[0], p, p->ainsn.boostable);
+ }
+ }
+ // Rn, Rm, Rs
+ else if (ARM_INSN_MATCH (DPRS, insn[0]))
+ {
+
+ uregs = 0xd;
+ if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_REG_RM (insn[0]) == 15) ||
+ (ARM_INSN_REG_RS (insn[0]) == 15))
+ {
+
+ pc_dep = 1;
+ DBPRINTF ("Unboostable insn %lx, DPRS\n", insn[0]);
+ }
+ }
+ // register list
+ else if (ARM_INSN_MATCH (SM, insn[0]))
+ {
+
+ uregs = 0x10;
+ if (ARM_INSN_REG_MR (insn[0], 15))
+ {
+
+ DBPRINTF ("Unboostable insn %lx, SM\n", insn[0]);
+ pc_dep = 1;
+ }
+ }
+ // check instructions that can write result to SP andu uses PC
+ if (pc_dep && (ARM_INSN_REG_RD (ainsn.insn[0]) == 13))
+ {
+ static int count;
+ count++;
+ //printk ("insn writes result to SP and uses PC: %lx/%d\n", ainsn.insn[0], count);
+ free_insn_slot (&kprobe_insn_pages, NULL, p->ainsn.insn, 0);
+ ret = -EFAULT;
+ }
+ else {
+ if (uregs && pc_dep)
+ {
+ memcpy (insns, pc_dep_insn_execbuf, sizeof (insns));
+ if (prep_pc_dep_insn_execbuf (insns, insn[0], uregs) != 0)
+ {
+ DBPRINTF ("failed to prepare exec buffer for insn %lx!", insn[0]);
+ free_insn_slot (&kprobe_insn_pages, NULL, p->ainsn.insn, 0);
+ return -EINVAL;
+ }
+ //insns[KPROBES_TRAMP_SS_BREAK_IDX] = BREAKPOINT_INSTRUCTION;
+ insns[6] = (kprobe_opcode_t) (p->addr + 2);
+ }
+ else
+ {
+ memcpy (insns, gen_insn_execbuf, sizeof (insns));
+ insns[KPROBES_TRAMP_INSN_IDX] = insn[0];
+ }
+ //insns[KPROBES_TRAMP_RET_BREAK_IDX] = UNDEF_INSTRUCTION;
+ insns[7] = (kprobe_opcode_t) (p->addr + 1);
+ DBPRINTF ("arch_prepare_kprobe: insn %lx", insn[0]);
+ DBPRINTF ("arch_prepare_kprobe: to %p - %lx %lx %lx %lx %lx %lx %lx %lx %lx",
+ p->ainsn.insn, insns[0], insns[1], insns[2], insns[3], insns[4],
+ insns[5], insns[6], insns[7], insns[8]);
+ memcpy (p->ainsn.insn, insns, sizeof(insns));
+ }
+#elif defined(CONFIG_MIPS)
+ p->ainsn.boostable = 0;
+ memcpy (insns, gen_insn_execbuf, sizeof (insns));
+ insns[KPROBES_TRAMP_INSN_IDX] = insn[0];
+ insns[KPROBES_TRAMP_SS_BREAK_IDX] = BREAKPOINT_INSTRUCTION;
+ insns[KPROBES_TRAMP_RET_BREAK_IDX] = UNDEF_INSTRUCTION;
+ DBPRINTF ("arch_prepare_kprobe: insn %lx", insn[0]);
+ DBPRINTF ("arch_prepare_kprobe: to %p - %lx %lx %lx",
+ p->ainsn.insn, insns[0], insns[1], insns[2]);
+ memcpy (p->ainsn.insn, insns, sizeof(insns));
+#elif defined(CONFIG_X86)
+ if (can_boost (p->addr))
+ p->ainsn.boostable = 0;
+ else
+ p->ainsn.boostable = -1;
+ memcpy (p->ainsn.insn, insn, MAX_INSN_SIZE * sizeof (kprobe_opcode_t));
+#endif
+ }
+ else
+ {
+ free_insn_slot (&kprobe_insn_pages, NULL, p->ainsn.insn, 0);
+ }
+ }
+
+ return ret;
+}
+
+int
+arch_prepare_kretprobe (struct kretprobe *p)
+{
+ int ret = 0;
+#if 0
+ if ((unsigned long) p->kp.addr & 0x01)
+ {
+ DBPRINTF ("Attempt to register kprobe at an unaligned address\n");
+ ret = -EINVAL;
+ }
+
+ /* XXX: Might be a good idea to check if p->addr is a valid
+ * kernel address as well... */
+
+ if (!ret)
+ {
+ kprobe_opcode_t insn;
+ struct arch_specific_insn ainsn;
+ memcpy (&insn, p->kp.addr, MAX_INSN_SIZE * sizeof (kprobe_opcode_t));
+ ainsn.insn = &insn;
+ ret = arch_check_insn (&ainsn);
+ if (!ret)
+ {
+ p->kp.opcode = *p->kp.addr;
+#if defined(CONFIG_X86)
+ memcpy (p->kp.ainsn.insn, p->kp.addr, MAX_INSN_SIZE * sizeof (kprobe_opcode_t));
+#endif
+ }
+ }
+#endif
+ return ret;
+}
+
+int
+arch_prepare_uprobe (struct kprobe *p, struct task_struct *task, int atomic)
+{
+ int ret = 0;
+ kprobe_opcode_t insns[UPROBES_TRAMP_LEN];
+#if defined(CONFIG_ARM)
+ int uregs, pc_dep;
+#endif
+
+#if !defined(CONFIG_X86)
+ if ((unsigned long) p->addr & 0x01)
+ {
+ DBPRINTF ("Attempt to register kprobe at an unaligned address");
+ ret = -EINVAL;
+ }
+#endif
+
+ if (!ret)
+ {
+ kprobe_opcode_t insn[MAX_INSN_SIZE];
+ struct arch_specific_insn ainsn;
+
+ if (!read_proc_vm_atomic (task, (unsigned long) p->addr, &insn, MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
+ panic ("failed to read memory %p!\n", p->addr);
+ ainsn.insn = insn;
+ ret = arch_check_insn (&ainsn);
+ if (!ret)
+ {
+ p->opcode = insn[0];
+ p->ainsn.insn = get_insn_slot(task, atomic);
+ if (!p->ainsn.insn)
+ return -ENOMEM;
+#if defined(CONFIG_ARM)
+ p->ainsn.boostable = 1;
+ uregs = pc_dep = 0;
+ // Rn, Rm ,Rd
+ if (ARM_INSN_MATCH (DPIS, insn[0]) || ARM_INSN_MATCH (LRO, insn[0]) ||
+ ARM_INSN_MATCH (SRO, insn[0]))
+ {
+
+ uregs = 0xb;
+ if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_REG_RM (insn[0]) == 15) ||
+ (ARM_INSN_MATCH (SRO, insn[0]) && (ARM_INSN_REG_RD (insn[0]) == 15)))
+ {
+
+ DBPRINTF ("Unboostable insn %lx, DPIS/LRO/SRO\n", insn[0]);
+ pc_dep = 1;
+ }
+ }
+ // Rn ,Rd
+ else if (ARM_INSN_MATCH (DPI, insn[0]) || ARM_INSN_MATCH (LIO, insn[0]) ||
+ ARM_INSN_MATCH (SIO, insn[0]))
+ {
+
+ uregs = 0x3;
+ if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_MATCH (SIO, insn[0]) &&
+ (ARM_INSN_REG_RD (insn[0]) == 15)))
+ {
+
+ pc_dep = 1;
+ DBPRINTF ("Unboostable insn %lx/%p/%d, DPI/LIO/SIO\n", insn[0], p, p->ainsn.boostable);
+ }
+ }
+ // Rn, Rm, Rs
+ else if (ARM_INSN_MATCH (DPRS, insn[0]))
+ {
+
+ uregs = 0xd;
+ if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_REG_RM (insn[0]) == 15) ||
+ (ARM_INSN_REG_RS (insn[0]) == 15))
+ {
+
+ pc_dep = 1;
+ DBPRINTF ("Unboostable insn %lx, DPRS\n", insn[0]);
+ }
+ }
+ // register list
+ else if (ARM_INSN_MATCH (SM, insn[0]))
+ {
+
+ uregs = 0x10;
+ if (ARM_INSN_REG_MR (insn[0], 15))
+ {
+
+ DBPRINTF ("Unboostable insn %lx, SM\n", insn[0]);
+ pc_dep = 1;
+ }
+ }
+ // check instructions that can write result to SP andu uses PC
+ if (pc_dep && (ARM_INSN_REG_RD (ainsn.insn[0]) == 13))
+ {
+ static int count;
+ count++;
+ //printk ("insn writes result to SP and uses PC: %lx/%d\n", ainsn.insn[0], count);
+ free_insn_slot (&uprobe_insn_pages, task, p->ainsn.insn, 0);
+ ret = -EFAULT;
+ }
+ else {
+ if (uregs && pc_dep)
+ {
+ memcpy (insns, pc_dep_insn_execbuf, sizeof (insns));
+ if (prep_pc_dep_insn_execbuf (insns, insn[0], uregs) != 0)
+ {
+ DBPRINTF ("failed to prepare exec buffer for insn %lx!", insn[0]);
+ free_insn_slot (&uprobe_insn_pages, task, p->ainsn.insn, 0);
+ return -EINVAL;
+ }
+ //insns[UPROBES_TRAMP_SS_BREAK_IDX] = BREAKPOINT_INSTRUCTION;
+ insns[6] = (kprobe_opcode_t) (p->addr + 2);
+ }
+ else
+ {
+ memcpy (insns, gen_insn_execbuf, sizeof (insns));
+ insns[UPROBES_TRAMP_INSN_IDX] = insn[0];
+ }
+ insns[UPROBES_TRAMP_RET_BREAK_IDX] = UNDEF_INSTRUCTION;
+ insns[7] = (kprobe_opcode_t) (p->addr + 1);
+ DBPRINTF ("arch_prepare_uprobe: to %p - %lx %lx %lx %lx %lx %lx %lx %lx %lx",
+ p->ainsn.insn, insns[0], insns[1], insns[2], insns[3], insns[4],
+ insns[5], insns[6], insns[7], insns[8]);
+ }
+#elif defined(CONFIG_MIPS)
+ p->ainsn.boostable = 0;
+ memcpy (insns, gen_insn_execbuf, sizeof (insns));
+ insns[UPROBES_TRAMP_INSN_IDX] = insn[0];
+ insns[UPROBES_TRAMP_SS_BREAK_IDX] = BREAKPOINT_INSTRUCTION;
+ insns[UPROBES_TRAMP_RET_BREAK_IDX] = UNDEF_INSTRUCTION;
+ DBPRINTF ("arch_prepare_uprobe: insn %lx", insn[0]);
+ DBPRINTF ("arch_prepare_uprobe: to %p - %lx %lx %lx",
+ p->ainsn.insn, insns[0], insns[1], insns[2]);
+#elif defined(CONFIG_X86)
+ if (can_boost (insn))
+ p->ainsn.boostable = 0;
+ else
+ p->ainsn.boostable = -1;
+ memcpy (&insns[UPROBES_TRAMP_INSN_IDX], insn, MAX_INSN_SIZE*sizeof(kprobe_opcode_t));
+ insns[UPROBES_TRAMP_RET_BREAK_IDX] = BREAKPOINT_INSTRUCTION;
+ /*printk ("arch_prepare_uprobe: to %p - %02x %02x %02x %02x %02x %02x %02x %02x "
+ "%02x %02x %02x %02x %02x %02x %02x %02x %02x", p->ainsn.insn
+ , insns[0], insns[1], insns[2], insns[3]
+ , insns[4], insns[5], insns[6], insns[7]
+ , insns[8], insns[9], insns[10], insns[11]
+ , insns[12], insns[13], insns[14], insns[15], insns[16]);*/
+#endif
+ if (!write_proc_vm_atomic (task, (unsigned long) p->ainsn.insn, insns, sizeof (insns)))
+ {
+ panic("failed to write memory %p!\n", p->ainsn.insn);
+ DBPRINTF ("failed to write insn slot to process memory: insn %p, addr %p, probe %p!", insn, p->ainsn.insn, p->addr);
+ /*printk ("failed to write insn slot to process memory: %p/%d insn %lx, addr %p, probe %p!\n",
+ task, task->pid, insn, p->ainsn.insn, p->addr);*/
+ free_insn_slot (&uprobe_insn_pages, task, p->ainsn.insn, 0);
+ return -EINVAL;
+ }
+ /*if(!read_proc_vm_atomic(task, (unsigned long)p->ainsn.insn, insns, 3*MAX_INSN_SIZE*sizeof(kprobe_opcode_t)))
+ panic("failed to read memory %p!\n", p->addr);
+ printk("arch_prepare_uprobe: from %p - %lx %lx %lx\n", p->ainsn.insn, insns[0], insns[1], insns[2]); */
+ }
+ }
+
+ return ret;
+}
+
+int
+arch_prepare_uretprobe (struct kretprobe *p, struct task_struct *task)//, struct vm_area_struct **vma, struct page **page, unsigned long **kaddr)
+{
+ int ret = 0;
+#if 0
+ if ((unsigned long) p->kp.addr & 0x01)
+ {
+ DBPRINTF ("Attempt to register kprobe at an unaligned address\n");
+ ret = -EINVAL;
+ }
+#if defined(CONFIG_X86)
+#warning arch_prepare_uretprobe is not implemented for this arch!!!
+#endif
+#endif
+ return ret;
+}
+
+void
+arch_remove_kprobe (struct kprobe *p, struct task_struct *task)
+{
+ //mutex_lock(&kprobe_mutex);
+ if(p->tgid)
+ free_insn_slot (&uprobe_insn_pages, task, p->ainsn.insn, (p->ainsn.boostable == 1));
+ else
+ free_insn_slot (&kprobe_insn_pages, NULL, p->ainsn.insn, (p->ainsn.boostable == 1));
+ //mutex_unlock(&kprobe_mutex)
+}
+
+static unsigned long alloc_user_pages(struct task_struct *task, unsigned long len,
+ unsigned long prot, unsigned long flags, int atomic)
+{
+#if 1
+ long ret = 0;
+ struct task_struct *otask = current;
+ struct mm_struct *mm;
+
+ mm = atomic ? task->active_mm : get_task_mm (task);
+ if (mm){
+ if(!atomic)
+ down_write (&mm->mmap_sem);
+ // FIXME: its seems to be bad decision to replace 'current' pointer temporarily
+ current_thread_info()->task = task;
+ ret = (unsigned long)do_mmap_pgoff(0, 0, len, prot, flags, 0);
+ current_thread_info()->task = otask;
+ //printk ("mmap proc %p/%d %p/%d (%ld/%lx)\n", task, task->pid, current, current->pid, ret, ret);
+ if(!atomic){
+ up_write (&mm->mmap_sem);
+ mmput(mm);
+ }
+ /*if(ret < 0){
+ printk ("failed to mmap page in proc %d (%ld)", task->pid, ret);
+ ret = 0;
+ }*/
+ }
+ else
+ printk ("proc %d has no mm", task->pid);
+ return (unsigned long)ret;
+#else
+ struct file * file = 0;
+ unsigned long addr = 0, pgoff = 0;
+ struct mm_struct * mm = task->mm;
+ struct vm_area_struct * vma, * prev;
+ struct inode *inode;
+ unsigned int vm_flags;
+ int correct_wcount = 0;
+ int error;
+ struct rb_node ** rb_link, * rb_parent;
+ int accountable = 1;
+ unsigned long charged = 0, reqprot = prot;
+
+ if (file) {
+ if (is_file_hugepages(file))
+ accountable = 0;
+
+ if (!file->f_op || !file->f_op->mmap)
+ return -ENODEV;
+
+ if ((prot & PROT_EXEC) &&
+ (file->f_vfsmnt->mnt_flags & MNT_NOEXEC))
+ return -EPERM;
+ }
+ /*
+ * Does the application expect PROT_READ to imply PROT_EXEC?
+ *
+ * (the exception is when the underlying filesystem is noexec
+ * mounted, in which case we dont add PROT_EXEC.)
+ */
+ if ((prot & PROT_READ) && (task->personality & READ_IMPLIES_EXEC))
+ if (!(file && (file->f_vfsmnt->mnt_flags & MNT_NOEXEC)))
+ prot |= PROT_EXEC;
+
+ if (!len)
+ return -EINVAL;
+
+ /* Careful about overflows.. */
+ len = PAGE_ALIGN(len);
+ if (!len || len > TASK_SIZE)
+ return -ENOMEM;
+
+ /* offset overflow? */
+ if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
+ return -EOVERFLOW;
+
+ /* Too many mappings? */
+ if (mm->map_count > sysctl_max_map_count)
+ return -ENOMEM;
+
+ /* Obtain the address to map to. we verify (or select) it and ensure
+ * that it represents a valid section of the address space.
+ */
+ addr = get_unmapped_area(file, addr, len, pgoff, flags);
+ if (addr & ~PAGE_MASK)
+ return addr;
+
+ /* Do simple checking here so the lower-level routines won't have
+ * to. we assume access permissions have been handled by the open
+ * of the memory object, so we don't do any here.
+ */
+ vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
+ mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
+
+ if (flags & MAP_LOCKED) {
+ if (!can_do_mlock())
+ return -EPERM;
+ vm_flags |= VM_LOCKED;
+ }
+ /* mlock MCL_FUTURE? */
+ if (vm_flags & VM_LOCKED) {
+ unsigned long locked, lock_limit;
+ locked = len >> PAGE_SHIFT;
+ locked += mm->locked_vm;
+ lock_limit = task->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
+ lock_limit >>= PAGE_SHIFT;
+ if (locked > lock_limit && !capable(CAP_IPC_LOCK))
+ return -EAGAIN;
+ }
+
+ inode = file ? file->f_dentry->d_inode : NULL;
+
+ if (file) {
+ switch (flags & MAP_TYPE) {
+ case MAP_SHARED:
+ if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
+ return -EACCES;
+
+ /*
+ * Make sure we don't allow writing to an append-only
+ * file..
+ */
+ if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
+ return -EACCES;
+
+ /*
+ * Make sure there are no mandatory locks on the file.
+ */
+ if (locks_verify_locked(inode))
+ return -EAGAIN;
+
+ vm_flags |= VM_SHARED | VM_MAYSHARE;
+ if (!(file->f_mode & FMODE_WRITE))
+ vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
+
+ /* fall through */
+ case MAP_PRIVATE:
+ if (!(file->f_mode & FMODE_READ))
+ return -EACCES;
+ break;
+
+ default:
+ return -EINVAL;
+ }
+ } else {
+ switch (flags & MAP_TYPE) {
+ case MAP_SHARED:
+ vm_flags |= VM_SHARED | VM_MAYSHARE;
+ break;
+ case MAP_PRIVATE:
+ /*
+ * Set pgoff according to addr for anon_vma.
+ */
+ pgoff = addr >> PAGE_SHIFT;
+ break;
+ default:
+ return -EINVAL;
+ }
+ }
+
+
+ error = security_file_mmap(file, reqprot, prot, flags);
+ if (error)
+ return error;
+
+ /* Clear old maps */
+ error = -ENOMEM;
+munmap_back:
+ vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
+ if (vma && vma->vm_start < addr + len) {
+ if (do_munmap(mm, addr, len))
+ return -ENOMEM;
+ goto munmap_back;
+ }
+
+ /* Check against address space limit. */
+ if (!may_expand_vm(mm, len >> PAGE_SHIFT))
+ return -ENOMEM;
+
+ if (accountable && (!(flags & MAP_NORESERVE) ||
+ sysctl_overcommit_memory == OVERCOMMIT_NEVER)) {
+ if (vm_flags & VM_SHARED) {
+ /* Check memory availability in shmem_file_setup? */
+ vm_flags |= VM_ACCOUNT;
+ } else if (vm_flags & VM_WRITE) {
+ /*
+ * Private writable mapping: check memory availability
+ */
+ charged = len >> PAGE_SHIFT;
+ if (security_vm_enough_memory(charged))
+ return -ENOMEM;
+ vm_flags |= VM_ACCOUNT;
+ }
+ }
+
+ /*
+ * Can we just expand an old private anonymous mapping?
+ * The VM_SHARED test is necessary because shmem_zero_setup
+ * will create the file object for a shared anonymous map below.
+ */
+ if (!file && !(vm_flags & VM_SHARED) &&
+ vma_merge(mm, prev, addr, addr + len, vm_flags,
+ NULL, NULL, pgoff, NULL))
+ goto out;
+
+ /*
+ * Determine the object being mapped and call the appropriate
+ * specific mapper. the address has already been validated, but
+ * not unmapped, but the maps are removed from the list.
+ */
+ vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
+ if (!vma) {
+ error = -ENOMEM;
+ goto unacct_error;
+ }
+ memset(vma, 0, sizeof(*vma));
+
+ vma->vm_mm = mm;
+ vma->vm_start = addr;
+ vma->vm_end = addr + len;
+ vma->vm_flags = vm_flags;
+ vma->vm_page_prot = protection_map[vm_flags & 0x0f];
+ vma->vm_pgoff = pgoff;
+
+ if (file) {
+ error = -EINVAL;
+ if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
+ goto free_vma;
+ if (vm_flags & VM_DENYWRITE) {
+ error = deny_write_access(file);
+ if (error)
+ goto free_vma;
+ correct_wcount = 1;
+ }
+ vma->vm_file = file;
+ get_file(file);
+ error = file->f_op->mmap(file, vma);
+ if (error)
+ goto unmap_and_free_vma;
+ } else if (vm_flags & VM_SHARED) {
+ error = shmem_zero_setup(vma);
+ if (error)
+ goto free_vma;
+ }
+
+ /* We set VM_ACCOUNT in a shared mapping's vm_flags, to inform
+ * shmem_zero_setup (perhaps called through /dev/zero's ->mmap)
+ * that memory reservation must be checked; but that reservation
+ * belongs to shared memory object, not to vma: so now clear it.
+ */
+ if ((vm_flags & (VM_SHARED|VM_ACCOUNT)) == (VM_SHARED|VM_ACCOUNT))
+ vma->vm_flags &= ~VM_ACCOUNT;
+
+ /* Can addr have changed??
+ *
+ * Answer: Yes, several device drivers can do it in their
+ * f_op->mmap method. -DaveM
+ */
+ addr = vma->vm_start;
+ pgoff = vma->vm_pgoff;
+ vm_flags = vma->vm_flags;
+
+ if (!file || !vma_merge(mm, prev, addr, vma->vm_end,
+ vma->vm_flags, NULL, file, pgoff, vma_policy(vma))) {
+ file = vma->vm_file;
+ vma_link(mm, vma, prev, rb_link, rb_parent);
+ if (correct_wcount)
+ atomic_inc(&inode->i_writecount);
+ } else {
+ if (file) {
+ if (correct_wcount)
+ atomic_inc(&inode->i_writecount);
+ fput(file);
+ }
+ mpol_free(vma_policy(vma));
+ kmem_cache_free(vm_area_cachep, vma);
+ }
+out:
+ mm->total_vm += len >> PAGE_SHIFT;
+ vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
+ if (vm_flags & VM_LOCKED) {
+ mm->locked_vm += len >> PAGE_SHIFT;
+ make_pages_present(addr, addr + len);
+ }
+ if (flags & MAP_POPULATE) {
+ up_write(&mm->mmap_sem);
+ sys_remap_file_pages(addr, len, 0,
+ pgoff, flags & MAP_NONBLOCK);
+ down_write(&mm->mmap_sem);
+ }
+ return addr;
+
+unmap_and_free_vma:
+ if (correct_wcount)
+ atomic_inc(&inode->i_writecount);
+ vma->vm_file = NULL;
+ fput(file);
+
+ /* Undo any partial mapping done by a device driver. */
+ unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
+ charged = 0;
+free_vma:
+ kmem_cache_free(vm_area_cachep, vma);
+unacct_error:
+ if (charged)
+ vm_unacct_memory(charged);
+ return error;
+#endif
+}
+
+static int __kprobes
+check_safety (void)
+{
+ int ret = 0;
+#if defined(CONFIG_PREEMPT) && defined(CONFIG_PM)
+ ret = freeze_processes ();
+ if (ret == 0)
+ {
+ struct task_struct *p, *q;
+ do_each_thread (p, q)
+ {
+ if (p != current && p->state == TASK_RUNNING && p->pid != 0)
+ {
+ printk ("Check failed: %s is running\n", p->comm);
+ ret = -1;
+ goto loop_end;
+ }
+ }
+ while_each_thread (p, q);
+ }
+loop_end:
+ thaw_processes ();
+#else
+ synchronize_sched ();
+#endif
+ return ret;
+}
+
+/**
+ * get_us_insn_slot() - Find a slot on an executable page for an instruction.
+ * We allocate an executable page if there's no room on existing ones.
+ */
+kprobe_opcode_t __kprobes *
+get_insn_slot (struct task_struct *task, int atomic)
+{
+ struct kprobe_insn_page *kip;
+ struct hlist_node *pos;
+ struct hlist_head *page_list = task ? &uprobe_insn_pages : &kprobe_insn_pages;
+ unsigned slots_per_page = INSNS_PER_PAGE, slot_size = MAX_INSN_SIZE;
+
+ if(task) {
+ slots_per_page = INSNS_PER_PAGE/UPROBES_TRAMP_LEN;
+ slot_size = UPROBES_TRAMP_LEN;
+ }
+ else {
+ slots_per_page = INSNS_PER_PAGE/KPROBES_TRAMP_LEN;
+ slot_size = KPROBES_TRAMP_LEN;
+ }
+
+retry:
+ hlist_for_each_entry (kip, pos, page_list, hlist)
+ {
+ if (kip->nused < slots_per_page)
+ {
+ int i;
+ for (i = 0; i < slots_per_page; i++)
+ {
+ if (kip->slot_used[i] == SLOT_CLEAN)
+ {
+ if(!task || (kip->tgid == task->tgid)){
+ kip->slot_used[i] = SLOT_USED;
+ kip->nused++;
+ return kip->insns + (i * slot_size);
+ }
+ }
+ }
+ /* Surprise! No unused slots. Fix kip->nused. */
+ kip->nused = slots_per_page;
+ }
+ }
+
+ /* If there are any garbage slots, collect it and try again. */
+ if(task) {
+ if (uprobe_garbage_slots && collect_garbage_slots(page_list, task) == 0)
+ goto retry;
+ }
+ else {
+ if (kprobe_garbage_slots && collect_garbage_slots(page_list, task) == 0)
+ goto retry;
+ }
+
+ /* All out of space. Need to allocate a new page. Use slot 0. */
+ kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);
+ if (!kip)
+ return NULL;
+
+ kip->slot_used = kmalloc(sizeof(char)*slots_per_page, GFP_KERNEL);
+ if (!kip->slot_used){
+ kfree(kip);
+ return NULL;
+ }
+
+ if(task) {
+ kip->insns = (kprobe_opcode_t *)alloc_user_pages(task, PAGE_SIZE,
+ PROT_EXEC|PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_SHARED, atomic);
+ }
+ else {
+#if defined(CONFIG_X86)
+ kip->insns = module_alloc (PAGE_SIZE);
+#else
+ kip->insns = kmalloc(PAGE_SIZE, GFP_KERNEL);
+#endif
+ }
+ if (!kip->insns)
+ {
+ kfree (kip->slot_used);
+ kfree (kip);
+ return NULL;
+ }
+ INIT_HLIST_NODE (&kip->hlist);
+ hlist_add_head (&kip->hlist, page_list);
+ memset(kip->slot_used, SLOT_CLEAN, slots_per_page);
+ kip->slot_used[0] = SLOT_USED;
+ kip->nused = 1;
+ kip->ngarbage = 0;
+ kip->tgid = task ? task->tgid : 0;
+ return kip->insns;
+}
+
+/* Return 1 if all garbages are collected, otherwise 0. */
+static int __kprobes
+collect_one_slot (struct hlist_head *page_list, struct task_struct *task,
+ struct kprobe_insn_page *kip, int idx)
+{
+ struct mm_struct *mm;
+
+ kip->slot_used[idx] = SLOT_CLEAN;
+ kip->nused--;
+ DBPRINTF("collect_one_slot: nused=%d", kip->nused);
+ if (kip->nused == 0)
+ {
+ /*
+ * Page is no longer in use. Free it unless
+ * it's the last one. We keep the last one
+ * so as not to have to set it up again the
+ * next time somebody inserts a probe.
+ */
+ hlist_del (&kip->hlist);
+ if (!task && hlist_empty (page_list))
+ {
+ INIT_HLIST_NODE (&kip->hlist);
+ hlist_add_head (&kip->hlist, page_list);
+ }
+ else
+ {
+ if(task){
+ //printk("collect_one_slot %p/%d\n", task, task->pid);
+ mm = get_task_mm (task);
+ if (mm){
+ down_write (&mm->mmap_sem);
+ do_munmap(mm, (unsigned long)(kip->insns), PAGE_SIZE);
+ up_write (&mm->mmap_sem);
+ mmput(mm);
+ }
+ kip->tgid = 0;
+ }
+ else {
+#if defined(CONFIG_X86)
+ module_free (NULL, kip->insns);
+#else
+ vfree(kip->insns);
+#endif
+ }
+ kfree (kip->slot_used);
+ kfree (kip);
+ }
+ return 1;
+ }
+ return 0;
+}
+
+static int __kprobes
+collect_garbage_slots (struct hlist_head *page_list, struct task_struct *task)
+{
+ struct kprobe_insn_page *kip;
+ struct hlist_node *pos, *next;
+ unsigned slots_per_page = INSNS_PER_PAGE;
+
+ /* Ensure no-one is preepmted on the garbages */
+ if (!task && check_safety() != 0)
+ return -EAGAIN;
+
+ if(task)
+ slots_per_page = INSNS_PER_PAGE/UPROBES_TRAMP_LEN;
+ else
+ slots_per_page = INSNS_PER_PAGE/KPROBES_TRAMP_LEN;
+
+ hlist_for_each_entry_safe (kip, pos, next, page_list, hlist)
+ {
+ int i;
+ if ((task && (kip->tgid != task->tgid)) || (kip->ngarbage == 0))
+ continue;
+ kip->ngarbage = 0; /* we will collect all garbages */
+ for (i = 0; i < slots_per_page; i++)
+ {
+ if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot (page_list, task, kip, i))
+ break;
+ }
+ }
+ if(task) uprobe_garbage_slots = 0;
+ else kprobe_garbage_slots = 0;
+ return 0;
+}
+
+void purge_garbage_uslots(struct task_struct *task, int atomic)
+{
+ if(collect_garbage_slots(&uprobe_insn_pages, task))
+ panic("failed to collect garbage slotsfo for task %s/%d/%d", task->comm, task->tgid, task->pid);
+}
+
+void __kprobes
+free_insn_slot (struct hlist_head *page_list, struct task_struct *task, kprobe_opcode_t *slot, int dirty)
+{
+ struct kprobe_insn_page *kip;
+ struct hlist_node *pos;
+ unsigned slots_per_page = INSNS_PER_PAGE, slot_size = MAX_INSN_SIZE;
+
+ if(task) {
+ slots_per_page = INSNS_PER_PAGE/UPROBES_TRAMP_LEN;
+ slot_size = UPROBES_TRAMP_LEN;
+ }
+ else {
+ slots_per_page = INSNS_PER_PAGE/KPROBES_TRAMP_LEN;
+ slot_size = KPROBES_TRAMP_LEN;
+ }
+
+ DBPRINTF("free_insn_slot: dirty %d, %p/%d", dirty, task, task?task->pid:0);
+ hlist_for_each_entry (kip, pos, page_list, hlist)
+ {
+ DBPRINTF("free_insn_slot: kip->insns=%p slot=%p", kip->insns, slot);
+ if ((kip->insns <= slot) && (slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)))
+ {
+ int i = (slot - kip->insns) / slot_size;
+ if (dirty)
+ {
+ kip->slot_used[i] = SLOT_DIRTY;
+ kip->ngarbage++;
+ }
+ else
+ {
+ collect_one_slot (page_list, task, kip, i);
+ }
+ break;
+ }
+ }
+
+ if (dirty){
+ if(task){
+ if(++uprobe_garbage_slots > slots_per_page)
+ collect_garbage_slots (page_list, task);
+ }
+ else if(++kprobe_garbage_slots > slots_per_page)
+ collect_garbage_slots (page_list, task);
+ }
+}
+
+static void
+prepare_singlestep (struct kprobe *p, struct pt_regs *regs)
+{
+#if defined(CONFIG_X86)
+ if(p->ss_addr)
+ {
+ regs->EREG (ip) = (unsigned long)p->ss_addr;
+ p->ss_addr = NULL;
+ }
+ else
+ {
+ regs->EREG (flags) |= TF_MASK;
+ regs->EREG (flags) &= ~IF_MASK;
+ /*single step inline if the instruction is an int3 */
+ if (p->opcode == BREAKPOINT_INSTRUCTION){
+ regs->EREG (ip) = (unsigned long) p->addr;
+ //printk("break_insn!!!\n");
+ }
+ else
+ regs->EREG (ip) = (unsigned long) p->ainsn.insn;
+ }
+ //printk("singlestep %p/%lx\n", p->addr, p->ainsn.insn);
+#elif defined(CONFIG_ARM)
+ if(p->ss_addr)
+ {
+ regs->uregs[15] = (unsigned long) p->ss_addr;
+ p->ss_addr = NULL;
+ }
+ else
+ regs->uregs[15] = (unsigned long) p->ainsn.insn;
+ //DBPRINTF("prepare_singlestep: %p/%p/%d\n", p, p->addr, p->ainsn.boostable);
+#elif defined(CONFIG_MIPS)
+ if(p->ss_addr)
+ {
+ regs->cp0_epc = (unsigned long) p->ss_addr;
+ p->ss_addr = NULL;
+ }
+ else
+ regs->cp0_epc = (unsigned long) p->ainsn.insn;
+#endif
+ //if(p->tgid)
+ //printk("prepare_singlestep: %p/%d to %lx\n", p->addr, p->ainsn.boostable, regs->EREG (ip));
+ //printk("SS[%lx] to %lx/%lx/%lx\n", p->addr, regs->uregs[15], p->ss_addr, p);
+}
+
+static void __kprobes
+save_previous_kprobe (struct kprobe_ctlblk *kcb, struct kprobe *cur_p)
+{
+ if (kcb->prev_kprobe.kp != NULL)
+ {
+ panic ("no space to save new probe[%lu]: task = %d/%s, prev %d/%p, current %d/%p, new %d/%p,",
+ nCount, current->pid, current->comm, kcb->prev_kprobe.kp->tgid, kcb->prev_kprobe.kp->addr,
+ kprobe_running()->tgid, kprobe_running()->addr, cur_p->tgid, cur_p->addr);
+ }
+#if defined(CONFIG_X86)
+ kcb->prev_kprobe.old_eflags = kcb->kprobe_old_eflags;
+ kcb->prev_kprobe.saved_eflags = kcb->kprobe_saved_eflags;
+#endif
+ kcb->prev_kprobe.kp = kprobe_running ();
+ kcb->prev_kprobe.status = kcb->kprobe_status;
+}
+
+static void __kprobes
+restore_previous_kprobe (struct kprobe_ctlblk *kcb)
+{
+ __get_cpu_var (current_kprobe) = kcb->prev_kprobe.kp;
+ kcb->kprobe_status = kcb->prev_kprobe.status;
+ kcb->prev_kprobe.kp = NULL;
+ kcb->prev_kprobe.status = 0;
+#if defined(CONFIG_X86)
+ kcb->kprobe_old_eflags = kcb->prev_kprobe.old_eflags;
+ kcb->kprobe_saved_eflags = kcb->prev_kprobe.saved_eflags;
+#endif
+}
+
+static void __kprobes
+set_current_kprobe (struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb)
+{
+ __get_cpu_var (current_kprobe) = p;
+ DBPRINTF ("set_current_kprobe[%lu]: p=%p addr=%p\n", nCount, p, p->addr);
+#if defined(CONFIG_X86)
+ kcb->kprobe_saved_eflags = kcb->kprobe_old_eflags = (regs->EREG (flags) & (TF_MASK | IF_MASK));
+ if (is_IF_modifier (p->opcode))
+ kcb->kprobe_saved_eflags &= ~IF_MASK;
+#endif
+}
+
+#define REENTER
+
+#ifdef _DEBUG
+int gSilent = 1;
+#endif
+
+#if defined(CONFIG_X86)
+int
+kprobe_handler (struct pt_regs *regs)
+{
+ struct kprobe *p = 0;
+ int ret = 0, pid = 0, retprobe = 0, reenter = 0;
+ kprobe_opcode_t *addr = NULL;
+ struct kprobe_ctlblk *kcb;
+
+ nCount++;
+
+ /* We're in an interrupt, but this is clear and BUG()-safe. */
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+
+ addr = (kprobe_opcode_t *) (regs->EREG (ip) - sizeof (kprobe_opcode_t));
+ DBPRINTF ("KPROBE[%lu]: regs->eip = 0x%lx addr = 0x%p\n", nCount, regs->EREG (ip), addr);
+
+ preempt_disable ();
+
+ kcb = get_kprobe_ctlblk ();
+
+ if (user_mode_vm(regs))
+ {
+#ifdef _DEBUG
+ gSilent = 0;
+#endif
+ //printk("exception[%lu] from user mode %s/%u/%u addr %p.\n", nCount, current->comm, current->pid, current->tgid, addr);
+ pid = current->tgid;
+ }
+
+ /* Check we're not actually recursing */
+ if (kprobe_running ())
+ {
+ DBPRINTF ("lock???");
+ p = get_kprobe (addr, pid, current);
+ if (p)
+ {
+ DBPRINTF ("reenter p = %p", p);
+ if(!pid){
+ if (kcb->kprobe_status == KPROBE_HIT_SS && *p->ainsn.insn == BREAKPOINT_INSTRUCTION)
+ {
+ regs->EREG (flags) &= ~TF_MASK;
+ regs->EREG (flags) |= kcb->kprobe_saved_eflags;
+ goto no_kprobe;
+ }
+ }
+ else {
+ //#warning BREAKPOINT_INSTRUCTION user mode handling is missed!!!
+ }
+
+ /* We have reentered the kprobe_handler(), since
+ * another probe was hit while within the handler.
+ * We here save the original kprobes variables and
+ * just single step on the instruction of the new probe
+ * without calling any user handlers.
+ */
+ save_previous_kprobe (kcb, p);
+ set_current_kprobe (p, regs, kcb);
+ kprobes_inc_nmissed_count (p);
+ prepare_singlestep (p, regs);
+ kcb->kprobe_status = KPROBE_REENTER;
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+ return 1;
+ }
+ else
+ {
+ if(!pid){
+ if (*addr != BREAKPOINT_INSTRUCTION)
+ {
+ /* The breakpoint instruction was removed by
+ * another cpu right after we hit, no further
+ * handling of this interrupt is appropriate
+ */
+ regs->EREG (ip) -= sizeof (kprobe_opcode_t);
+ ret = 1;
+ goto no_kprobe;
+ }
+ }
+ else {
+ //#warning BREAKPOINT_INSTRUCTION user mode handling is missed!!!
+ //we can reenter probe upon uretprobe exception
+ DBPRINTF ("check for UNDEF_INSTRUCTION %p\n", addr);
+ // UNDEF_INSTRUCTION from user space
+ p = get_kprobe_by_insn_slot (addr-UPROBES_TRAMP_RET_BREAK_IDX, pid, current);
+ if (p) {
+ save_previous_kprobe (kcb, p);
+ kcb->kprobe_status = KPROBE_REENTER;
+ reenter = 1;
+ retprobe = 1;
+ DBPRINTF ("uretprobe %p\n", addr);
+ }
+ }
+ if(!p){
+ p = __get_cpu_var (current_kprobe);
+ if(p->tgid)
+ panic("after uhandler");
+ DBPRINTF ("kprobe_running !!! p = 0x%p p->break_handler = 0x%p", p, p->break_handler);
+ if (p->break_handler && p->break_handler (p, regs))
+ {
+ DBPRINTF ("kprobe_running !!! goto ss");
+ goto ss_probe;
+ }
+ DBPRINTF ("kprobe_running !!! goto no");
+ DBPRINTF ("no_kprobe");
+ goto no_kprobe;
+ }
+ }
+ }
+
+ DBPRINTF ("get_kprobe %p", addr);
+ if (!p)
+ p = get_kprobe (addr, pid, current);
+ if (!p)
+ {
+ if(!pid){
+ if (*addr != BREAKPOINT_INSTRUCTION)
+ {
+ /*
+ * The breakpoint instruction was removed right
+ * after we hit it. Another cpu has removed
+ * either a probepoint or a debugger breakpoint
+ * at this address. In either case, no further
+ * handling of this interrupt is appropriate.
+ * Back up over the (now missing) int3 and run
+ * the original instruction.
+ */
+ regs->EREG (ip) -= sizeof (kprobe_opcode_t);
+ ret = 1;
+ }
+ }
+ else {
+ //#warning BREAKPOINT_INSTRUCTION user mode handling is missed!!!
+ DBPRINTF ("search UNDEF_INSTRUCTION %p\n", addr);
+ // UNDEF_INSTRUCTION from user space
+ p = get_kprobe_by_insn_slot (addr-UPROBES_TRAMP_RET_BREAK_IDX, pid, current);
+ if (!p) {
+ // Not one of ours: let kernel handle it
+ DBPRINTF ("no_kprobe");
+ //printk("no_kprobe2 ret = %d\n", ret);
+ goto no_kprobe;
+ }
+ retprobe = 1;
+ DBPRINTF ("uretprobe %p\n", addr);
+ }
+ if(!p) {
+ /* Not one of ours: let kernel handle it */
+ DBPRINTF ("no_kprobe");
+ goto no_kprobe;
+ }
+ }
+ set_current_kprobe (p, regs, kcb);
+ if(!reenter)
+ kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+
+ if (retprobe) //(einsn == UNDEF_INSTRUCTION)
+ ret = trampoline_probe_handler (p, regs);
+ else if (p->pre_handler)
+ ret = p->pre_handler (p, regs);
+
+ if (ret)
+ {
+ if (ret == 2) { // we have alreadyc called the handler, so just single step the instruction
+ DBPRINTF ("p->pre_handler[%lu] 2", nCount);
+ goto ss_probe;
+ }
+ DBPRINTF ("p->pre_handler[%lu] 1", nCount);
+ /* handler has already set things up, so skip ss setup */
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+ return 1;
+ }
+ DBPRINTF ("p->pre_handler[%lu] 0", nCount);
+
+ss_probe:
+ DBPRINTF ("p = %p\n", p);
+ DBPRINTF ("p->opcode = 0x%lx *p->addr = 0x%lx p->addr = 0x%p\n", (unsigned long) p->opcode, p->tgid ? 0 : (unsigned long) (*p->addr), p->addr);
+
+#if !defined(CONFIG_PREEMPT) || defined(CONFIG_PM)
+ if (p->ainsn.boostable == 1 && !p->post_handler)
+ {
+ /* Boost up -- we can execute copied instructions directly */
+ reset_current_kprobe ();
+ regs->EREG (ip) = (unsigned long) p->ainsn.insn;
+ preempt_enable_no_resched ();
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+ return 1;
+ }
+#endif // !CONFIG_PREEMPT
+ prepare_singlestep (p, regs);
+ kcb->kprobe_status = KPROBE_HIT_SS;
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+ return 1;
+
+no_kprobe:
+
+ preempt_enable_no_resched ();
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+ return ret;
+}
+
+#else
+int
+kprobe_handler (struct pt_regs *regs)
+{
+ struct kprobe *p = 0;
+ int ret = 0, pid = 0, retprobe = 0, reenter = 0;
+ kprobe_opcode_t *addr = NULL, *ssaddr = 0;
+ struct kprobe_ctlblk *kcb;
+
+ nCount++;
+ /* We're in an interrupt, but this is clear and BUG()-safe. */
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+#if defined(CONFIG_MIPS)
+ addr = (kprobe_opcode_t *) regs->cp0_epc;
+ DBPRINTF ("regs->regs[ 31 ] = 0x%lx\n", regs->regs[31]);
+#elif defined(CONFIG_ARM)
+ addr = (kprobe_opcode_t *) (regs->uregs[15] - 4);
+ DBPRINTF ("KPROBE[%lu]: regs->uregs[15] = 0x%lx addr = 0x%p\n", nCount, regs->uregs[15], addr);
+ regs->uregs[15] -= 4;
+ //DBPRINTF("regs->uregs[14] = 0x%lx\n", regs->uregs[14]);
+#else
+#error implement how to get exception address for this arch!!!
+#endif // ARCH
+
+ preempt_disable ();
+
+ kcb = get_kprobe_ctlblk ();
+
+ if (user_mode (regs))
+ {
+#ifdef _DEBUG
+ gSilent = 0;
+#endif
+ //DBPRINTF("exception[%lu] from user mode %s/%u addr %p (%lx).", nCount, current->comm, current->pid, addr, regs->uregs[14]);
+ pid = current->tgid;
+ }
+
+ /* Check we're not actually recursing */
+ if (kprobe_running ())
+ {
+ DBPRINTF ("lock???");
+ p = get_kprobe (addr, pid, current);
+ if (p)
+ {
+ if(!pid && (addr == (kprobe_opcode_t *)kretprobe_trampoline)){
+ save_previous_kprobe (kcb, p);
+ kcb->kprobe_status = KPROBE_REENTER;
+ reenter = 1;
+ }
+ else {
+ /* We have reentered the kprobe_handler(), since
+ * another probe was hit while within the handler.
+ * We here save the original kprobes variables and
+ * just single step on the instruction of the new probe
+ * without calling any user handlers.
+ */
+ if(!p->ainsn.boostable){
+ save_previous_kprobe (kcb, p);
+ set_current_kprobe (p, regs, kcb);
+ }
+ kprobes_inc_nmissed_count (p);
+ prepare_singlestep (p, regs);
+ if(!p->ainsn.boostable)
+ kcb->kprobe_status = KPROBE_REENTER;
+ preempt_enable_no_resched ();
+ return 1;
+ }
+ }
+ else
+ {
+ if(pid) { //we can reenter probe upon uretprobe exception
+ DBPRINTF ("check for UNDEF_INSTRUCTION %p\n", addr);
+ // UNDEF_INSTRUCTION from user space
+ p = get_kprobe_by_insn_slot (addr-UPROBES_TRAMP_RET_BREAK_IDX, pid, current);
+ if (p) {
+ save_previous_kprobe (kcb, p);
+ kcb->kprobe_status = KPROBE_REENTER;
+ reenter = 1;
+ retprobe = 1;
+ DBPRINTF ("uretprobe %p\n", addr);
+ }
+ }
+ if(!p) {
+ p = __get_cpu_var (current_kprobe);
+#ifdef _DEBUG
+ if (p->tgid) gSilent = 0;
+#endif
+ DBPRINTF ("kprobe_running !!! p = 0x%p p->break_handler = 0x%p", p, p->break_handler);
+ /*if (p->break_handler && p->break_handler(p, regs)) {
+ DBPRINTF("kprobe_running !!! goto ss");
+ goto ss_probe;
+ } */
+ DBPRINTF ("unknown uprobe at %p cur at %p/%p\n", addr, p->addr, p->ainsn.insn);
+ if(pid)
+ ssaddr = p->ainsn.insn + UPROBES_TRAMP_SS_BREAK_IDX;
+ else
+ ssaddr = p->ainsn.insn + KPROBES_TRAMP_SS_BREAK_IDX;
+ if (addr == ssaddr)
+ {
+#if defined(CONFIG_ARM)
+ regs->uregs[15] = (unsigned long) (p->addr + 1);
+ DBPRINTF ("finish step at %p cur at %p/%p, redirect to %lx\n", addr, p->addr, p->ainsn.insn, regs->uregs[15]);
+#elif defined(CONFIG_MIPS)
+ regs->cp0_epc = (unsigned long) (p->addr + 1);
+ DBPRINTF ("finish step at %p cur at %p/%p, redirect to %lx\n", addr, p->addr, p->ainsn.insn, regs->cp0_epc);
+#else
+#warning uprobe single step is not implemented for this arch!!!
+#endif
+ if (kcb->kprobe_status == KPROBE_REENTER) {
+ restore_previous_kprobe (kcb);
+ }
+ else {
+ reset_current_kprobe ();
+ }
+ }
+ DBPRINTF ("kprobe_running !!! goto no");
+ ret = 1;
+ /* If it's not ours, can't be delete race, (we hold lock). */
+ DBPRINTF ("no_kprobe");
+ goto no_kprobe;
+ }
+ }
+ }
+
+ //if(einsn != UNDEF_INSTRUCTION) {
+ DBPRINTF ("get_kprobe %p-%d", addr, pid);
+ if (!p)
+ p = get_kprobe (addr, pid, current);
+ if (!p)
+ {
+ if(pid) {
+ DBPRINTF ("search UNDEF_INSTRUCTION %p\n", addr);
+ // UNDEF_INSTRUCTION from user space
+ p = get_kprobe_by_insn_slot (addr-UPROBES_TRAMP_RET_BREAK_IDX, pid, current);
+ if (!p) {
+ /* Not one of ours: let kernel handle it */
+ DBPRINTF ("no_kprobe");
+ //printk("no_kprobe2 ret = %d\n", ret);
+ goto no_kprobe;
+ }
+ retprobe = 1;
+ DBPRINTF ("uretprobe %p\n", addr);
+ }
+ else {
+ /* Not one of ours: let kernel handle it */
+ DBPRINTF ("no_kprobe");
+ //printk("no_kprobe2 ret = %d\n", ret);
+ goto no_kprobe;
+ }
+ }
+#ifdef _DEBUG
+ if (p->tgid) gSilent = 0;
+#endif
+
+ set_current_kprobe (p, regs, kcb);
+ if(!reenter)
+ kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+
+ if (retprobe) //(einsn == UNDEF_INSTRUCTION)
+ ret = trampoline_probe_handler (p, regs);
+ else if (p->pre_handler)
+ {
+ ret = p->pre_handler (p, regs);
+ if(!p->ainsn.boostable)
+ kcb->kprobe_status = KPROBE_HIT_SS;
+ else if(p->pre_handler != trampoline_probe_handler)
+ reset_current_kprobe ();
+ }
+
+ if (ret)
+ {
+ DBPRINTF ("p->pre_handler[%lu] 1", nCount);
+ /* handler has already set things up, so skip ss setup */
+ return 1;
+ }
+ DBPRINTF ("p->pre_handler 0");
+
+no_kprobe:
+ preempt_enable_no_resched ();
+#ifdef _DEBUG
+ gSilent = 1;
+#endif
+ return ret;
+}
+#endif
+
+extern struct kretprobe *sched_rp;
+
+static void patch_suspended_task_ret_addr(struct task_struct *p, struct kretprobe *rp)
+{
+ struct kretprobe_instance *ri = NULL;
+ struct hlist_node *node, *tmp;
+ struct hlist_head *head;
+ unsigned long flags;
+ int found = 0;
+
+ spin_lock_irqsave (&kretprobe_lock, flags);
+ head = kretprobe_inst_table_head (p);
+ hlist_for_each_entry_safe (ri, node, tmp, head, hlist){
+ if ((ri->rp == rp) && (p == ri->task)){
+ found = 1;
+ break;
+ }
+ }
+ spin_unlock_irqrestore (&kretprobe_lock, flags);
+
+#ifdef CONFIG_ARM
+
+#ifndef task_thread_info
+#define task_thread_info(task) (task)->thread_info
+#endif // task_thread_info
+
+ if (found){
+ // update PC
+ if(thread_saved_pc(p) != (unsigned long)&kretprobe_trampoline){
+ ri->ret_addr = (kprobe_opcode_t *)thread_saved_pc(p);
+ task_thread_info(p)->cpu_context.pc = (unsigned long) &kretprobe_trampoline;
+ }
+ return;
+ }
+
+ if ((ri = get_free_rp_inst(rp)) != NULL)
+ {
+ ri->rp = rp;
+ ri->rp2 = NULL;
+ ri->task = p;
+ ri->ret_addr = (kprobe_opcode_t *)thread_saved_pc(p);
+ task_thread_info(p)->cpu_context.pc = (unsigned long) &kretprobe_trampoline;
+ add_rp_inst (ri);
+// printk("change2 saved pc %p->%p for %d/%d/%p\n", ri->ret_addr, &kretprobe_trampoline, p->tgid, p->pid, p);
+ }
+ else{
+ printk("no ri for %d\n", p->pid);
+ BUG();
+ }
+#endif // CONFIG_ARM
+}
+
+typedef kprobe_opcode_t (*entry_point_t) (unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long);
+int
+setjmp_pre_handler (struct kprobe *p, struct pt_regs *regs)
+{
+ struct jprobe *jp = container_of (p, struct jprobe, kp);
+ kprobe_pre_entry_handler_t pre_entry;
+ entry_point_t entry;
+
+#if defined(CONFIG_X86)
+ unsigned long addr, args[6];
+ struct kprobe_ctlblk *kcb = get_kprobe_ctlblk ();
+
+ DBPRINTF ("setjmp_pre_handler %p:%d", p->addr, p->tgid);
+ pre_entry = (kprobe_pre_entry_handler_t) jp->pre_entry;
+ entry = (entry_point_t) jp->entry;
+ if(p->tgid) {
+ regs->EREG (flags) &= ~IF_MASK;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
+ trace_hardirqs_off ();
+#endif
+ if (p->tgid == current->tgid)
+ {
+ // read first 6 args from stack
+ if (!read_proc_vm_atomic (current, regs->EREG(sp)+4, args, sizeof(args)))
+ panic ("failed to read user space func arguments %lx!\n", regs->EREG(sp)+4);
+ if (pre_entry)
+ p->ss_addr = pre_entry (jp->priv_arg, regs);
+ if (entry)
+ entry (args[0], args[1], args[2], args[3], args[4], args[5]);
+ }
+ else
+ uprobe_return ();
+
+ return 2;
+ }
+ else {
+ kcb->jprobe_saved_regs = *regs;
+ kcb->jprobe_saved_esp = ®s->EREG (sp);
+ addr = (unsigned long) (kcb->jprobe_saved_esp);
+
+ /*
+ * TBD: As Linus pointed out, gcc assumes that the callee
+ * owns the argument space and could overwrite it, e.g.
+ * tailcall optimization. So, to be absolutely safe
+ * we also save and restore enough stack bytes to cover
+ * the argument area.
+ */
+ memcpy (kcb->jprobes_stack, (kprobe_opcode_t *) addr, MIN_STACK_SIZE (addr));
+ regs->EREG (flags) &= ~IF_MASK;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
+ trace_hardirqs_off ();
+#endif
+ if (pre_entry)
+ p->ss_addr = pre_entry (jp->priv_arg, regs);
+ regs->EREG (ip) = (unsigned long) (jp->entry);
+ }
+
+ return 1;
+#else //!CONFIG_X86
+# ifdef REENTER
+ p = __get_cpu_var (current_kprobe);
+# endif
+
+ DBPRINTF ("pjp = 0x%p jp->entry = 0x%p", jp, jp->entry);
+ entry = (entry_point_t) jp->entry;
+ pre_entry = (kprobe_pre_entry_handler_t) jp->pre_entry;
+ //if(!entry)
+ // DIE("entry NULL", regs)
+ DBPRINTF ("entry = 0x%p jp->entry = 0x%p", entry, jp->entry);
+
+ //call handler for all kernel probes and user space ones which belong to current tgid
+ if (!p->tgid || (p->tgid == current->tgid))
+ {
+ if(!p->tgid && (p->addr == sched_addr) && sched_rp){
+ struct task_struct *p, *g;
+ rcu_read_lock();
+ //swapper task
+ if(current != &init_task)
+ patch_suspended_task_ret_addr(&init_task, sched_rp);
+ // other tasks
+ do_each_thread(g, p){
+ if(p == current)
+ continue;
+ patch_suspended_task_ret_addr(p, sched_rp);
+ } while_each_thread(g, p);
+ rcu_read_unlock();
+ }
+ if (pre_entry)
+ p->ss_addr = pre_entry (jp->priv_arg, regs);
+ if (entry){
+# if defined(CONFIG_MIPS)
+ entry (regs->regs[4], regs->regs[5], regs->regs[6], regs->regs[7], regs->regs[8], regs->regs[9]);
+# elif defined(CONFIG_ARM)
+ entry (regs->ARM_r0, regs->ARM_r1, regs->ARM_r2, regs->ARM_r3, regs->ARM_r4, regs->ARM_r5);
+# endif // ARCH
+ }
+ else {
+ if (p->tgid)
+ uprobe_return ();
+ else
+ jprobe_return ();
+ }
+ }
+ else if (p->tgid)
+ uprobe_return ();
+
+ prepare_singlestep (p, regs);
+# ifdef _DEBUG
+ p->step_count++;
+# endif
+
+ return 1;
+#endif //!CONFIG_X86
+}
+
+void __kprobes
+jprobe_return (void)
+{
+#if defined(CONFIG_X86)
+ struct kprobe_ctlblk *kcb = get_kprobe_ctlblk ();
+
+ asm volatile(" xchgl %%ebx,%%esp \n"
+ " int3 \n"
+ " .globl jprobe_return_end \n"
+ " jprobe_return_end: \n"
+ " nop \n"::"b" (kcb->jprobe_saved_esp):"memory");
+#else
+ preempt_enable_no_resched();
+#endif
+}
+
+void __kprobes
+uprobe_return (void)
+{
+#if defined(CONFIG_X86)
+ /*struct kprobe_ctlblk *kcb = get_kprobe_ctlblk ();
+
+ asm volatile(" xchgl %%ebx,%%esp \n"
+ " int3 \n"
+ " .globl jprobe_return_end \n"
+ " jprobe_return_end: \n"
+ " nop \n"::"b" (kcb->jprobe_saved_esp):"memory");*/
+#else
+ preempt_enable_no_resched ();
+#endif
+}
+
+#if defined(CONFIG_X86)
+/*
+ * Called after single-stepping. p->addr is the address of the
+ * instruction whose first byte has been replaced by the "int 3"
+ * instruction. To avoid the SMP problems that can occur when we
+ * temporarily put back the original opcode to single-step, we
+ * single-stepped a copy of the instruction. The address of this
+ * copy is p->ainsn.insn.
+ *
+ * This function prepares to return from the post-single-step
+ * interrupt. We have to fix up the stack as follows:
+ *
+ * 0) Except in the case of absolute or indirect jump or call instructions,
+ * the new eip is relative to the copied instruction. We need to make
+ * it relative to the original instruction.
+ *
+ * 1) If the single-stepped instruction was pushfl, then the TF and IF
+ * flags are set in the just-pushed eflags, and may need to be cleared.
+ *
+ * 2) If the single-stepped instruction was a call, the return address
+ * that is atop the stack is the address following the copied instruction.
+ * We need to make it the address following the original instruction.
+ *
+ * This function also checks instruction size for preparing direct execution.
+ */
+static void __kprobes
+resume_execution (struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb)
+{
+ unsigned long *tos, tos_dword = 0;
+ unsigned long copy_eip = (unsigned long) p->ainsn.insn;
+ unsigned long orig_eip = (unsigned long) p->addr;
+ kprobe_opcode_t insns[2];
+
+ regs->EREG (flags) &= ~TF_MASK;
+
+ if(p->tgid){
+ tos = (unsigned long *) &tos_dword;
+ if (!read_proc_vm_atomic (current, regs->EREG (sp), &tos_dword, sizeof(tos_dword)))
+ panic ("failed to read dword from top of the user space stack %lx!\n", regs->EREG (sp));
+ if (!read_proc_vm_atomic (current, (unsigned long)p->ainsn.insn, insns, 2*sizeof(kprobe_opcode_t)))
+ panic ("failed to read first 2 opcodes of instruction copy from user space %p!\n", p->ainsn.insn);
+ }
+ else {
+ tos = (unsigned long *) ®s->EREG (sp);
+ insns[0] = p->ainsn.insn[0];
+ insns[1] = p->ainsn.insn[1];
+ }
+
+ switch (insns[0])
+ {
+ case 0x9c: /* pushfl */
+ *tos &= ~(TF_MASK | IF_MASK);
+ *tos |= kcb->kprobe_old_eflags;
+ break;
+ case 0xc2: /* iret/ret/lret */
+ case 0xc3:
+ case 0xca:
+ case 0xcb:
+ case 0xcf:
+ case 0xea: /* jmp absolute -- eip is correct */
+ /* eip is already adjusted, no more changes required */
+ p->ainsn.boostable = 1;
+ goto no_change;
+ case 0xe8: /* call relative - Fix return addr */
+ *tos = orig_eip + (*tos - copy_eip);
+ break;
+ case 0x9a: /* call absolute -- same as call absolute, indirect */
+ *tos = orig_eip + (*tos - copy_eip);
+ if(p->tgid){
+ if (!write_proc_vm_atomic (current, regs->EREG (sp), &tos_dword, sizeof(tos_dword)))
+ panic ("failed to write dword to top of the user space stack %lx!\n", regs->EREG (sp));
+ }
+ goto no_change;
+ case 0xff:
+ if ((insns[1] & 0x30) == 0x10)
+ {
+ /*
+ * call absolute, indirect
+ * Fix return addr; eip is correct.
+ * But this is not boostable
+ */
+ *tos = orig_eip + (*tos - copy_eip);
+ if(p->tgid){
+ if (!write_proc_vm_atomic (current, regs->EREG (sp), &tos_dword, sizeof(tos_dword)))
+ panic ("failed to write dword to top of the user space stack %lx!\n", regs->EREG (sp));
+ }
+ goto no_change;
+ }
+ else if (((insns[1] & 0x31) == 0x20) || /* jmp near, absolute indirect */
+ ((insns[1] & 0x31) == 0x21))
+ { /* jmp far, absolute indirect */
+ /* eip is correct. And this is boostable */
+ p->ainsn.boostable = 1;
+ goto no_change;
+ }
+ default:
+ break;
+ }
+
+ if(p->tgid){
+ if (!write_proc_vm_atomic (current, regs->EREG (sp), &tos_dword, sizeof(tos_dword)))
+ panic ("failed to write dword to top of the user space stack %lx!\n", regs->EREG (sp));
+ }
+
+ if (p->ainsn.boostable == 0)
+ {
+ if ((regs->EREG (ip) > copy_eip) && (regs->EREG (ip) - copy_eip) + 5 < MAX_INSN_SIZE)
+ {
+ /*
+ * These instructions can be executed directly if it
+ * jumps back to correct address.
+ */
+ if(p->tgid)
+ set_user_jmp_op ((void *) regs->EREG (ip), (void *) orig_eip + (regs->EREG (ip) - copy_eip));
+ else
+ set_jmp_op ((void *) regs->EREG (ip), (void *) orig_eip + (regs->EREG (ip) - copy_eip));
+ p->ainsn.boostable = 1;
+ }
+ else
+ {
+ p->ainsn.boostable = -1;
+ }
+ }
+
+ regs->EREG (ip) = orig_eip + (regs->EREG (ip) - copy_eip);
+
+no_change:
+ return;
+}
+
+/*
+ * Interrupts are disabled on entry as trap1 is an interrupt gate and they
+ * remain disabled thoroughout this function.
+ */
+static int __kprobes
+post_kprobe_handler (struct pt_regs *regs)
+{
+ struct kprobe *cur = kprobe_running ();
+ struct kprobe_ctlblk *kcb = get_kprobe_ctlblk ();
+
+ if (!cur)
+ return 0;
+ if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler)
+ {
+ kcb->kprobe_status = KPROBE_HIT_SSDONE;
+ cur->post_handler (cur, regs, 0);
+ }
+
+ resume_execution (cur, regs, kcb);
+ regs->EREG (flags) |= kcb->kprobe_saved_eflags;
+#ifndef CONFIG_X86
+ trace_hardirqs_fixup_flags (regs->EREG (flags));
+#endif // CONFIG_X86
+ /*Restore back the original saved kprobes variables and continue. */
+ if (kcb->kprobe_status == KPROBE_REENTER)
+ {
+ restore_previous_kprobe (kcb);
+ goto out;
+ }
+ reset_current_kprobe ();
+out:
+ preempt_enable_no_resched ();
+
+ /*
+ * if somebody else is singlestepping across a probe point, eflags
+ * will have TF set, in which case, continue the remaining processing
+ * of do_debug, as if this is not a probe hit.
+ */
+ if (regs->EREG (flags) & TF_MASK)
+ return 0;
+
+ return 1;
+}
+
+static int __kprobes
+kprobe_fault_handler (struct pt_regs *regs, int trapnr)
+{
+ struct kprobe *cur = kprobe_running ();
+ struct kprobe_ctlblk *kcb = get_kprobe_ctlblk ();
+
+ switch (kcb->kprobe_status)
+ {
+ case KPROBE_HIT_SS:
+ case KPROBE_REENTER:
+ /*
+ * We are here because the instruction being single
+ * stepped caused a page fault. We reset the current
+ * kprobe and the eip points back to the probe address
+ * and allow the page fault handler to continue as a
+ * normal page fault.
+ */
+ regs->EREG (ip) = (unsigned long) cur->addr;
+ regs->EREG (flags) |= kcb->kprobe_old_eflags;
+ if (kcb->kprobe_status == KPROBE_REENTER)
+ restore_previous_kprobe (kcb);
+ else
+ reset_current_kprobe ();
+ preempt_enable_no_resched ();
+ break;
+ case KPROBE_HIT_ACTIVE:
+ case KPROBE_HIT_SSDONE:
+ /*
+ * We increment the nmissed count for accounting,
+ * we can also use npre/npostfault count for accouting
+ * these specific fault cases.
+ */
+ kprobes_inc_nmissed_count (cur);
+
+ /*
+ * We come here because instructions in the pre/post
+ * handler caused the page_fault, this could happen
+ * if handler tries to access user space by
+ * copy_from_user(), get_user() etc. Let the
+ * user-specified handler try to fix it first.
+ */
+ if (cur->fault_handler && cur->fault_handler (cur, regs, trapnr))
+ return 1;
+
+ /*
+ * In case the user-specified fault handler returned
+ * zero, try to fix up.
+ */
+ if (fixup_exception (regs))
+ return 1;
+
+ /*
+ * fixup_exception() could not handle it,
+ * Let do_page_fault() fix it.
+ */
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+int
+kprobe_exceptions_notify (struct notifier_block *self, unsigned long val, void *data)
+{
+ struct die_args *args = (struct die_args *) data;
+ int ret = NOTIFY_DONE;
+
+ DBPRINTF ("val = %ld, data = 0x%X", val, (unsigned int) data);
+
+ /*if (args->regs && user_mode_vm (args->regs))
+ return ret;*/
+
+ DBPRINTF ("switch (val) %lu %d %d", val, DIE_INT3, DIE_TRAP);
+ switch (val)
+ {
+//#ifdef CONFIG_KPROBES
+// case DIE_INT3:
+//#else
+ case DIE_TRAP:
+//#endif
+ DBPRINTF ("before kprobe_handler ret=%d %p", ret, args->regs);
+ if (kprobe_handler (args->regs))
+ ret = NOTIFY_STOP;
+ DBPRINTF ("after kprobe_handler ret=%d %p", ret, args->regs);
+ break;
+ case DIE_DEBUG:
+ if (post_kprobe_handler (args->regs))
+ ret = NOTIFY_STOP;
+ break;
+ case DIE_GPF:
+ // kprobe_running() needs smp_processor_id()
+ preempt_disable ();
+ if (kprobe_running () && kprobe_fault_handler (args->regs, args->trapnr))
+ ret = NOTIFY_STOP;
+ preempt_enable ();
+ break;
+ default:
+ break;
+ }
+ DBPRINTF ("ret=%d", ret);
+ if(ret == NOTIFY_STOP)
+ handled_exceptions++;
+
+ return ret;
+}
+#endif // CONFIG_X86
+
+int
+longjmp_break_handler (struct kprobe *p, struct pt_regs *regs)
+{
+#if defined(CONFIG_X86)
+ struct kprobe_ctlblk *kcb = get_kprobe_ctlblk ();
+ u8 *addr = (u8 *) (regs->EREG (ip) - 1);
+ unsigned long stack_addr = (unsigned long) (kcb->jprobe_saved_esp);
+ struct jprobe *jp = container_of (p, struct jprobe, kp);
+
+ DBPRINTF ("p = %p\n", p);
+
+ if ((addr > (u8 *) jprobe_return) && (addr < (u8 *) jprobe_return_end))
+ {
+ if ((unsigned long *)(®s->EREG(sp)) != kcb->jprobe_saved_esp)
+ {
+ struct pt_regs *saved_regs = &kcb->jprobe_saved_regs;
+ printk ("current esp %p does not match saved esp %p\n", ®s->EREG (sp), kcb->jprobe_saved_esp);
+ printk ("Saved registers for jprobe %p\n", jp);
+ show_registers (saved_regs);
+ printk ("Current registers\n");
+ show_registers (regs);
+ panic("BUG");
+ //BUG ();
+ }
+ *regs = kcb->jprobe_saved_regs;
+ memcpy ((kprobe_opcode_t *) stack_addr, kcb->jprobes_stack, MIN_STACK_SIZE (stack_addr));
+ preempt_enable_no_resched ();
+ return 1;
+ }
+#else //non x86
+ DBPRINTF ("p = %p\n", p);
+ //DBPRINTF("p->opcode = 0x%lx *p->addr = 0x%lx p->addr = 0x%p\n", p->opcode, p->pid?*kaddr[0]:*p->addr, p->pid?kaddr[0]:p->addr);
+# ifndef REENTER
+ //kprobe_opcode_t insn = BREAKPOINT_INSTRUCTION;
+ kprobe_opcode_t insns[2];
+
+ if (p->pid)
+ {
+ insns[0] = BREAKPOINT_INSTRUCTION;
+ insns[1] = p->opcode;
+ //p->opcode = *p->addr;
+ if (read_proc_vm_atomic (current, (unsigned long) (p->addr), &(p->opcode), sizeof (p->opcode)) < sizeof (p->opcode))
+ {
+ printk ("ERROR[%lu]: failed to read vm of proc %s/%u addr %p.", nCount, current->comm, current->pid, p->addr);
+ return -1;
+ }
+ //*p->addr = BREAKPOINT_INSTRUCTION;
+ //*(p->addr+1) = p->opcode;
+ if (write_proc_vm_atomic (current, (unsigned long) (p->addr), insns, sizeof (insns)) < sizeof (insns))
+ {
+ printk ("ERROR[%lu]: failed to write vm of proc %s/%u addr %p.", nCount, current->comm, current->pid, p->addr);
+ return -1;
+ }
+ }
+ else
+ {
+ DBPRINTF ("p->opcode = 0x%lx *p->addr = 0x%lx p->addr = 0x%p\n", p->opcode, *p->addr, p->addr);
+ *(p->addr + 1) = p->opcode;
+ p->opcode = *p->addr;
+ *p->addr = BREAKPOINT_INSTRUCTION;
+ flush_icache_range ((unsigned int) p->addr, (unsigned int) (((unsigned int) p->addr) + (sizeof (kprobe_opcode_t) * 2)));
+ }
+
+ reset_current_kprobe ();
+# endif //!reenter
+#endif //non x86
+
+ return 0;
+}
+
+void __kprobes
+arch_arm_kprobe (struct kprobe *p)
+{
+#if defined(CONFIG_X86)
+ text_poke (p->addr, ((unsigned char[])
+ {BREAKPOINT_INSTRUCTION}), 1);
+#else
+ *p->addr = BREAKPOINT_INSTRUCTION;
+ flush_icache_range ((unsigned long) p->addr, (unsigned long) p->addr + sizeof (kprobe_opcode_t));
+#endif
+}
+
+void __kprobes
+arch_disarm_kprobe (struct kprobe *p)
+{
+#if defined(CONFIG_X86)
+ text_poke (p->addr, &p->opcode, 1);
+#else
+ *p->addr = p->opcode;
+ flush_icache_range ((unsigned long) p->addr, (unsigned long) p->addr + sizeof (kprobe_opcode_t));
+#endif
+}
+
+void __kprobes
+arch_arm_uprobe (struct kprobe *p, struct task_struct *tsk)
+{
+ kprobe_opcode_t insn = BREAKPOINT_INSTRUCTION;
+
+ if (!write_proc_vm_atomic (tsk, (unsigned long) p->addr, &insn, sizeof (insn)))
+ panic ("failed to write memory %p!\n", p->addr);
+}
+
+void __kprobes
+arch_arm_uretprobe (struct kretprobe *p, struct task_struct *tsk)
+{
+}
+
+void __kprobes
+arch_disarm_uprobe (struct kprobe *p, struct task_struct *tsk)
+{
+ if (!write_proc_vm_atomic (tsk, (unsigned long) p->addr, &p->opcode, sizeof (p->opcode)))
+ panic ("failed to write memory %p!\n", p->addr);
+}
+
+void __kprobes
+arch_disarm_uretprobe (struct kretprobe *p, struct task_struct *tsk)//, struct vm_area_struct *vma, struct page *page, unsigned long *kaddr)
+{
+}
+
+#if defined(CONFIG_X86)
+/*fastcall*/ void *__kprobes trampoline_probe_handler_x86 (struct pt_regs *regs)
+{
+ return (void *)trampoline_probe_handler(NULL, regs);
+}
+#endif
+
+/*
+ * Function return probe trampoline:
+ * - init_kprobes() establishes a probepoint here
+ * - When the probed function returns, this probe
+ * causes the handlers to fire
+ */
+void
+kretprobe_trampoline_holder (void)
+{
+ asm volatile (".global kretprobe_trampoline\n"
+ "kretprobe_trampoline:\n"
+#if defined(CONFIG_MIPS)
+ "nop\n"
+ "nop\n");
+#elif defined(CONFIG_ARM)
+ "nop\n"
+ "nop\n"
+ "mov pc, r14\n");
+#elif defined(CONFIG_X86)
+ " pushf\n"
+ /* skip cs, eip, orig_eax */
+ " subl $12, %esp\n"
+ " pushl %fs\n"
+ " pushl %ds\n"
+ " pushl %es\n"
+ " pushl %eax\n"
+ " pushl %ebp\n"
+ " pushl %edi\n"
+ " pushl %esi\n"
+ " pushl %edx\n"
+ " pushl %ecx\n"
+ " pushl %ebx\n"
+ " movl %esp, %eax\n"
+ " call trampoline_probe_handler_x86\n"
+ /* move eflags to cs */
+ " movl 52(%esp), %edx\n"
+ " movl %edx, 48(%esp)\n"
+ /* save true return address on eflags */
+ " movl %eax, 52(%esp)\n"
+ " popl %ebx\n" ""
+ " popl %ecx\n"
+ " popl %edx\n"
+ " popl %esi\n"
+ " popl %edi\n"
+ " popl %ebp\n"
+ " popl %eax\n"
+ /* skip eip, orig_eax, es, ds, fs */
+ " addl $20, %esp\n"
+ " popf\n"
+ " ret\n");
+#else
+# error kretprobe_trampoline_holder is not implemented for this arch!!!
+#endif // ARCH
+}
+
+/*
+ * Called when the probe at kretprobe trampoline is hit
+ */
+int __kprobes trampoline_probe_handler (struct kprobe *p, struct pt_regs *regs)
+{
+ struct kretprobe_instance *ri = NULL;
+ struct hlist_head *head, empty_rp;
+ struct hlist_node *node, *tmp;
+ unsigned long flags, orig_ret_address = 0;
+ unsigned long trampoline_address = (unsigned long) &kretprobe_trampoline;
+ struct kretprobe *crp = NULL;
+ struct kprobe_ctlblk *kcb = get_kprobe_ctlblk ();
+
+ DBPRINTF ("start");
+
+ if (p && p->tgid){
+ // in case of user space retprobe trampoline is at the Nth instruction of US tramp
+ trampoline_address = (unsigned long)(p->ainsn.insn + UPROBES_TRAMP_RET_BREAK_IDX);
+ }
+
+ INIT_HLIST_HEAD (&empty_rp);
+ spin_lock_irqsave (&kretprobe_lock, flags);
+ head = kretprobe_inst_table_head (current);
+#if defined(CONFIG_X86)
+ if(!p){ // X86 kernel space
+ DBPRINTF ("regs %p", regs);
+ /* fixup registers */
+ regs->XREG (cs) = __KERNEL_CS | get_kernel_rpl ();
+ regs->EREG (ip) = trampoline_address;
+ regs->ORIG_EAX_REG = 0xffffffff;
+ }
+#endif
+ /*
+ * It is possible to have multiple instances associated with a given
+ * task either because an multiple functions in the call path
+ * have a return probe installed on them, and/or more then one
+ * return probe was registered for a target function.
+ *
+ * We can handle this because:
+ * - instances are always inserted at the head of the list
+ * - when multiple return probes are registered for the same
+ * function, the first instance's ret_addr will point to the
+ * real return address, and all the rest will point to
+ * kretprobe_trampoline
+ */
+ hlist_for_each_entry_safe (ri, node, tmp, head, hlist)
+ {
+ if (ri->task != current)
+ /* another task is sharing our hash bucket */
+ continue;
+ if (ri->rp && ri->rp->handler){
+#if defined(CONFIG_X86)
+ if(!p){ // X86 kernel space
+ __get_cpu_var (current_kprobe) = &ri->rp->kp;
+ get_kprobe_ctlblk ()->kprobe_status = KPROBE_HIT_ACTIVE;
+ }
+#endif
+ ri->rp->handler (ri, regs, ri->rp->priv_arg);
+#if defined(CONFIG_X86)
+ if(!p) // X86 kernel space
+ __get_cpu_var (current_kprobe) = NULL;
+#endif
+ }
+
+ orig_ret_address = (unsigned long) ri->ret_addr;
+ recycle_rp_inst (ri, &empty_rp);
+ if (orig_ret_address != trampoline_address)
+ /*
+ * This is the real return address. Any other
+ * instances associated with this task are for
+ * other calls deeper on the call stack
+ */
+ break;
+ }
+ kretprobe_assert (ri, orig_ret_address, trampoline_address);
+ //BUG_ON(!orig_ret_address || (orig_ret_address == trampoline_address));
+ if (trampoline_address != (unsigned long) &kretprobe_trampoline){
+ if (ri->rp2) BUG_ON (ri->rp2->kp.tgid == 0);
+ if (ri->rp) BUG_ON (ri->rp->kp.tgid == 0);
+ else if (ri->rp2) BUG_ON (ri->rp2->kp.tgid == 0);
+ }
+ if ((ri->rp && ri->rp->kp.tgid) || (ri->rp2 && ri->rp2->kp.tgid))
+ BUG_ON (trampoline_address == (unsigned long) &kretprobe_trampoline);
+#if defined(CONFIG_MIPS)
+ regs->regs[31] = orig_ret_address;
+ DBPRINTF ("regs->cp0_epc = 0x%lx", regs->cp0_epc);
+ if (trampoline_address != (unsigned long) &kretprobe_trampoline)
+ regs->cp0_epc = orig_ret_address;
+ else
+ regs->cp0_epc = regs->cp0_epc + 4;
+ DBPRINTF ("regs->cp0_epc = 0x%lx", regs->cp0_epc);
+ DBPRINTF ("regs->cp0_status = 0x%lx", regs->cp0_status);
+#elif defined(CONFIG_ARM)
+ regs->uregs[14] = orig_ret_address;
+ DBPRINTF ("regs->uregs[14] = 0x%lx\n", regs->uregs[14]);
+ DBPRINTF ("regs->uregs[15] = 0x%lx\n", regs->uregs[15]);
+ if (trampoline_address != (unsigned long) &kretprobe_trampoline)
+ regs->uregs[15] = orig_ret_address;
+ else
+ regs->uregs[15] += 4;
+ DBPRINTF ("regs->uregs[15] = 0x%lx\n", regs->uregs[15]);
+#elif defined(CONFIG_X86)
+ if(p){ // X86 user space
+ regs->EREG(ip) = orig_ret_address;
+ //printk (" uretprobe regs->eip = 0x%lx\n", regs->EREG(ip));
+ }
+#endif // ARCH
+
+ if(p){ // ARM, MIPS, X86 user space
+ if (kcb->kprobe_status == KPROBE_REENTER)
+ restore_previous_kprobe (kcb);
+ else
+ reset_current_kprobe ();
+
+ //TODO: test - enter function, delete us retprobe, exit function
+ // for user space retprobes only - deferred deletion
+ if (trampoline_address != (unsigned long) &kretprobe_trampoline)
+ {
+ // if we are not at the end of the list and current retprobe should be disarmed
+ if (node && ri->rp2)
+ {
+ crp = ri->rp2;
+ /*sprintf(die_msg, "deferred disarm p->addr = %p [%lx %lx %lx]\n",
+ crp->kp.addr, *kaddrs[0], *kaddrs[1], *kaddrs[2]);
+ DIE(die_msg, regs); */
+ // look for other instances for the same retprobe
+ hlist_for_each_entry_continue (ri, node, hlist)
+ {
+ if (ri->task != current)
+ continue; /* another task is sharing our hash bucket */
+ if (ri->rp2 == crp) //if instance belong to the same retprobe
+ break;
+ }
+ if (!node)
+ { // if there are no more instances for this retprobe
+ // delete retprobe
+ DBPRINTF ("defered retprobe deletion p->addr = %p", crp->kp.addr);
+ unregister_uprobe (&crp->kp, current, 1);
+ kfree (crp);
+ }
+ }
+ }
+ }
+
+ spin_unlock_irqrestore (&kretprobe_lock, flags);
+ hlist_for_each_entry_safe (ri, node, tmp, &empty_rp, hlist)
+ {
+ hlist_del (&ri->hlist);
+ kfree (ri);
+ }
+#if defined(CONFIG_X86)
+ if(!p) // X86 kernel space
+ return (int)orig_ret_address;
+#endif
+ preempt_enable_no_resched ();
+ /*
+ * By returning a non-zero value, we are telling
+ * kprobe_handler() that we don't want the post_handler
+ * to run (and have re-enabled preemption)
+ */
+ return 1;
+}
+
+/* Called with kretprobe_lock held */
+void __kprobes __arch_prepare_kretprobe (struct kretprobe *rp, struct pt_regs *regs)
+{
+ struct kretprobe_instance *ri;
+
+ DBPRINTF ("start\n");
+ //TODO: test - remove retprobe after func entry but before its exit
+ if ((ri = get_free_rp_inst (rp)) != NULL)
+ {
+ ri->rp = rp;
+ ri->rp2 = NULL;
+ ri->task = current;
+#if defined(CONFIG_MIPS)
+ ri->ret_addr = (kprobe_opcode_t *) regs->regs[31];
+ if (rp->kp.tgid)
+ regs->regs[31] = (unsigned long) (rp->kp.ainsn.insn + UPROBES_TRAMP_RET_BREAK_IDX);
+ else /* Replace the return addr with trampoline addr */
+ regs->regs[31] = (unsigned long) &kretprobe_trampoline;
+#elif defined(CONFIG_ARM)
+ ri->ret_addr = (kprobe_opcode_t *) regs->uregs[14];
+ if (rp->kp.tgid)
+ regs->uregs[14] = (unsigned long) (rp->kp.ainsn.insn + UPROBES_TRAMP_RET_BREAK_IDX);
+ else /* Replace the return addr with trampoline addr */
+ regs->uregs[14] = (unsigned long) &kretprobe_trampoline;
+ DBPRINTF ("ret addr set to %p->%lx\n", ri->ret_addr, regs->uregs[14]);
+#elif defined(CONFIG_X86)
+ /* Replace the return addr with trampoline addr */
+ if (rp->kp.tgid){
+ unsigned long ra = (unsigned long) (rp->kp.ainsn.insn + UPROBES_TRAMP_RET_BREAK_IDX);/*, stack[6];
+ if (!read_proc_vm_atomic (current, regs->EREG(sp), stack, sizeof(stack)))
+ panic ("failed to read user space func stack %lx!\n", regs->EREG(sp));
+ printk("stack: %lx %lx %lx %lx %lx %lx\n", stack[0], stack[1], stack[2], stack[3], stack[4], stack[5]);*/
+ if (!read_proc_vm_atomic (current, regs->EREG(sp), &(ri->ret_addr), sizeof(ri->ret_addr)))
+ panic ("failed to read user space func ra %lx!\n", regs->EREG(sp));
+ if (!write_proc_vm_atomic (current, regs->EREG(sp), &ra, sizeof(ra)))
+ panic ("failed to write user space func ra %lx!\n", regs->EREG(sp));
+ //printk("__arch_prepare_kretprobe: ra %lx %p->%lx\n",regs->EREG(sp), ri->ret_addr, ra);
+ }
+ else {
+ unsigned long *sara = (unsigned long *)®s->EREG(sp);
+ ri->ret_addr = (kprobe_opcode_t *)*sara;
+ *sara = (unsigned long)&kretprobe_trampoline;
+ DBPRINTF ("ra loc %p, origr_ra %p new ra %lx\n", sara, ri->ret_addr, *sara);
+ }
+#else
+#error __arch_prepare_kretprobe is not implemented for this arch!!!
+#endif // ARCH
+ add_rp_inst (ri);
+ }
+ else {
+ DBPRINTF ("WARNING: missed retprobe %p\n", rp->kp.addr);
+ rp->nmissed++;
+ }
+}
+
+#if !defined(CONFIG_X86)
+static struct kprobe trampoline_p =
+{
+ .addr = (kprobe_opcode_t *) & kretprobe_trampoline,
+ .pre_handler = trampoline_probe_handler
+};
+#endif
+
+/*static void do_exit_probe_handler (void)
+{
+ printk("do_exit_probe_handler\n");
+ unregister_all_uprobes(current, 1);
+ jprobe_return();
+}
+
+static struct jprobe do_exit_p =
+{
+ .entry = (kprobe_pre_entry_handler_t)do_exit_probe_handler
+};*/
+
+//--------------------- Declaration of module dependencies ------------------------//
+#define DECLARE_MOD_FUNC_DEP(name, ret, ...) ret(*__ref_##name)(__VA_ARGS__)
+#define DECLARE_MOD_CB_DEP(name, ret, ...) ret(*name)(__VA_ARGS__)
+// persistent deps
+DECLARE_MOD_CB_DEP(kallsyms_search, unsigned long, const char *name);
+DECLARE_MOD_FUNC_DEP(access_process_vm, int, struct task_struct * tsk, unsigned long addr, void *buf, int len, int write);
+// deps controled by config macros
+#ifdef KERNEL_HAS_ISPAGEPRESENT
+DECLARE_MOD_FUNC_DEP(is_page_present, int, struct mm_struct * mm, unsigned long address);
+#endif
+#if defined(CONFIG_PREEMPT) && defined(CONFIG_PM)
+DECLARE_MOD_FUNC_DEP(freeze_processes, int, void);
+DECLARE_MOD_FUNC_DEP(thaw_processes, void, void);
+#endif
+// deps controled by arch type
+#if defined(CONFIG_MIPS)
+DECLARE_MOD_CB_DEP(flush_icache_range, void, unsigned long __user start, unsigned long __user end);
+DECLARE_MOD_CB_DEP(flush_icache_page, void, struct vm_area_struct * vma, struct page * page);
+DECLARE_MOD_CB_DEP(flush_cache_page, void, struct vm_area_struct * vma, unsigned long page);
+#elif defined(CONFIG_X86)
+DECLARE_MOD_FUNC_DEP(module_alloc, void *, unsigned long size);
+DECLARE_MOD_FUNC_DEP(module_free, void, struct module *mod, void *module_region);
+DECLARE_MOD_FUNC_DEP(fixup_exception, int, struct pt_regs * regs);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26))
+DECLARE_MOD_FUNC_DEP(text_poke, void, void *addr, unsigned char *opcode, int len);
+#else
+DECLARE_MOD_FUNC_DEP(text_poke, void *, void *addr, const void *opcode, size_t len);
+#endif
+DECLARE_MOD_FUNC_DEP(show_registers, void, struct pt_regs * regs);
+#elif defined(CONFIG_ARM)
+#if defined(CONFIG_CPU_CACHE_VIPT) && (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18))
+//DECLARE_MOD_FUNC_DEP(flush_ptrace_access, void, struct vm_area_struct * vma, struct page * page, unsigned long uaddr, void *kaddr, unsigned long len, int write);
+#endif
+#endif
+// deps controled by kernel version
+#if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
+DECLARE_MOD_FUNC_DEP(put_task_struct, void, struct task_struct *tsk);
+#else //2.6.16
+DECLARE_MOD_FUNC_DEP(put_task_struct, void, struct rcu_head * rhp);
+#endif
+
+//----------------- Implementation of module dependencies wrappers -----------------//
+#define DECLARE_MOD_DEP_WRAPPER(name, ret, ...) ret name(__VA_ARGS__)
+#define IMP_MOD_DEP_WRAPPER(name, ...) \
+{ \
+ return __ref_##name(__VA_ARGS__); \
+}
+/*#define IMP_MOD_DEP_WRAPPER_NORET(name, ...) \
+{ \
+ return __ref_##name(__VA_ARGS__); \
+}*/
+// persistent deps
+DECLARE_MOD_DEP_WRAPPER(access_process_vm, int, struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
+IMP_MOD_DEP_WRAPPER(access_process_vm, tsk, addr, buf, len, write)
+// deps controled by config macros
+#ifdef KERNEL_HAS_ISPAGEPRESENT
+int is_page_present (struct mm_struct *mm, unsigned long address)
+{
+ int ret;
+
+ spin_lock (&(mm->page_table_lock));
+ ret = __ref_is_page_present (mm, address);
+ spin_unlock (&(mm->page_table_lock));
+ return ret;
+}
+#endif
+
+#if defined(CONFIG_PREEMPT) && defined(CONFIG_PM)
+DECLARE_MOD_DEP_WRAPPER(freeze_processes, int, void)
+IMP_MOD_DEP_WRAPPER(freeze_processes)
+DECLARE_MOD_DEP_WRAPPER(thaw_processes, void, void)
+IMP_MOD_DEP_WRAPPER(thaw_processes)
+#endif
+
+// deps controled by arch type
+#if defined(CONFIG_MIPS)
+#elif defined(CONFIG_X86)
+DECLARE_MOD_DEP_WRAPPER(module_alloc, void *, unsigned long size)
+IMP_MOD_DEP_WRAPPER(module_alloc, size)
+DECLARE_MOD_DEP_WRAPPER(module_free, void, struct module *mod, void *module_region)
+IMP_MOD_DEP_WRAPPER(module_free, mod, module_region)
+DECLARE_MOD_DEP_WRAPPER(fixup_exception, int, struct pt_regs * regs)
+IMP_MOD_DEP_WRAPPER(fixup_exception, regs)
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26))
+DECLARE_MOD_DEP_WRAPPER(text_poke, void, void *addr, unsigned char *opcode, int len)
+#else
+DECLARE_MOD_DEP_WRAPPER(text_poke, void *, void *addr, const void *opcode, size_t len)
+#endif
+IMP_MOD_DEP_WRAPPER(text_poke, addr, opcode, len)
+DECLARE_MOD_DEP_WRAPPER(show_registers, void, struct pt_regs * regs)
+IMP_MOD_DEP_WRAPPER(show_registers, regs)
+#elif defined(CONFIG_ARM)
+#endif
+// deps controled by kernel version
+#if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
+//DECLARE_MOD_FUNC_DEP(put_task_struct, void, struct task_struct *tsk);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11))
+DECLARE_MOD_DEP_WRAPPER(put_task_struct, void, struct task_struct *tsk)
+IMP_MOD_DEP_WRAPPER(put_task_struct, tsk)
+#else // >= 2.6.11 and != 2.6.16.x
+DECLARE_MOD_DEP_WRAPPER(__put_task_struct, void, struct task_struct *tsk)
+IMP_MOD_DEP_WRAPPER(put_task_struct, tsk)
+#endif
+#else //2.6.16
+DECLARE_MOD_DEP_WRAPPER(__put_task_struct_cb, void, struct rcu_head *rhp)
+IMP_MOD_DEP_WRAPPER(put_task_struct, rhp)
+#endif
+
+//---------------------- Module dependencies initialization --------------------//
+#define INIT_MOD_DEP_VAR(dep, name) \
+{ \
+ __ref_##dep = (void *) kallsyms_search (#name); \
+ if (!__ref_##dep) \
+ { \
+ DBPRINTF (#name " is not found! Oops. Where is it?"); \
+ return -ESRCH; \
+ } \
+}
+
+#define INIT_MOD_DEP_CB(dep, name) \
+{ \
+ dep = (void *) kallsyms_search (#name); \
+ if (!dep) \
+ { \
+ DBPRINTF (#name " is not found! Oops. Where is it?"); \
+ return -ESRCH; \
+ } \
+}
+
+int __init arch_init_kprobes (void)
+{
+#if !defined(CONFIG_X86)
+ unsigned int xDoBp; unsigned int xKProbeHandler;
+#endif
+#if defined(CONFIG_MIPS)
+ unsigned int xRegHi; unsigned int xRegLo;
+#endif // ARCH
+ int ret = 0;
+
+ // Prepare to lookup names
+ kallsyms_search = (void *) ksyms;
+ DBPRINTF ("kallsyms=0x%08x\n", ksyms);
+
+ sched_addr = (kprobe_opcode_t *)kallsyms_search("__switch_to");//"schedule");
+ fork_addr = (kprobe_opcode_t *)kallsyms_search("do_fork");
+
+ INIT_MOD_DEP_VAR(access_process_vm, access_process_vm);
+#ifdef KERNEL_HAS_ISPAGEPRESENT
+ INIT_MOD_DEP_VAR(is_page_present, is_page_present);
+#endif
+#if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
+# if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11))
+ INIT_MOD_DEP_VAR(put_task_struct, put_task_struct);
+# else
+ INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct);
+# endif
+#else /*2.6.16 */
+ INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct_cb);
+#endif
+#if defined(CONFIG_MIPS)
+ INIT_MOD_DEP_CB(flush_icache_range, r4k_flush_icache_range);
+ INIT_MOD_DEP_CB(flush_icache_page, r4k_flush_icache_page);
+ INIT_MOD_DEP_CB(flush_cache_page, r4k_flush_cache_page);
+#elif defined(CONFIG_X86)
+ INIT_MOD_DEP_VAR(module_alloc, module_alloc);
+ INIT_MOD_DEP_VAR(module_free, module_free);
+ INIT_MOD_DEP_VAR(fixup_exception, fixup_exception);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23)
+# error this kernel version has no text_poke function which is necessaryf for x86 ach!!!
+#else
+ INIT_MOD_DEP_VAR(text_poke, text_poke);
+#endif
+ INIT_MOD_DEP_VAR(show_registers, show_registers);
+#if defined(CONFIG_PREEMPT) && defined(CONFIG_PM)
+ INIT_MOD_DEP_VAR(freeze_processes, freeze_processes);
+ INIT_MOD_DEP_VAR(thaw_processes, thaw_processes);
+#endif
+ return ret;
+#endif // CONFIG_X86
+
+#if !defined(CONFIG_X86)
+ // Get instruction addresses
+# if defined(CONFIG_MIPS)
+ xDoBp = (unsigned int) kallsyms_search ("do_bp");
+# elif defined(CONFIG_ARM)
+ xDoBp = (unsigned int) kallsyms_search ("do_undefinstr");
+# endif // ARCH
+ xKProbeHandler = (unsigned int) &kprobe_handler;
+ gl_nNumberOfInstructions = sizeof (arrTrapsTemplate) / sizeof (arrTrapsTemplate[0]);
+ gl_nCodeSize = gl_nNumberOfInstructions * sizeof (unsigned int);
+ DBPRINTF ("nNumberOfInstructions = %d\n", gl_nNumberOfInstructions);
+ // Save original code
+ arrTrapsOriginal = kmalloc (gl_nCodeSize /* + sizeof(unsigned int) */ , GFP_KERNEL);
+ if (!arrTrapsOriginal)
+ {
+ DBPRINTF ("Unable to allocate space for original code of <do_bp>!\n");
+ return -1;
+ }
+ memcpy (arrTrapsOriginal, (void *) xDoBp, gl_nCodeSize);
+ // Fill in template
+#if defined(CONFIG_MIPS)
+ xRegHi = HIWORD (xKProbeHandler);
+ xRegLo = LOWORD (xKProbeHandler);
+ if (xRegLo >= 0x8000)
+ xRegHi += 0x0001;
+ arrTrapsTemplate[REG_HI_INDEX] |= xRegHi;
+ arrTrapsTemplate[REG_LO_INDEX] |= xRegLo;
+#elif defined(CONFIG_ARM)
+ arrTrapsTemplate[NOTIFIER_CALL_CHAIN_INDEX] = arch_construct_brunch (xKProbeHandler, xDoBp + NOTIFIER_CALL_CHAIN_INDEX * 4, 1);
+ //arrTrapsTemplate[NOTIFIER_CALL_CHAIN_INDEX1] = arch_construct_brunch(xKProbeHandler,
+ // xDoBp + NOTIFIER_CALL_CHAIN_INDEX1 * 4, 1);
+ //arrTrapsTemplate[NOTIFIER_CALL_CHAIN_INDEX2] = arch_construct_brunch((unsigned int)arrTrapsOriginal,
+ // xDoBp + NOTIFIER_CALL_CHAIN_INDEX2 * 4, 1);
+ //arrTrapsOriginal[gl_nNumberOfInstructions] = arch_construct_brunch(xDoBp + gl_nNumberOfInstructions * 4,
+ // (unsigned int)(arrTrapsOriginal + gl_nNumberOfInstructions), 1);
+#endif // ARCH
+ /*for(i = 0; i < gl_nNumberOfInstructions+1; i++)
+ {
+ printk("%08x\n", arrTrapsOriginal[i]);
+ } */
+ /*do_exit_p.kp.addr = (kprobe_opcode_t *)kallsyms_search ("do_exit");
+ if (!do_exit_p.kp.addr)
+ {
+ DBPRINTF ("do_exit is not found! Oops. Where is it?");
+ return -ESRCH;
+ }
+ if((ret = register_jprobe (&do_exit_p, 0)) != 0)
+ return ret;*/
+
+ // Insert new code
+ memcpy ((void *) xDoBp, arrTrapsTemplate, gl_nCodeSize);
+ flush_icache_range (xDoBp, xDoBp + gl_nCodeSize);
+ if((ret = register_kprobe (&trampoline_p, 0)) != 0){
+ //unregister_jprobe(&do_exit_p, 0);
+ return ret;
+ }
+
+ return ret;
+#endif
+}
+
+void __exit arch_exit_kprobes (void)
+{
+#if !defined(CONFIG_X86)
+ unsigned int xDoBp;
+ // Get instruction address
+#if defined(CONFIG_MIPS)
+ xDoBp = (unsigned int) kallsyms_search ("do_bp");
+#elif defined(CONFIG_ARM)
+ xDoBp = (unsigned int) kallsyms_search ("do_undefinstr");
+#endif // ARCH
+ //unregister_jprobe(&do_exit_p, 0);
+ // Replace back the original code
+ memcpy ((void *) xDoBp, arrTrapsOriginal, gl_nCodeSize);
+ flush_icache_range (xDoBp, xDoBp + gl_nCodeSize);
+ kfree (arrTrapsOriginal);
+ arrTrapsOriginal = NULL;
+#endif
+}
+
+MODULE_LICENSE ("Dual BSD/GPL");
projects / kernel / swap-modules.git / commitdiff