D7net
Home
Console
Upload
information
Create File
Create Folder
About
Tools
:
/
usr
/
share
/
systemtap
/
runtime
/
linux
/
uprobes
/
Filename :
uprobes.c
back
Copy
#include <linux/utrace.h> #ifndef UTRACE_ACTION_RESUME /* * Assume the kernel is running the 2008 version of utrace. * Skip the code in this file and instead use uprobes 2. */ #include "../uprobes2/uprobes.c" #else /* uprobes 1 (based on original utrace) */ /* * Userspace Probes (UProbes) * kernel/uprobes_core.c * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. * * Copyright (C) IBM Corporation, 2006 */ #include <linux/types.h> #include <linux/hash.h> #include <linux/init.h> #include <linux/module.h> #include <linux/sched.h> #include <linux/rcupdate.h> #include <linux/err.h> #include <linux/kref.h> #include <linux/utrace.h> #define UPROBES_IMPLEMENTATION 1 #include "uprobes.h" #include <linux/tracehook.h> #include <linux/mm.h> #include <linux/string.h> #include <linux/uaccess.h> #include <asm/tracehook.h> #include <asm/errno.h> #include <asm/mman.h> #define SET_ENGINE_FLAGS 1 #define CLEAR_ENGINE_FLAGS 0 #define MAX_SSOL_SLOTS 1024 #ifdef NO_ACCESS_PROCESS_VM_EXPORT static int __access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write); #define access_process_vm __access_process_vm #else extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write); #endif static int utask_fake_quiesce(struct uprobe_task *utask); static void uprobe_release_ssol_vma(struct uprobe_process *uproc); static void uretprobe_handle_entry(struct uprobe *u, struct pt_regs *regs, struct uprobe_task *utask); static void uretprobe_handle_return(struct pt_regs *regs, struct uprobe_task *utask); static void uretprobe_set_trampoline(struct uprobe_process *uproc, struct task_struct *tsk); static void zap_uretprobe_instances(struct uprobe *u, struct uprobe_process *uproc); typedef void (*uprobe_handler_t)(struct uprobe*, struct pt_regs*); #define URETPROBE_HANDLE_ENTRY ((uprobe_handler_t)-1L) #define is_uretprobe(u) (u->handler == URETPROBE_HANDLE_ENTRY) /* Point utask->active_probe at this while running uretprobe handler. */ static struct uprobe_probept uretprobe_trampoline_dummy_probe; /* Table of currently probed processes, hashed by tgid. */ static struct hlist_head uproc_table[UPROBE_TABLE_SIZE]; /* Protects uproc_table during uprobe (un)registration */ static DEFINE_MUTEX(uproc_mutex); /* Table of uprobe_tasks, hashed by task_struct pointer. */ static struct hlist_head utask_table[UPROBE_TABLE_SIZE]; static DEFINE_SPINLOCK(utask_table_lock); #define lock_uproc_table() mutex_lock(&uproc_mutex) #define unlock_uproc_table() mutex_unlock(&uproc_mutex) #define lock_utask_table(flags) spin_lock_irqsave(&utask_table_lock, (flags)) #define unlock_utask_table(flags) \ spin_unlock_irqrestore(&utask_table_lock, (flags)) /* p_uprobe_utrace_ops = &uprobe_utrace_ops. Fwd refs are a pain w/o this. */ static const struct utrace_engine_ops *p_uprobe_utrace_ops; struct deferred_registration { struct list_head list; struct uprobe *uprobe; int regflag; /* 0 - unregister, 1 - register */ enum uprobe_type type; }; /* * Calling a signal handler cancels single-stepping, so uprobes delays * calling the handler, as necessary, until after single-stepping is completed. */ struct delayed_signal { struct list_head list; siginfo_t info; }; static struct uprobe_task *uprobe_find_utask_locked(struct task_struct *tsk) { struct hlist_head *head; struct hlist_node *node; struct uprobe_task *utask; head = &utask_table[hash_ptr(tsk, UPROBE_HASH_BITS)]; hlist_for_each_entry(utask, node, head, hlist) { if (utask->tsk == tsk) return utask; } return NULL; } static struct uprobe_task *uprobe_find_utask(struct task_struct *tsk) { struct uprobe_task *utask; unsigned long flags; lock_utask_table(flags); utask = uprobe_find_utask_locked(tsk); unlock_utask_table(flags); return utask; } static void uprobe_hash_utask(struct uprobe_task *utask) { struct hlist_head *head; unsigned long flags; INIT_HLIST_NODE(&utask->hlist); lock_utask_table(flags); head = &utask_table[hash_ptr(utask->tsk, UPROBE_HASH_BITS)]; hlist_add_head(&utask->hlist, head); unlock_utask_table(flags); } static void uprobe_unhash_utask(struct uprobe_task *utask) { unsigned long flags; lock_utask_table(flags); hlist_del(&utask->hlist); unlock_utask_table(flags); } static struct uprobe_process * uprobe_get_process(struct uprobe_process *uproc) { if (atomic_inc_not_zero(&uproc->refcount)) return uproc; return NULL; } /* * Decrement uproc's refcount in a situation where we "know" it can't * reach zero. It's OK to call this with uproc locked. Compare with * uprobe_put_process(). */ static inline void uprobe_decref_process(struct uprobe_process *uproc) { if (atomic_dec_and_test(&uproc->refcount)) BUG(); } /* * Runs with the uproc_mutex held. Returns with uproc ref-counted and * write-locked. * * Around exec time, briefly, it's possible to have one (finished) uproc * for the old image and one for the new image. We find the latter. */ static struct uprobe_process *uprobe_find_process(pid_t tgid) { struct hlist_head *head; struct hlist_node *node; struct uprobe_process *uproc; head = &uproc_table[hash_long(tgid, UPROBE_HASH_BITS)]; hlist_for_each_entry(uproc, node, head, hlist) { if (uproc->tgid == tgid && !uproc->finished) { uproc = uprobe_get_process(uproc); if (uproc) down_write(&uproc->rwsem); return uproc; } } return NULL; } /* * In the given uproc's hash table of probepoints, find the one with the * specified virtual address. Runs with uproc->rwsem locked. */ static struct uprobe_probept *uprobe_find_probept(struct uprobe_process *uproc, unsigned long vaddr) { struct uprobe_probept *ppt; struct hlist_node *node; struct hlist_head *head = &uproc->uprobe_table[hash_long(vaddr, UPROBE_HASH_BITS)]; hlist_for_each_entry(ppt, node, head, ut_node) { if (ppt->vaddr == vaddr && ppt->state != UPROBE_DISABLED) return ppt; } return NULL; } /* * set_bp: Store a breakpoint instruction at ppt->vaddr. * Returns BP_INSN_SIZE on success. * * NOTE: BREAKPOINT_INSTRUCTION on all archs is the same size as * uprobe_opcode_t. */ static int set_bp(struct uprobe_probept *ppt, struct task_struct *tsk) { uprobe_opcode_t bp_insn = BREAKPOINT_INSTRUCTION; return access_process_vm(tsk, ppt->vaddr, &bp_insn, BP_INSN_SIZE, 1); } /* * set_orig_insn: For probepoint ppt, replace the breakpoint instruction * with the original opcode. Returns BP_INSN_SIZE on success. */ static int set_orig_insn(struct uprobe_probept *ppt, struct task_struct *tsk) { return access_process_vm(tsk, ppt->vaddr, &ppt->opcode, BP_INSN_SIZE, 1); } static void bkpt_insertion_failed(struct uprobe_probept *ppt, const char *why) { printk(KERN_ERR "Can't place uprobe at pid %d vaddr %#lx: %s\n", ppt->uproc->tgid, ppt->vaddr, why); } /* * Save a copy of the original instruction (so it can be single-stepped * out of line), insert the breakpoint instruction, and awake * register_uprobe(). */ static void insert_bkpt(struct uprobe_probept *ppt, struct task_struct *tsk) { struct uprobe_kimg *uk; long result = 0; int len; if (!tsk) { /* No surviving tasks associated with ppt->uproc */ result = -ESRCH; goto out; } /* * If access_process_vm() transfers fewer bytes than the maximum * instruction size, assume that the probed instruction is smaller * than the max and near the end of the last page of instructions. * But there must be room at least for a breakpoint-size instruction. */ len = access_process_vm(tsk, ppt->vaddr, ppt->insn, MAX_UINSN_BYTES, 0); if (len < BP_INSN_SIZE) { bkpt_insertion_failed(ppt, "error reading original instruction"); result = -EIO; goto out; } memcpy(&ppt->opcode, ppt->insn, BP_INSN_SIZE); if (ppt->opcode == BREAKPOINT_INSTRUCTION) { /* * To avoid filling up the log file with complaints * about breakpoints already existing, don't log this * error. */ //bkpt_insertion_failed(ppt, "bkpt already exists at that addr"); result = -EEXIST; goto out; } if ((result = arch_validate_probed_insn(ppt, tsk)) < 0) { bkpt_insertion_failed(ppt, "instruction type cannot be probed"); goto out; } len = set_bp(ppt, tsk); if (len < BP_INSN_SIZE) { bkpt_insertion_failed(ppt, "failed to insert bkpt instruction"); result = -EIO; goto out; } out: ppt->state = (result ? UPROBE_DISABLED : UPROBE_BP_SET); list_for_each_entry(uk, &ppt->uprobe_list, list) uk->status = result; wake_up_all(&ppt->waitq); } static void remove_bkpt(struct uprobe_probept *ppt, struct task_struct *tsk) { int len; if (tsk) { len = set_orig_insn(ppt, tsk); if (len < BP_INSN_SIZE) { printk(KERN_ERR "Error removing uprobe at pid %d vaddr %#lx:" " can't restore original instruction\n", tsk->tgid, ppt->vaddr); /* * This shouldn't happen, since we were previously * able to write the breakpoint at that address. * There's not much we can do besides let the * process die with a SIGTRAP the next time the * breakpoint is hit. */ } } /* Wake up unregister_uprobe(). */ ppt->state = UPROBE_DISABLED; wake_up_all(&ppt->waitq); } /* * Runs with all of uproc's threads quiesced and uproc->rwsem write-locked. * As specified, insert or remove the breakpoint instruction for each * uprobe_probept on uproc's pending list. * tsk = one of the tasks associated with uproc -- NULL if there are * no surviving threads. * It's OK for uproc->pending_uprobes to be empty here. It can happen * if a register and an unregister are requested (by different probers) * simultaneously for the same pid/vaddr. * Note that the current task may be a thread in uproc, or it may be * a task running [un]register_uprobe() (or both). */ static void handle_pending_uprobes(struct uprobe_process *uproc, struct task_struct *tsk) { struct uprobe_probept *ppt, *tmp; list_for_each_entry_safe(ppt, tmp, &uproc->pending_uprobes, pd_node) { switch (ppt->state) { case UPROBE_INSERTING: insert_bkpt(ppt, tsk); break; case UPROBE_REMOVING: remove_bkpt(ppt, tsk); break; default: BUG(); } list_del(&ppt->pd_node); } } static void utask_adjust_flags(struct uprobe_task *utask, int set, unsigned long flags) { unsigned long newflags, oldflags; newflags = oldflags = utask->engine->flags; if (set) newflags |= flags; else newflags &= ~flags; if (newflags != oldflags) utrace_set_flags(utask->tsk, utask->engine, newflags); } static inline void clear_utrace_quiesce(struct uprobe_task *utask) { utask_adjust_flags(utask, CLEAR_ENGINE_FLAGS, UTRACE_ACTION_QUIESCE | UTRACE_EVENT(QUIESCE)); } /* Opposite of quiesce_all_threads(). Same locking applies. */ static void rouse_all_threads(struct uprobe_process *uproc) { struct uprobe_task *utask; list_for_each_entry(utask, &uproc->thread_list, list) { if (utask->quiescing) { utask->quiescing = 0; if (utask->state == UPTASK_QUIESCENT) { utask->state = UPTASK_RUNNING; uproc->n_quiescent_threads--; clear_utrace_quiesce(utask); } } } /* Wake any threads that decided to sleep rather than quiesce. */ wake_up_all(&uproc->waitq); } /* * If all of uproc's surviving threads have quiesced, do the necessary * breakpoint insertions or removals and then un-quiesce everybody. * tsk is a surviving thread, or NULL if there is none. Runs with * uproc->rwsem write-locked. */ static void check_uproc_quiesced(struct uprobe_process *uproc, struct task_struct *tsk) { if (uproc->n_quiescent_threads >= uproc->nthreads) { handle_pending_uprobes(uproc, tsk); rouse_all_threads(uproc); } } /* * Quiesce all threads in the specified process -- e.g., prior to * breakpoint insertion. Runs with uproc->rwsem write-locked. * Returns the number of threads that haven't died yet. */ static int quiesce_all_threads(struct uprobe_process *uproc, struct uprobe_task **cur_utask_quiescing) { struct uprobe_task *utask; struct task_struct *survivor = NULL; // any survivor int survivors = 0; *cur_utask_quiescing = NULL; list_for_each_entry(utask, &uproc->thread_list, list) { survivor = utask->tsk; survivors++; if (!utask->quiescing) { /* * If utask is currently handling a probepoint, it'll * check utask->quiescing and quiesce when it's done. */ utask->quiescing = 1; if (utask->tsk == current) *cur_utask_quiescing = utask; else if (utask->state == UPTASK_RUNNING) { utask->quiesce_master = current; utask_adjust_flags(utask, SET_ENGINE_FLAGS, UTRACE_ACTION_QUIESCE | UTRACE_EVENT(QUIESCE)); utask->quiesce_master = NULL; } } } /* * If any task was already quiesced (in utrace's opinion) when we * called utask_adjust_flags() on it, uprobe_report_quiesce() was * called, but wasn't in a position to call check_uproc_quiesced(). */ check_uproc_quiesced(uproc, survivor); return survivors; } static void utask_free_uretprobe_instances(struct uprobe_task *utask) { struct uretprobe_instance *ri; struct hlist_node *r1, *r2; hlist_for_each_entry_safe(ri, r1, r2, &utask->uretprobe_instances, hlist) { hlist_del(&ri->hlist); kfree(ri); uprobe_decref_process(utask->uproc); } } /* Called with utask->uproc write-locked. */ static void uprobe_free_task(struct uprobe_task *utask) { struct deferred_registration *dr, *d; struct delayed_signal *ds, *ds2; /* printk(KERN_INFO "uprobe_free_task %p (tid %ld), caller %pS, ctid %ld\n", utask, utask->tsk->pid, _RET_IP_, current->pid); */ /* * Do this first, since a utask that's still in the utask_table * is assumed (e.g., by uprobe_report_exit) to be valid. */ uprobe_unhash_utask(utask); list_del(&utask->list); list_for_each_entry_safe(dr, d, &utask->deferred_registrations, list) { list_del(&dr->list); kfree(dr); } list_for_each_entry_safe(ds, ds2, &utask->delayed_signals, list) { list_del(&ds->list); kfree(ds); } utask_free_uretprobe_instances(utask); kfree(utask); } /* Runs with uproc_mutex held and uproc->rwsem write-locked. */ static void uprobe_free_process(struct uprobe_process *uproc) { struct uprobe_task *utask, *tmp; struct uprobe_ssol_area *area = &uproc->ssol_area; /* printk(KERN_INFO "uprobe_free_process %p (pid %ld), caller %pS, ctid %ld\n", uproc, uproc->tgid, _RET_IP_, current->pid); */ if (!uproc->finished) uprobe_release_ssol_vma(uproc); if (area->slots) kfree(area->slots); if (!hlist_unhashed(&uproc->hlist)) hlist_del(&uproc->hlist); list_for_each_entry_safe(utask, tmp, &uproc->thread_list, list) { /* * utrace_detach() is OK here (required, it seems) even if * utask->tsk == current and we're in a utrace callback. */ if (utask->engine) utrace_detach(utask->tsk, utask->engine); uprobe_free_task(utask); } up_write(&uproc->rwsem); // So kfree doesn't complain /* printk(KERN_INFO "uprobe_free_process zap %p\n", uproc);*/ kfree(uproc); } /* * Decrement uproc's ref count. If it's zero, free uproc and return 1. * Else return 0. If uproc is locked, don't call this; use * uprobe_decref_process(). * * If we free uproc, we also decrement the ref-count on the uprobes * module, if any. If somebody is doing "rmmod --wait uprobes", this * function could schedule removal of the module. Therefore, don't call * this function and then sleep in uprobes code, unless you know you'll * return with the module ref-count > 0. */ static int uprobe_put_process(struct uprobe_process *uproc) { int freed = 0; if (atomic_dec_and_test(&uproc->refcount)) { lock_uproc_table(); down_write(&uproc->rwsem); if (unlikely(atomic_read(&uproc->refcount) != 0)) { /* * The works because uproc_mutex is held any * time the ref count can go from 0 to 1 -- e.g., * register_uprobe() snuck in with a new probe, * or a callback such as uprobe_report_exit() * just started. */ up_write(&uproc->rwsem); } else { uprobe_free_process(uproc); freed = 1; } unlock_uproc_table(); } if (freed) module_put(THIS_MODULE); return freed; } static struct uprobe_kimg *uprobe_mk_kimg(struct uprobe *u) { struct uprobe_kimg *uk = (struct uprobe_kimg*)kzalloc(sizeof *uk, GFP_USER); if (unlikely(!uk)) return ERR_PTR(-ENOMEM); u->kdata = uk; uk->uprobe = u; uk->ppt = NULL; INIT_LIST_HEAD(&uk->list); uk->status = -EBUSY; return uk; } /* * Allocate a uprobe_task object for t and add it to uproc's list. * Called with t "got" and uproc->rwsem write-locked. Called in one of * the following cases: * - before setting the first uprobe in t's process * - we're in uprobe_report_clone() and t is the newly added thread * Returns: * - pointer to new uprobe_task on success * - NULL if t dies before we can utrace_attach it * - negative errno otherwise */ static struct uprobe_task *uprobe_add_task(struct task_struct *t, struct uprobe_process *uproc) { struct uprobe_task *utask; struct utrace_attached_engine *engine; utask = (struct uprobe_task *)kzalloc(sizeof *utask, GFP_USER); if (unlikely(utask == NULL)) return ERR_PTR(-ENOMEM); utask->tsk = t; utask->state = UPTASK_RUNNING; utask->quiescing = 0; utask->uproc = uproc; utask->active_probe = NULL; utask->doomed = 0; INIT_HLIST_HEAD(&utask->uretprobe_instances); INIT_LIST_HEAD(&utask->deferred_registrations); INIT_LIST_HEAD(&utask->delayed_signals); INIT_LIST_HEAD(&utask->list); list_add_tail(&utask->list, &uproc->thread_list); uprobe_hash_utask(utask); engine = utrace_attach(t, UTRACE_ATTACH_CREATE, p_uprobe_utrace_ops, utask); if (IS_ERR(engine)) { long err = PTR_ERR(engine); printk("uprobes: utrace_attach failed, returned %ld\n", err); uprobe_free_task(utask); if (err == -ESRCH) return NULL; return ERR_PTR(err); } utask->engine = engine; /* * Always watch for traps, clones, execs and exits. Caller must * set any other engine flags. */ utask_adjust_flags(utask, SET_ENGINE_FLAGS, UTRACE_EVENT(SIGNAL) | UTRACE_EVENT(SIGNAL_IGN) | UTRACE_EVENT(SIGNAL_CORE) | UTRACE_EVENT(EXEC) | UTRACE_EVENT(CLONE) | UTRACE_EVENT(EXIT)); /* * Note that it's OK if t dies just after utrace_attach, because * with the engine in place, the appropriate report_* callback * should handle it after we release uproc->rwsem. */ return utask; } /* See comment in uprobe_mk_process(). */ static struct task_struct *find_next_thread_to_add(struct uprobe_process *uproc, struct task_struct *start) { struct task_struct *t; struct uprobe_task *utask; read_lock(&tasklist_lock); t = start; do { if (unlikely(t->flags & PF_EXITING)) goto dont_add; list_for_each_entry(utask, &uproc->thread_list, list) { if (utask->tsk == t) /* Already added */ goto dont_add; } /* Found thread/task to add. */ get_task_struct(t); read_unlock(&tasklist_lock); return t; dont_add: t = next_thread(t); } while (t != start); read_unlock(&tasklist_lock); return NULL; } /* * Create a per process uproc struct. * at_fork: indicates uprobe_mk_process is called from * a fork context of a probe process. refer uprobe_fork_uproc * for more details. * * Runs with uproc_mutex held; * Returns with uproc->rwsem write-locked when not called * from fork context. */ static struct uprobe_process *uprobe_mk_process(struct task_struct *p, bool at_fork) { struct uprobe_process *uproc; struct uprobe_task *utask; struct task_struct *add_me; int i; long err; uproc = (struct uprobe_process *)kzalloc(sizeof *uproc, GFP_USER); if (unlikely(uproc == NULL)) return ERR_PTR(-ENOMEM); /* Initialize fields */ atomic_set(&uproc->refcount, 1); init_rwsem(&uproc->rwsem); if (!at_fork) /* not called from fork context. */ down_write(&uproc->rwsem); init_waitqueue_head(&uproc->waitq); for (i = 0; i < UPROBE_TABLE_SIZE; i++) INIT_HLIST_HEAD(&uproc->uprobe_table[i]); uproc->nppt = 0; INIT_LIST_HEAD(&uproc->pending_uprobes); INIT_LIST_HEAD(&uproc->thread_list); uproc->nthreads = 0; uproc->n_quiescent_threads = 0; INIT_HLIST_NODE(&uproc->hlist); uproc->tgid = p->tgid; uproc->finished = 0; uproc->uretprobe_trampoline_addr = NULL; uproc->ssol_area.insn_area = NULL; uproc->ssol_area.initialized = 0; mutex_init(&uproc->ssol_area.setup_mutex); /* Initialize rest of area in uprobe_init_ssol(). */ #ifdef CONFIG_UPROBES_SSOL uproc->sstep_out_of_line = 1; #else uproc->sstep_out_of_line = 0; #endif /* * Create and populate one utask per thread in this process. We * can't call uprobe_add_task() while holding tasklist_lock, so we: * 1. Lock task list. * 2. Find the next task, add_me, in this process that's not * already on uproc's thread_list. (Start search at previous * one found.) * 3. Unlock task list. * 4. uprobe_add_task(add_me, uproc) * Repeat 1-4 'til we have utasks for all tasks. */ add_me = p; while ((add_me = find_next_thread_to_add(uproc, add_me)) != NULL) { utask = uprobe_add_task(add_me, uproc); put_task_struct(add_me); if (IS_ERR(utask)) { err = PTR_ERR(utask); goto fail; } if (utask) uproc->nthreads++; } if (uproc->nthreads == 0) { /* All threads -- even p -- are dead. */ err = -ESRCH; goto fail; } return uproc; fail: uprobe_free_process(uproc); return ERR_PTR(err); } /* * Creates a uprobe_probept and connects it to uk and uproc. Runs with * uproc->rwsem write-locked. */ static struct uprobe_probept *uprobe_add_probept(struct uprobe_kimg *uk, struct uprobe_process *uproc) { struct uprobe_probept *ppt; ppt = (struct uprobe_probept *)kzalloc(sizeof *ppt, GFP_USER); if (unlikely(ppt == NULL)) return ERR_PTR(-ENOMEM); init_waitqueue_head(&ppt->waitq); mutex_init(&ppt->ssil_mutex); mutex_init(&ppt->slot_mutex); ppt->slot = NULL; /* Connect to uk. */ INIT_LIST_HEAD(&ppt->uprobe_list); list_add_tail(&uk->list, &ppt->uprobe_list); uk->ppt = ppt; uk->status = -EBUSY; ppt->vaddr = uk->uprobe->vaddr; /* Connect to uproc. */ ppt->state = UPROBE_INSERTING; ppt->uproc = uproc; INIT_LIST_HEAD(&ppt->pd_node); list_add_tail(&ppt->pd_node, &uproc->pending_uprobes); INIT_HLIST_NODE(&ppt->ut_node); hlist_add_head(&ppt->ut_node, &uproc->uprobe_table[hash_long(ppt->vaddr, UPROBE_HASH_BITS)]); uproc->nppt++; uprobe_get_process(uproc); return ppt; } /* ppt is going away. Free its slot (if it owns one) in the SSOL area. */ static void uprobe_free_slot(struct uprobe_probept *ppt) { struct uprobe_ssol_slot *slot = ppt->slot; if (slot) { down_write(&slot->rwsem); if (slot->owner == ppt) { unsigned long flags; struct uprobe_ssol_area *area = &ppt->uproc->ssol_area; spin_lock_irqsave(&area->lock, flags); slot->state = SSOL_FREE; slot->owner = NULL; area->nfree++; spin_unlock_irqrestore(&area->lock, flags); } up_write(&slot->rwsem); } } /* * Runs with ppt->uproc write-locked. Frees ppt and decrements the ref count * on ppt->uproc (but ref count shouldn't hit 0). */ static void uprobe_free_probept(struct uprobe_probept *ppt) { struct uprobe_process *uproc = ppt->uproc; uprobe_free_slot(ppt); hlist_del(&ppt->ut_node); uproc->nppt--; kfree(ppt); uprobe_decref_process(uproc); } static void uprobe_free_kimg(struct uprobe_kimg *uk) { uk->uprobe->kdata = NULL; kfree(uk); } /* * Runs with uprobe_process write-locked. * Note that we never free u, because the user owns that. */ static void purge_uprobe(struct uprobe_kimg *uk) { struct uprobe_probept *ppt = uk->ppt; list_del(&uk->list); uprobe_free_kimg(uk); if (list_empty(&ppt->uprobe_list)) uprobe_free_probept(ppt); } /* TODO: Avoid code duplication with uprobe_validate_vaddr(). */ static int uprobe_validate_vma(struct task_struct *t, unsigned long vaddr) { struct vm_area_struct *vma; struct mm_struct *mm; int ret = 0; mm = get_task_mm(t); if (!mm) return -EINVAL; down_read(&mm->mmap_sem); vma = find_vma(mm, vaddr); if (!vma || vaddr < vma->vm_start) ret = -ENOENT; else if (!(vma->vm_flags & VM_EXEC)) ret = -EFAULT; up_read(&mm->mmap_sem); mmput(mm); return ret; } /* Probed address must be in an executable VM area, outside the SSOL area. */ static int uprobe_validate_vaddr(struct task_struct *p, unsigned long vaddr, struct uprobe_process *uproc) { struct vm_area_struct *vma; struct mm_struct *mm = p->mm; if (!mm) return -EINVAL; down_read(&mm->mmap_sem); vma = find_vma(mm, vaddr); if (!vma || vaddr < vma->vm_start || !(vma->vm_flags & VM_EXEC) || vma->vm_start == (unsigned long) uproc->ssol_area.insn_area) { up_read(&mm->mmap_sem); return -EINVAL; } up_read(&mm->mmap_sem); return 0; } static struct task_struct *uprobe_get_task(pid_t pid) { struct task_struct *p; rcu_read_lock(); p = find_task_by_pid(pid); if (p) get_task_struct(p); rcu_read_unlock(); return p; } /* Runs with utask->uproc read-locked. Returns -EINPROGRESS on success. */ static int defer_registration(struct uprobe *u, int regflag, struct uprobe_task *utask) { struct deferred_registration *dr = kmalloc(sizeof(struct deferred_registration), GFP_USER); if (!dr) return -ENOMEM; dr->type = (is_uretprobe(u) ? UPTY_URETPROBE : UPTY_UPROBE); dr->uprobe = u; dr->regflag = regflag; INIT_LIST_HEAD(&dr->list); list_add_tail(&dr->list, &utask->deferred_registrations); return -EINPROGRESS; } /* See Documentation/uprobes.txt. */ int register_uprobe(struct uprobe *u) { struct task_struct *p; struct uprobe_process *uproc; struct uprobe_kimg *uk; struct uprobe_probept *ppt; struct uprobe_task *cur_utask, *cur_utask_quiescing = NULL; int survivors, ret = 0, uproc_is_new = 0; if (!u || !u->handler) return -EINVAL; p = uprobe_get_task(u->pid); if (!p) return -ESRCH; cur_utask = uprobe_find_utask(current); if (cur_utask && cur_utask->active_probe) { /* * Called from handler; cur_utask->uproc is read-locked. * Do this registration later. */ put_task_struct(p); return defer_registration(u, 1, cur_utask); } /* Get the uprobe_process for this pid, or make a new one. */ lock_uproc_table(); uproc = uprobe_find_process(p->tgid); if (uproc) unlock_uproc_table(); else { /* Creating a new uprobe_process. Ref-count the module. */ if (!try_module_get(THIS_MODULE)) { /* uprobes.ko is being removed. */ ret = -ENOSYS; unlock_uproc_table(); goto fail_tsk; } uproc = uprobe_mk_process(p, 0); if (IS_ERR(uproc)) { ret = (int) PTR_ERR(uproc); unlock_uproc_table(); module_put(THIS_MODULE); goto fail_tsk; } /* Hold uproc_mutex until we've added uproc to uproc_table. */ uproc_is_new = 1; } if (is_uretprobe(u) && IS_ERR(uproc->uretprobe_trampoline_addr)) { /* Previously failed to set up trampoline. */ ret = -ENOMEM; goto fail_uproc; } if ((ret = uprobe_validate_vaddr(p, u->vaddr, uproc)) < 0) goto fail_uproc; if (u->kdata) { /* * Probe is already/still registered. This is the only * place we return -EBUSY to the user. */ ret = -EBUSY; goto fail_uproc; } uk = uprobe_mk_kimg(u); if (IS_ERR(uk)) { ret = (int) PTR_ERR(uk); goto fail_uproc; } /* See if we already have a probepoint at the vaddr. */ ppt = (uproc_is_new ? NULL : uprobe_find_probept(uproc, u->vaddr)); if (ppt) { /* Breakpoint is already in place, or soon will be. */ uk->ppt = ppt; list_add_tail(&uk->list, &ppt->uprobe_list); switch (ppt->state) { case UPROBE_INSERTING: uk->status = -EBUSY; // in progress if (uproc->tgid == current->tgid) { cur_utask_quiescing = cur_utask; BUG_ON(!cur_utask_quiescing); } break; case UPROBE_REMOVING: /* Wait! Don't remove that bkpt after all! */ ppt->state = UPROBE_BP_SET; list_del(&ppt->pd_node); // Remove from pending list. wake_up_all(&ppt->waitq); // Wake unregister_uprobe(). /*FALLTHROUGH*/ case UPROBE_BP_SET: uk->status = 0; break; default: BUG(); } up_write(&uproc->rwsem); put_task_struct(p); if (uk->status == 0) { uprobe_put_process(uproc); return 0; } goto await_bkpt_insertion; } else { ppt = uprobe_add_probept(uk, uproc); if (IS_ERR(ppt)) { ret = (int) PTR_ERR(ppt); goto fail_uk; } } if (uproc_is_new) { hlist_add_head(&uproc->hlist, &uproc_table[hash_long(uproc->tgid, UPROBE_HASH_BITS)]); unlock_uproc_table(); } put_task_struct(p); survivors = quiesce_all_threads(uproc, &cur_utask_quiescing); if (survivors == 0) { purge_uprobe(uk); up_write(&uproc->rwsem); uprobe_put_process(uproc); return -ESRCH; } up_write(&uproc->rwsem); await_bkpt_insertion: if (cur_utask_quiescing) /* Current task is probing its own process. */ (void) utask_fake_quiesce(cur_utask_quiescing); else wait_event(ppt->waitq, ppt->state != UPROBE_INSERTING); ret = uk->status; if (ret != 0) { down_write(&uproc->rwsem); purge_uprobe(uk); up_write(&uproc->rwsem); } uprobe_put_process(uproc); return ret; fail_uk: uprobe_free_kimg(uk); fail_uproc: if (uproc_is_new) { uprobe_free_process(uproc); unlock_uproc_table(); module_put(THIS_MODULE); } else { up_write(&uproc->rwsem); uprobe_put_process(uproc); } fail_tsk: put_task_struct(p); return ret; } EXPORT_SYMBOL_GPL(register_uprobe); void __unregister_uprobe(struct uprobe *u, bool remove_bkpt) { struct task_struct *p; struct uprobe_process *uproc; struct uprobe_kimg *uk; struct uprobe_probept *ppt; struct uprobe_task *cur_utask, *cur_utask_quiescing = NULL; if (!u) return; p = uprobe_get_task(u->pid); if (!p) return; cur_utask = uprobe_find_utask(current); if (cur_utask && cur_utask->active_probe) { /* Called from handler; uproc is read-locked; do this later */ put_task_struct(p); (void) defer_registration(u, 0, cur_utask); return; } /* * Lock uproc before walking the graph, in case the process we're * probing is exiting. */ lock_uproc_table(); uproc = uprobe_find_process(p->tgid); unlock_uproc_table(); put_task_struct(p); if (!uproc) return; uk = (struct uprobe_kimg *)u->kdata; if (!uk) /* * This probe was never successfully registered, or * has already been unregistered. */ goto done; if (uk->status == -EBUSY) /* Looks like register or unregister is already in progress. */ goto done; ppt = uk->ppt; list_del(&uk->list); uprobe_free_kimg(uk); if (is_uretprobe(u)) zap_uretprobe_instances(u, uproc); if (!list_empty(&ppt->uprobe_list)) goto done; /* The last uprobe at ppt's probepoint is being unregistered. */ if (!remove_bkpt) { uprobe_free_probept(ppt); goto done; } /* Queue the breakpoint for removal. */ ppt->state = UPROBE_REMOVING; list_add_tail(&ppt->pd_node, &uproc->pending_uprobes); (void) quiesce_all_threads(uproc, &cur_utask_quiescing); up_write(&uproc->rwsem); if (cur_utask_quiescing) /* Current task is probing its own process. */ (void) utask_fake_quiesce(cur_utask_quiescing); else wait_event(ppt->waitq, ppt->state != UPROBE_REMOVING); if (likely(ppt->state == UPROBE_DISABLED)) { down_write(&uproc->rwsem); uprobe_free_probept(ppt); /* else somebody else's register_uprobe() resurrected ppt. */ up_write(&uproc->rwsem); } uprobe_put_process(uproc); return; done: up_write(&uproc->rwsem); uprobe_put_process(uproc); } /* See Documentation/uprobes.txt. */ void unregister_uprobe(struct uprobe *u) { __unregister_uprobe(u, true); } EXPORT_SYMBOL_GPL(unregister_uprobe); void unmap_uprobe(struct uprobe *u) { __unregister_uprobe(u, false); } EXPORT_SYMBOL_GPL(unmap_uprobe); /* Find a surviving thread in uproc. Runs with uproc->rwsem locked. */ static struct task_struct *find_surviving_thread(struct uprobe_process *uproc) { struct uprobe_task *utask; list_for_each_entry(utask, &uproc->thread_list, list) return utask->tsk; return NULL; } /* * Run all the deferred_registrations previously queued by the current utask. * Runs with no locks or mutexes held. The current utask's uprobe_process * is ref-counted, so they won't disappear as the result of * unregister_u*probe() called here. */ static void uprobe_run_def_regs(struct list_head *drlist) { struct deferred_registration *dr, *d; list_for_each_entry_safe(dr, d, drlist, list) { int result = 0; struct uprobe *u = dr->uprobe; if (dr->type == UPTY_URETPROBE) { struct uretprobe *rp = container_of(u, struct uretprobe, u); if (dr->regflag) result = register_uretprobe(rp); else unregister_uretprobe(rp); } else { if (dr->regflag) result = register_uprobe(u); else unregister_uprobe(u); } if (u && u->registration_callback) u->registration_callback(u, dr->regflag, dr->type, result); list_del(&dr->list); kfree(dr); } } /* * Functions for allocation of the SSOL area, and the instruction slots * therein */ /* * We leave the SSOL vma in place even after all the probes are gone. * We used to remember its address in current->mm->context.uprobes_ssol_area, * but adding that field to mm_context broke KAPI compatibility. * Instead, when we shut down the uproc for lack of probes, we "tag" the vma * for later identification. This is not particularly robust, but it's * no more vulnerable to ptrace or mprotect mischief than any other part * of the address space. */ #define UPROBES_SSOL_VMA_TAG \ "This is the SSOL area for uprobes. Mess with it at your own risk." #define UPROBES_SSOL_TAGSZ ((int)sizeof(UPROBES_SSOL_VMA_TAG)) /* * Searching downward from ceiling address (0 signifies top of memory), * find the next vma whose flags indicate it could be an SSOL area. * Return its address, or 0 for no match. */ static unsigned long find_next_possible_ssol_vma(unsigned long ceiling) { struct mm_struct *mm = current->mm; struct rb_node *rb_node; struct vm_area_struct *vma; unsigned long good_flags = VM_EXEC | VM_DONTEXPAND; unsigned long bad_flags = VM_WRITE | VM_GROWSDOWN | VM_GROWSUP; unsigned long addr = 0; down_read(&mm->mmap_sem); for (rb_node=rb_last(&mm->mm_rb); rb_node; rb_node=rb_prev(rb_node)) { vma = rb_entry(rb_node, struct vm_area_struct, vm_rb); if (ceiling && vma->vm_start >= ceiling) continue; if ((vma->vm_flags & good_flags) != good_flags) continue; if ((vma->vm_flags & bad_flags) != 0) continue; addr = vma->vm_start; break; } up_read(&mm->mmap_sem); return addr; } static noinline unsigned long find_old_ssol_vma(void) { unsigned long addr; unsigned long ceiling = 0; // top of memory char buf[UPROBES_SSOL_TAGSZ]; while ((addr = find_next_possible_ssol_vma(ceiling)) != 0) { ceiling = addr; if (copy_from_user(buf, (const void __user*)addr, UPROBES_SSOL_TAGSZ)) continue; if (!strcmp(buf, UPROBES_SSOL_VMA_TAG)) return addr; } return 0; } /* * Mmap nbytes bytes for the uprobes SSOL area for the current process. * Returns the address of the page, or a negative errno. * This approach was suggested by Roland McGrath. */ static noinline unsigned long uprobe_setup_ssol_vma(unsigned long nbytes) { unsigned long addr; struct mm_struct *mm = current->mm; struct vm_area_struct *vma; BUG_ON(nbytes & ~PAGE_MASK); if ((addr = find_old_ssol_vma()) != 0) return addr; down_write(&mm->mmap_sem); /* * Find the end of the top mapping and skip a page. * If there is no space for PAGE_SIZE above * that, mmap will ignore our address hint. */ vma = rb_entry(rb_last(&mm->mm_rb), struct vm_area_struct, vm_rb); addr = vma->vm_end + PAGE_SIZE; addr = do_mmap_pgoff(NULL, addr, nbytes, PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS, 0); if (addr & ~PAGE_MASK) { up_write(&mm->mmap_sem); printk(KERN_ERR "Uprobes failed to allocate a vma for" " pid/tgid %d/%d for single-stepping out of line.\n", current->pid, current->tgid); return addr; } vma = find_vma(mm, addr); BUG_ON(!vma); /* * Don't expand vma on mremap(). Allow vma to be copied on * fork() -- see uprobe_fork_uproc(). */ vma->vm_flags |= VM_DONTEXPAND; up_write(&mm->mmap_sem); return addr; } /** * uprobe_init_ssol -- initialize per-process area for single stepping * out-of-line. * @uproc: probed process * @tsk: probed task: must be current if @insn_area is %NULL * @insn_area: virtual address of the already-established SSOL vma -- * see uprobe_fork_uproc(). * * Returns with @uproc->ssol_area.insn_area pointing to the initialized * area, or set to a negative errno. */ static void uprobe_init_ssol(struct uprobe_process *uproc, struct task_struct *tsk, __user uprobe_opcode_t *insn_area) { struct uprobe_ssol_area *area = &uproc->ssol_area; struct uprobe_ssol_slot *slot; int i; char *slot_addr; // Simplify pointer arithmetic /* Trampoline setup will either fail or succeed here. */ uproc->uretprobe_trampoline_addr = ERR_PTR(-ENOMEM); if (insn_area) { BUG_ON(IS_ERR(insn_area)); area->insn_area = insn_area; } else { BUG_ON(tsk != current); area->insn_area = (uprobe_opcode_t *) uprobe_setup_ssol_vma(PAGE_SIZE); if (IS_ERR(area->insn_area)) return; } area->nfree = area->nslots = PAGE_SIZE / MAX_UINSN_BYTES; if (area->nslots > MAX_SSOL_SLOTS) area->nfree = area->nslots = MAX_SSOL_SLOTS; area->slots = (struct uprobe_ssol_slot *) kzalloc(sizeof(struct uprobe_ssol_slot) * area->nslots, GFP_USER); if (!area->slots) { area->insn_area = ERR_PTR(-ENOMEM); return; } mutex_init(&area->populate_mutex); spin_lock_init(&area->lock); area->next_slot = 0; slot_addr = (char*) area->insn_area; for (i = 0; i < area->nslots; i++) { slot = &area->slots[i]; init_rwsem(&slot->rwsem); slot->state = SSOL_FREE; slot->owner = NULL; slot->last_used = 0; slot->insn = (__user uprobe_opcode_t *) slot_addr; slot_addr += MAX_UINSN_BYTES; } uretprobe_set_trampoline(uproc, tsk); } /* * Verify that the SSOL area has been set up for uproc. Returns a * pointer to the SSOL area, or a negative erro if we couldn't set it up. */ static __user uprobe_opcode_t *uprobe_verify_ssol(struct uprobe_process *uproc) { struct uprobe_ssol_area *area = &uproc->ssol_area; if (unlikely(!area->initialized)) { /* First time through for this probed process */ mutex_lock(&uproc->ssol_area.setup_mutex); if (likely(!area->initialized)) { /* Nobody snuck in and set things up ahead of us. */ uprobe_init_ssol(uproc, current, NULL); area->initialized = 1; } mutex_unlock(&uproc->ssol_area.setup_mutex); } return area->insn_area; } /* * uproc is going away, but the process lives on. Tag the SSOL vma so a new * uproc can reuse it if more probes are requested. */ static void uprobe_release_ssol_vma(struct uprobe_process *uproc) { unsigned long addr; struct task_struct *tsk; static const char *buf = UPROBES_SSOL_VMA_TAG; int nb; /* No need to muck with dying image's mm_struct. */ BUG_ON(uproc->finished); addr = (unsigned long) uproc->ssol_area.insn_area; if (!addr || IS_ERR_VALUE(addr)) return; tsk = find_surviving_thread(uproc); if (!tsk) return; nb = access_process_vm(tsk, addr, (void*)buf, UPROBES_SSOL_TAGSZ, 1); if (nb != UPROBES_SSOL_TAGSZ) printk(KERN_ERR "Failed to tag uprobes SSOL vma: " "pid/tgid=%d/%d, vaddr=%#lx\n", tsk->pid, tsk->tgid, addr); } static inline int advance_slot(int slot, struct uprobe_ssol_area *area) { /* Slot 0 is reserved for uretprobe trampoline. */ slot++; if (unlikely(slot >= area->nslots)) slot = 1; return slot; } /* * Return the slot number of the least-recently-used slot in the * neighborhood of area->next_slot. Limit the number of slots we test * to keep it fast. Nobody dies if this isn't the best choice. */ static int uprobe_lru_insn_slot(struct uprobe_ssol_area *area) { #define MAX_LRU_TESTS 10 struct uprobe_ssol_slot *s; int lru_slot = -1; unsigned long lru_time = ULONG_MAX; int nr_lru_tests = 0; int slot = area->next_slot; do { s = &area->slots[slot]; if (likely(s->state == SSOL_ASSIGNED)) { if( lru_time > s->last_used) { lru_time = s->last_used; lru_slot = slot; } if (++nr_lru_tests >= MAX_LRU_TESTS) break; } slot = advance_slot(slot, area); } while (slot != area->next_slot); if (unlikely(lru_slot < 0)) /* All slots are in the act of being stolen. Join the melee. */ return area->next_slot; else return lru_slot; } /* * Choose an instruction slot and take it. Choose a free slot if there is one. * Otherwise choose the least-recently-used slot. Returns with slot * read-locked and containing the desired instruction. Runs with * ppt->slot_mutex locked. */ static struct uprobe_ssol_slot *uprobe_take_insn_slot(struct uprobe_probept *ppt) { struct uprobe_process *uproc = ppt->uproc; struct uprobe_ssol_area *area = &uproc->ssol_area; struct uprobe_ssol_slot *s; int len, slot; unsigned long flags; spin_lock_irqsave(&area->lock, flags); if (area->nfree) { for (slot = 0; slot < area->nslots; slot++) { if (area->slots[slot].state == SSOL_FREE) { area->nfree--; goto found_slot; } } /* Shouldn't get here. Fix nfree and get on with life. */ area->nfree = 0; } slot = uprobe_lru_insn_slot(area); found_slot: area->next_slot = advance_slot(slot, area); s = &area->slots[slot]; s->state = SSOL_BEING_STOLEN; spin_unlock_irqrestore(&area->lock, flags); /* Wait for current users of slot to finish. */ down_write(&s->rwsem); ppt->slot = s; s->owner = ppt; s->last_used = jiffies; s->state = SSOL_ASSIGNED; /* Copy the original instruction to the chosen slot. */ mutex_lock(&area->populate_mutex); len = access_process_vm(current, (unsigned long)s->insn, ppt->insn, MAX_UINSN_BYTES, 1); mutex_unlock(&area->populate_mutex); if (unlikely(len < MAX_UINSN_BYTES)) { up_write(&s->rwsem); printk(KERN_ERR "Failed to copy instruction at %#lx" " to SSOL area (%#lx)\n", ppt->vaddr, (unsigned long) area->slots); return NULL; } /* Let other threads single-step in this slot. */ downgrade_write(&s->rwsem); return s; } /* ppt doesn't own a slot. Get one for ppt, and return it read-locked. */ static struct uprobe_ssol_slot *uprobe_find_insn_slot(struct uprobe_probept *ppt) { struct uprobe_ssol_slot *slot; mutex_lock(&ppt->slot_mutex); slot = ppt->slot; if (unlikely(slot && slot->owner == ppt)) { /* Looks like another thread snuck in and got a slot for us. */ down_read(&slot->rwsem); if (likely(slot->owner == ppt)) { slot->last_used = jiffies; mutex_unlock(&ppt->slot_mutex); return slot; } /* ... but then somebody stole it. */ up_read(&slot->rwsem); } slot = uprobe_take_insn_slot(ppt); mutex_unlock(&ppt->slot_mutex); return slot; } /* * Ensure that ppt owns an instruction slot for single-stepping. * Returns with the slot read-locked and ppt->slot pointing at it. */ static struct uprobe_ssol_slot *uprobe_get_insn_slot(struct uprobe_probept *ppt) { struct uprobe_ssol_slot *slot; retry: slot = ppt->slot; if (unlikely(!slot)) return uprobe_find_insn_slot(ppt); down_read(&slot->rwsem); if (unlikely(slot != ppt->slot)) { up_read(&slot->rwsem); goto retry; } if (unlikely(slot->owner != ppt)) { up_read(&slot->rwsem); return uprobe_find_insn_slot(ppt); } slot->last_used = jiffies; return slot; } /* * utrace engine report callbacks */ /* * We've been asked to quiesce, but aren't in a position to do so. * This could happen in either of the following cases: * * 1) Our own thread is doing a register or unregister operation -- * e.g., as called from a u[ret]probe handler or a non-uprobes utrace * callback. We can't wait_event() for ourselves in [un]register_uprobe(). * * 2) We've been asked to quiesce, but we hit a probepoint first. Now * we're in the report_signal callback, having handled the probepoint. * We'd like to just set the UTRACE_ACTION_QUIESCE and * UTRACE_EVENT(QUIESCE) flags and coast into quiescence. Unfortunately, * it's possible to hit a probepoint again before we quiesce. When * processing the SIGTRAP, utrace would call uprobe_report_quiesce(), * which must decline to take any action so as to avoid removing the * uprobe just hit. As a result, we could keep hitting breakpoints * and never quiescing. * * So here we do essentially what we'd prefer to do in uprobe_report_quiesce(). * If we're the last thread to quiesce, handle_pending_uprobes() and * rouse_all_threads(). Otherwise, pretend we're quiescent and sleep until * the last quiescent thread handles that stuff and then wakes us. * * Called and returns with no mutexes held. Returns 1 if we free utask->uproc, * else 0. */ static int utask_fake_quiesce(struct uprobe_task *utask) { struct uprobe_process *uproc = utask->uproc; enum uprobe_task_state prev_state = utask->state; down_write(&uproc->rwsem); /* In case we're somehow set to quiesce for real... */ clear_utrace_quiesce(utask); if (uproc->n_quiescent_threads == uproc->nthreads-1) { /* We're the last thread to "quiesce." */ handle_pending_uprobes(uproc, utask->tsk); rouse_all_threads(uproc); up_write(&uproc->rwsem); return 0; } else { utask->state = UPTASK_SLEEPING; uproc->n_quiescent_threads++; up_write(&uproc->rwsem); /* We ref-count sleepers. */ uprobe_get_process(uproc); wait_event(uproc->waitq, !utask->quiescing); down_write(&uproc->rwsem); utask->state = prev_state; uproc->n_quiescent_threads--; up_write(&uproc->rwsem); /* * If uproc's last uprobe has been unregistered, and * unregister_uprobe() woke up before we did, it's up * to us to free uproc. */ return uprobe_put_process(uproc); } } /* Prepare to single-step ppt's probed instruction inline. */ static inline void uprobe_pre_ssin(struct uprobe_task *utask, struct uprobe_probept *ppt, struct pt_regs *regs) { int len; arch_reset_ip_for_sstep(regs); mutex_lock(&ppt->ssil_mutex); len = set_orig_insn(ppt, utask->tsk); if (unlikely(len != BP_INSN_SIZE)) { printk("Failed to temporarily restore original " "instruction for single-stepping: " "pid/tgid=%d/%d, vaddr=%#lx\n", utask->tsk->pid, utask->tsk->tgid, ppt->vaddr); utask->doomed = 1; } } /* Prepare to continue execution after single-stepping inline. */ static inline void uprobe_post_ssin(struct uprobe_task *utask, struct uprobe_probept *ppt) { int len = set_bp(ppt, utask->tsk); if (unlikely(len != BP_INSN_SIZE)) { printk("Couldn't restore bp: pid/tgid=%d/%d, addr=%#lx\n", utask->tsk->pid, utask->tsk->tgid, ppt->vaddr); ppt->state = UPROBE_DISABLED; } mutex_unlock(&ppt->ssil_mutex); } /* uprobe_pre_ssout() and uprobe_post_ssout() are architecture-specific. */ /* * Delay delivery of the indicated signal until after single-step. * Otherwise single-stepping will be cancelled as part of calling * the signal handler. */ static u32 uprobe_delay_signal(struct uprobe_task *utask, siginfo_t *info) { struct delayed_signal *ds = kmalloc(sizeof(*ds), GFP_USER); if (ds) { ds->info = *info; INIT_LIST_HEAD(&ds->list); list_add_tail(&ds->list, &utask->delayed_signals); } return UTRACE_ACTION_HIDE | UTRACE_SIGNAL_IGN | UTRACE_ACTION_SINGLESTEP | UTRACE_ACTION_NEWSTATE; } static void uprobe_inject_delayed_signals(struct list_head *delayed_signals) { struct delayed_signal *ds, *tmp; list_for_each_entry_safe(ds, tmp, delayed_signals, list) { send_sig_info(ds->info.si_signo, &ds->info, current); list_del(&ds->list); kfree(ds); } } /* * Signal callback: * * We get called here with: * state = UPTASK_RUNNING => we are here due to a breakpoint hit * - Read-lock the process * - Figure out which probepoint, based on regs->IP * - Set state = UPTASK_BP_HIT * - Reset regs->IP to beginning of the insn, if necessary * - Invoke handler for each uprobe at this probepoint * - Set singlestep in motion (UTRACE_ACTION_SINGLESTEP), * with state = UPTASK_SSTEP * * state = UPTASK_SSTEP => here after single-stepping * - Validate we are here per the state machine * - Clean up after single-stepping * - Set state = UPTASK_RUNNING * - Read-unlock the process * - If it's time to quiesce, take appropriate action. * - If the handler(s) we ran called [un]register_uprobe(), * complete those via uprobe_run_def_regs(). * * state = ANY OTHER STATE * - Not our signal, pass it on (UTRACE_ACTION_RESUME) * Note: Intermediate states such as UPTASK_POST_SSTEP help * uprobe_report_exit() decide what to unlock if we die. */ static u32 uprobe_report_signal(struct utrace_attached_engine *engine, struct task_struct *tsk, struct pt_regs *regs, u32 action, siginfo_t *info, const struct k_sigaction *orig_ka, struct k_sigaction *return_ka) { struct uprobe_task *utask; struct uprobe_probept *ppt; struct uprobe_process *uproc; struct uprobe_kimg *uk; u32 ret; unsigned long probept; int hit_uretprobe_trampoline = 0; int registrations_deferred = 0; int uproc_freed = 0; struct list_head delayed_signals; utask = (struct uprobe_task *)rcu_dereference(engine->data); BUG_ON(!utask); /* * info will be null if we're called with action=UTRACE_SIGNAL_HANDLER, * which means that single-stepping has been disabled so a signal * handler can be called in the probed process. That should never * happen because we intercept and delay handled signals (action = * UTRACE_ACTION_RESUME) until after we're done single-stepping. */ BUG_ON(!info); if (action == UTRACE_ACTION_RESUME && utask->active_probe && info->si_signo != SSTEP_SIGNAL) return uprobe_delay_signal(utask, info); if (info->si_signo != BREAKPOINT_SIGNAL && info->si_signo != SSTEP_SIGNAL) goto no_interest; /* * Set up the SSOL area if it's not already there. We do this * here because we have to do it before handling the first * probepoint hit, the probed process has to do it, and this may * be the first time our probed process runs uprobes code. * * We need the SSOL area for the uretprobe trampoline even if * this architectures doesn't single-step out of line. */ uproc = utask->uproc; #ifdef CONFIG_UPROBES_SSOL if (uproc->sstep_out_of_line && unlikely(IS_ERR(uprobe_verify_ssol(uproc)))) uproc->sstep_out_of_line = 0; #elif defined(CONFIG_URETPROBES) (void) uprobe_verify_ssol(uproc); #endif switch (utask->state) { case UPTASK_RUNNING: if (info->si_signo != BREAKPOINT_SIGNAL) goto no_interest; down_read(&uproc->rwsem); clear_utrace_quiesce(utask); probept = arch_get_probept(regs); hit_uretprobe_trampoline = (probept == (unsigned long) uproc->uretprobe_trampoline_addr); if (hit_uretprobe_trampoline) { uretprobe_handle_return(regs, utask); goto bkpt_done; } ppt = uprobe_find_probept(uproc, probept); if (!ppt) { up_read(&uproc->rwsem); goto no_interest; } utask->active_probe = ppt; utask->state = UPTASK_BP_HIT; if (likely(ppt->state == UPROBE_BP_SET)) { list_for_each_entry(uk, &ppt->uprobe_list, list) { struct uprobe *u = uk->uprobe; if (is_uretprobe(u)) uretprobe_handle_entry(u, regs, utask); else if (u->handler) u->handler(u, regs); } } if (uprobe_emulate_insn(regs, ppt)) goto bkpt_done; utask->state = UPTASK_PRE_SSTEP; #ifdef CONFIG_UPROBES_SSOL if (uproc->sstep_out_of_line) uprobe_pre_ssout(utask, ppt, regs); else #endif uprobe_pre_ssin(utask, ppt, regs); if (unlikely(utask->doomed)) do_exit(SIGSEGV); utask->state = UPTASK_SSTEP; /* * No other engines must see this signal, and the * signal shouldn't be passed on either. */ ret = UTRACE_ACTION_HIDE | UTRACE_SIGNAL_IGN | UTRACE_ACTION_SINGLESTEP | UTRACE_ACTION_NEWSTATE; break; case UPTASK_SSTEP: if (info->si_signo != SSTEP_SIGNAL) goto no_interest; ppt = utask->active_probe; BUG_ON(!ppt); utask->state = UPTASK_POST_SSTEP; #ifdef CONFIG_UPROBES_SSOL if (uproc->sstep_out_of_line) uprobe_post_ssout(utask, ppt, regs); else #endif uprobe_post_ssin(utask, ppt); bkpt_done: /* Note: Can come here after running uretprobe handlers */ if (unlikely(utask->doomed)) do_exit(SIGSEGV); utask->active_probe = NULL; if (!list_empty(&utask->deferred_registrations)) { /* * Make sure utask doesn't go away before we run * the deferred registrations. This also keeps * the module from getting unloaded before we're * ready. */ registrations_deferred = 1; uprobe_get_process(uproc); } /* * Delayed signals are a little different. We want * them delivered even if all the probes get unregistered * and uproc and utask go away. So disconnect the list * from utask and make it a local list. */ INIT_LIST_HEAD(&delayed_signals); list_splice_init(&utask->delayed_signals, &delayed_signals); ret = UTRACE_ACTION_HIDE | UTRACE_SIGNAL_IGN | UTRACE_ACTION_NEWSTATE; utask->state = UPTASK_RUNNING; if (utask->quiescing) { up_read(&uproc->rwsem); uproc_freed |= utask_fake_quiesce(utask); } else up_read(&uproc->rwsem); if (hit_uretprobe_trampoline) /* * It's possible that the uretprobe_instance * we just recycled was the last reason for * keeping uproc around. */ uproc_freed |= uprobe_put_process(uproc); if (registrations_deferred) { uprobe_run_def_regs(&utask->deferred_registrations); uproc_freed |= uprobe_put_process(uproc); } uprobe_inject_delayed_signals(&delayed_signals); if (uproc_freed) ret |= UTRACE_ACTION_DETACH; break; default: goto no_interest; } return ret; no_interest: return UTRACE_ACTION_RESUME; } /* * utask_quiesce_pending_sigtrap: The utask entered the quiesce callback * through the signal delivery path, apparently. Check if the associated * signal happened due to a uprobe hit. * * Called with utask->uproc write-locked. Returns 1 if quiesce was * entered with SIGTRAP pending due to a uprobe hit. */ static int utask_quiesce_pending_sigtrap(struct uprobe_task *utask) { const struct utrace_regset_view *view; const struct utrace_regset *regset; struct uprobe_probept *ppt; unsigned long insn_ptr; view = utrace_native_view(utask->tsk); regset = utrace_regset(utask->tsk, utask->engine, view, 0); if (unlikely(regset == NULL)) return -EIO; if ((*regset->get)(utask->tsk, regset, SLOT_IP(utask->tsk) * regset->size, regset->size, &insn_ptr, NULL) != 0) return -EIO; if (regset->size != sizeof(insn_ptr)) { /* Assume 32-bit app and 64-bit kernel. */ u32 *insn_ptr32 = (u32*) &insn_ptr; BUG_ON(regset->size != sizeof(u32)); insn_ptr = *insn_ptr32; } ppt = uprobe_find_probept(utask->uproc, ARCH_BP_INST_PTR(insn_ptr)); return (ppt != NULL); } /* * Quiesce callback: The associated process has one or more breakpoint * insertions or removals pending. If we're the last thread in this * process to quiesce, do the insertion(s) and/or removal(s). */ static u32 uprobe_report_quiesce(struct utrace_attached_engine *engine, struct task_struct *tsk) { struct uprobe_task *utask; struct uprobe_process *uproc; rcu_read_lock(); utask = (struct uprobe_task *)rcu_dereference(engine->data); BUG_ON(!utask); uproc = uprobe_get_process(utask->uproc); rcu_read_unlock(); if (!uproc) return UTRACE_ACTION_DETACH|UTRACE_ACTION_RESUME; if (current == utask->quiesce_master) { /* * tsk was already quiescent when quiesce_all_threads() * called utrace_set_flags(), which in turned called * here. uproc is already locked. Do as little as possible * and get out. */ utask->state = UPTASK_QUIESCENT; uproc->n_quiescent_threads++; return UTRACE_ACTION_RESUME; } BUG_ON(utask->active_probe); down_write(&uproc->rwsem); /* printk(KERN_INFO "uprobe_report_quiesce2 %p %ld=%ld\n", uproc, uproc->tgid, current->pid); */ /* * When a thread hits a breakpoint or single-steps, utrace calls * this quiesce callback before our signal callback. We must * let uprobe_report_signal() handle the uprobe hit and THEN * quiesce, because (a) there's a chance that we're quiescing * in order to remove that very uprobe, and (b) there's a tiny * chance that even though that uprobe isn't marked for removal * now, it may be before all threads manage to quiesce. */ if (!utask->quiescing || utask_quiesce_pending_sigtrap(utask) == 1) { clear_utrace_quiesce(utask); goto done; } utask->state = UPTASK_QUIESCENT; uproc->n_quiescent_threads++; check_uproc_quiesced(uproc, tsk); done: up_write(&uproc->rwsem); uprobe_put_process(uproc); /* printk(KERN_INFO "uprobe_report_quiesce3 %p %ld=%ld\n", uproc, uproc->tgid, current->pid); */ return UTRACE_ACTION_RESUME; } /* * uproc's process is exiting or exec-ing, so zap all the (now irrelevant) * probepoints and uretprobe_instances. Runs with uproc->rwsem write-locked. * Caller must ref-count uproc before calling this function, to ensure that * uproc doesn't get freed in the middle of this. */ static void uprobe_cleanup_process(struct uprobe_process *uproc) { int i; struct uprobe_probept *ppt; struct hlist_node *pnode1, *pnode2; struct hlist_head *head; struct uprobe_kimg *uk, *unode; struct uprobe_task *utask; uproc->finished = 1; for (i = 0; i < UPROBE_TABLE_SIZE; i++) { head = &uproc->uprobe_table[i]; hlist_for_each_entry_safe(ppt, pnode1, pnode2, head, ut_node) { if (ppt->state == UPROBE_INSERTING || ppt->state == UPROBE_REMOVING) { /* * This task is (exec/exit)ing with * a [un]register_uprobe pending. * [un]register_uprobe will free ppt. */ ppt->state = UPROBE_DISABLED; list_del(&ppt->pd_node); list_for_each_entry_safe(uk, unode, &ppt->uprobe_list, list) uk->status = -ESRCH; wake_up_all(&ppt->waitq); } else if (ppt->state == UPROBE_BP_SET) { list_for_each_entry_safe(uk, unode, &ppt->uprobe_list, list) { list_del(&uk->list); uprobe_free_kimg(uk); } uprobe_free_probept(ppt); /* else */ /* * If ppt is UPROBE_DISABLED, assume that * [un]register_uprobe() has been notified * and will free it soon. */ } } } /* * Free uretprobe_instances. This is a nop on exit, since all * the uprobe_tasks are already gone. We do this here on exec * (as opposed to letting uprobe_free_process() take care of it) * because uprobe_free_process() never gets called if we don't * tick down the ref count here (PR #7082). */ list_for_each_entry(utask, &uproc->thread_list, list) utask_free_uretprobe_instances(utask); } /* * Exit callback: The associated task/thread is exiting. */ static u32 uprobe_report_exit(struct utrace_attached_engine *engine, struct task_struct *tsk, long orig_code, long *code) { struct uprobe_task *utask; struct uprobe_process *uproc = NULL; struct uprobe_probept *ppt; int utask_quiescing; utask = (struct uprobe_task *)rcu_dereference(engine->data); if (utask) uproc = uprobe_get_process(utask->uproc); if (!utask || !uproc) /* uprobe_free_process() has probably clobbered utask->proc. */ return UTRACE_ACTION_DETACH; /* printk(KERN_INFO "uprobe_report_exit %p %ld=%ld\n", uproc, uproc->tgid, current->pid); */ ppt = utask->active_probe; if (ppt) { if (utask->state == UPTASK_TRAMPOLINE_HIT) printk(KERN_WARNING "Task died during uretprobe return:" " pid/tgid = %d/%d\n", tsk->pid, tsk->tgid); else printk(KERN_WARNING "Task died at uprobe probepoint:" " pid/tgid = %d/%d, probepoint = %#lx\n", tsk->pid, tsk->tgid, ppt->vaddr); /* Mutex cleanup depends on where we died and SSOL vs. SSIL. */ if (uproc->sstep_out_of_line) { if (utask->state == UPTASK_SSTEP && ppt->slot && ppt->slot->owner == ppt) up_read(&ppt->slot->rwsem); } else { switch (utask->state) { case UPTASK_PRE_SSTEP: case UPTASK_SSTEP: case UPTASK_POST_SSTEP: mutex_unlock(&ppt->ssil_mutex); break; default: break; } } up_read(&uproc->rwsem); if (utask->state == UPTASK_TRAMPOLINE_HIT) uprobe_decref_process(uproc); } down_write(&uproc->rwsem); /* printk(KERN_INFO "uprobe_report_exit2 %p %ld=%ld\n", uproc, uproc->tgid, current->pid); */ utask_quiescing = utask->quiescing; uprobe_free_task(utask); uproc->nthreads--; if (uproc->nthreads) { if (utask_quiescing) /* * In case other threads are waiting for * us to quiesce... */ check_uproc_quiesced(uproc, find_surviving_thread(uproc)); } else { /* * We were the last remaining thread - clean up the uprobe * remnants a la unregister_uprobe(). We don't have to * remove the breakpoints, though. */ uprobe_cleanup_process(uproc); } up_write(&uproc->rwsem); /* printk(KERN_INFO "uprobe_report_exit3 %p %ld=%ld\n", uproc, uproc->tgid, current->pid); */ uprobe_put_process(uproc); return UTRACE_ACTION_DETACH; } /* * Duplicate the FIFO of uretprobe_instances from parent_utask into * child_utask. Zap the uretprobe pointer, since all we care about is * vectoring to the proper return address. Where there are multiple * uretprobe_instances for the same function instance, copy only the * one that contains the real return address. */ static int uprobe_fork_uretprobe_instances(struct uprobe_task *parent_utask, struct uprobe_task *child_utask) { struct uprobe_process *parent_uproc = parent_utask->uproc; struct uprobe_process *child_uproc = child_utask->uproc; __user uprobe_opcode_t *trampoline_addr = child_uproc->uretprobe_trampoline_addr; struct hlist_node *tmp, *tail; struct uretprobe_instance *pri, *cri; BUG_ON(trampoline_addr != parent_uproc->uretprobe_trampoline_addr); /* Since there's no hlist_add_tail()... */ tail = NULL; hlist_for_each_entry(pri, tmp, &parent_utask->uretprobe_instances, hlist) { if (pri->ret_addr == (unsigned long) trampoline_addr) continue; cri = kmalloc(sizeof(*cri), GFP_USER); if (!cri) return -ENOMEM; cri->rp = NULL; cri->ret_addr = pri->ret_addr; cri->sp = pri->sp; INIT_HLIST_NODE(&cri->hlist); if (tail) hlist_add_after(tail, &cri->hlist); else hlist_add_head(&cri->hlist, &child_utask->uretprobe_instances); tail = &cri->hlist; /* Ref-count uretprobe_instances. */ uprobe_get_process(child_uproc); } BUG_ON(hlist_empty(&child_utask->uretprobe_instances)); return 0; } /* * A probed process is forking, and at least one function in the * call stack has a uretprobe on it. Since the child inherits the * call stack, it's possible that the child could attempt to return * through the uretprobe trampoline. Create a uprobe_process for * the child, initialize its SSOL vma (which has been cloned from * the parent), and clone the parent's list of uretprobe_instances. * * Called with uproc_table locked and parent_uproc->rwsem write-locked. * * (On architectures where it's easy to keep track of where in the * stack the return addresses are stored, we could just poke the real * return addresses back into the child's stack. We use this more * general solution.) */ static int uprobe_fork_uproc(struct uprobe_process *parent_uproc, struct uprobe_task *parent_utask, struct task_struct *child_tsk) { int ret = 0; struct uprobe_process *child_uproc; struct uprobe_task *child_utask; BUG_ON(!parent_uproc->uretprobe_trampoline_addr || IS_ERR(parent_uproc->uretprobe_trampoline_addr)); ret = uprobe_validate_vma(child_tsk, (unsigned long) parent_uproc->ssol_area.insn_area); if (ret) { int ret2; printk(KERN_ERR "uprobes: Child %d failed to inherit" " parent %d's SSOL vma at %p. Error = %d\n", child_tsk->pid, parent_utask->tsk->pid, parent_uproc->ssol_area.insn_area, ret); ret2 = uprobe_validate_vma(parent_utask->tsk, (unsigned long) parent_uproc->ssol_area.insn_area); if (ret2 != 0) printk(KERN_ERR "uprobes: Parent %d's SSOL vma" " is no longer valid. Error = %d\n", parent_utask->tsk->pid, ret2); return ret; } if (!try_module_get(THIS_MODULE)) return -ENOSYS; child_uproc = uprobe_mk_process(child_tsk, 1); if (IS_ERR(child_uproc)) { ret = (int) PTR_ERR(child_uproc); module_put(THIS_MODULE); return ret; } mutex_lock(&child_uproc->ssol_area.setup_mutex); uprobe_init_ssol(child_uproc, child_tsk, parent_uproc->ssol_area.insn_area); child_uproc->ssol_area.initialized = 1; mutex_unlock(&child_uproc->ssol_area.setup_mutex); child_utask = uprobe_find_utask(child_tsk); BUG_ON(!child_utask); ret = uprobe_fork_uretprobe_instances(parent_utask, child_utask); hlist_add_head(&child_uproc->hlist, &uproc_table[hash_long(child_uproc->tgid, UPROBE_HASH_BITS)]); uprobe_decref_process(child_uproc); return ret; } /* * Clone callback: The current task has spawned a thread/process. * * NOTE: For now, we don't pass on uprobes from the parent to the * child. We now do the necessary clearing of breakpoints in the * child's address space. * * TODO: * - Provide option for child to inherit uprobes. */ static u32 uprobe_report_clone(struct utrace_attached_engine *engine, struct task_struct *parent, unsigned long clone_flags, struct task_struct *child) { int len; struct uprobe_process *uproc; struct uprobe_task *ptask, *ctask; ptask = (struct uprobe_task *)rcu_dereference(engine->data); uproc = ptask->uproc; /* printk(KERN_INFO "uprobe_report_clone %p %ld=%ld\n", uproc, uproc->tgid, current->pid); */ /* * Lock uproc so no new uprobes can be installed 'til all * report_clone activities are completed. Lock uproc_table * in case we have to run uprobe_fork_uproc(). */ lock_uproc_table(); down_write(&uproc->rwsem); get_task_struct(child); /* printk(KERN_INFO "uprobe_report_clone2 %p %ld=%ld\n", uproc, uproc->tgid, current->pid); */ if (clone_flags & (CLONE_THREAD|CLONE_VM)) { /* New thread in the same process (CLONE_THREAD) or * processes sharing the same memory space (CLONE_VM). */ ctask = uprobe_add_task(child, uproc); BUG_ON(!ctask); if (IS_ERR(ctask)) goto done; uproc->nthreads++; /* * FIXME: Handle the case where uproc is quiescing * (assuming it's possible to clone while quiescing). */ } else { /* * New process spawned by parent. Remove the probepoints * in the child's text. * * Its not necessary to quiesce the child as we are assured * by utrace that this callback happens *before* the child * gets to run userspace. * * We also hold the uproc->rwsem for the parent - so no * new uprobes will be registered 'til we return. */ int i; struct uprobe_probept *ppt; struct hlist_node *node; struct hlist_head *head; for (i = 0; i < UPROBE_TABLE_SIZE; i++) { head = &uproc->uprobe_table[i]; hlist_for_each_entry(ppt, node, head, ut_node) { len = set_orig_insn(ppt, child); if (len != BP_INSN_SIZE) { /* Ratelimit this? */ printk(KERN_ERR "Pid %d forked %d;" " failed to remove probepoint" " at %#lx in child\n", parent->pid, child->pid, ppt->vaddr); } } } if (!hlist_empty(&ptask->uretprobe_instances)) (void) uprobe_fork_uproc(uproc, ptask, child); } done: put_task_struct(child); up_write(&uproc->rwsem); unlock_uproc_table(); return UTRACE_ACTION_RESUME; } /* * Exec callback: The associated process called execve() or friends * * The new program is about to start running and so there is no * possibility of a uprobe from the previous user address space * to be hit. * * NOTE: * Typically, this process would have passed through the clone * callback, where the necessary action *should* have been * taken. However, if we still end up at this callback: * - We don't have to clear the uprobes - memory image * will be overlaid. * - We have to free up uprobe resources associated with * this process. */ static u32 uprobe_report_exec(struct utrace_attached_engine *engine, struct task_struct *tsk, const struct linux_binprm *bprm, struct pt_regs *regs) { struct uprobe_process *uproc = NULL; struct uprobe_task *utask; u32 ret = UTRACE_ACTION_RESUME; utask = (struct uprobe_task *)rcu_dereference(engine->data); if (utask) uproc = uprobe_get_process(utask->uproc); if (!utask || !uproc) /* uprobe_free_process() has probably clobbered utask->proc. */ return UTRACE_ACTION_DETACH; /* printk(KERN_INFO "uprobe_report_exec %p %ld=%ld\n", uproc, uproc->tgid, current->pid); */ /* * Only cleanup if we're the last thread. If we aren't, * uprobe_report_exit() will handle cleanup. * * One instance of this can happen if vfork() was called, * creating 2 tasks that share the same memory space * (CLONE_VFORK|CLONE_VM). In this case we don't want to * remove the probepoints from the child, since that would * also remove them from the parent. Instead, just detach * as if this were a simple thread exit. */ down_write(&uproc->rwsem); if (uproc->nthreads == 1) { uprobe_cleanup_process(uproc); /* * If [un]register_uprobe() is in progress, cancel the * quiesce. Otherwise, utrace_report_exec() might * call uprobe_report_exec() while the * [un]register_uprobe thread is freeing the uproc. */ clear_utrace_quiesce(utask); } else { uprobe_free_task(utask); uproc->nthreads--; ret = UTRACE_ACTION_DETACH; } up_write(&uproc->rwsem); /* printk(KERN_INFO "uprobe_report_exec2 %p %ld=%ld\n", uproc, uproc->tgid, current->pid); */ /* If any [un]register_uprobe is pending, it'll clean up. */ if (uprobe_put_process(uproc)) ret = UTRACE_ACTION_DETACH; /* printk(KERN_INFO "uprobe_report_exec4 %p %ld=%ld ret=%lu\n", uproc, uproc->tgid, current->pid, (unsigned long)ret); */ return ret; } static const struct utrace_engine_ops uprobe_utrace_ops = { .report_quiesce = uprobe_report_quiesce, .report_signal = uprobe_report_signal, .report_exit = uprobe_report_exit, .report_clone = uprobe_report_clone, .report_exec = uprobe_report_exec }; static int __init init_uprobes(void) { int i; for (i = 0; i < UPROBE_TABLE_SIZE; i++) { INIT_HLIST_HEAD(&uproc_table[i]); INIT_HLIST_HEAD(&utask_table[i]); } p_uprobe_utrace_ops = &uprobe_utrace_ops; return 0; } static void __exit exit_uprobes(void) { } module_init(init_uprobes); module_exit(exit_uprobes); #ifdef CONFIG_URETPROBES /* Returns true if ri_sp lies outside the stack (beyond cursp). */ static inline bool compare_stack_ptrs(unsigned long cursp, unsigned long ri_sp) { #ifdef CONFIG_STACK_GROWSUP if (cursp < ri_sp) return true; #else if (cursp > ri_sp) return true; #endif return false; } /* * A longjmp may cause one or more uretprobed functions to terminate without * returning. Those functions' uretprobe_instances need to be recycled. * We detect this when any uretprobed function is subsequently called * or returns. A bypassed uretprobe_instance's stack_ptr is beyond the * current stack. */ static inline void uretprobe_bypass_instances(unsigned long cursp, struct uprobe_task *utask) { struct hlist_node *r1, *r2; struct uretprobe_instance *ri; struct hlist_head *head = &utask->uretprobe_instances; hlist_for_each_entry_safe(ri, r1, r2, head, hlist) { if (compare_stack_ptrs(cursp, ri->sp)) { hlist_del(&ri->hlist); kfree(ri); uprobe_decref_process(utask->uproc); } else return; } } /* Called when the entry-point probe u is hit. */ static void uretprobe_handle_entry(struct uprobe *u, struct pt_regs *regs, struct uprobe_task *utask) { struct uretprobe_instance *ri; unsigned long trampoline_addr; if (IS_ERR(utask->uproc->uretprobe_trampoline_addr)) return; trampoline_addr = (unsigned long) utask->uproc->uretprobe_trampoline_addr; ri = (struct uretprobe_instance *) kmalloc(sizeof(struct uretprobe_instance), GFP_USER); if (!ri) return; ri->ret_addr = arch_hijack_uret_addr(trampoline_addr, regs, utask); if (likely(ri->ret_addr)) { ri->sp = arch_predict_sp_at_ret(regs, utask->tsk); uretprobe_bypass_instances(ri->sp, utask); ri->rp = container_of(u, struct uretprobe, u); INIT_HLIST_NODE(&ri->hlist); hlist_add_head(&ri->hlist, &utask->uretprobe_instances); /* We ref-count outstanding uretprobe_instances. */ uprobe_get_process(utask->uproc); } else kfree(ri); } /* * For each uretprobe_instance pushed onto the LIFO for the function * instance that's now returning, call the handler, free the ri, and * decrement the uproc's ref count. Caller ref-counts uproc, so we * should never hit zero in this function. * * Returns the original return address. * * TODO: Handle longjmp out of uretprobed function. */ static unsigned long uretprobe_run_handlers(struct uprobe_task *utask, struct pt_regs *regs, unsigned long trampoline_addr) { unsigned long ret_addr, cur_sp; struct hlist_head *head = &utask->uretprobe_instances; struct uretprobe_instance *ri; struct hlist_node *r1, *r2; cur_sp = arch_get_cur_sp(regs); uretprobe_bypass_instances(cur_sp, utask); hlist_for_each_entry_safe(ri, r1, r2, head, hlist) { if (ri->rp && ri->rp->handler) ri->rp->handler(ri, regs); ret_addr = ri->ret_addr; hlist_del(&ri->hlist); kfree(ri); uprobe_decref_process(utask->uproc); if (ret_addr != trampoline_addr) /* * This is the first ri (chronologically) pushed for * this particular instance of the probed function. */ return ret_addr; } printk(KERN_ERR "No uretprobe instance with original return address!" " pid/tgid=%d/%d", utask->tsk->pid, utask->tsk->tgid); utask->doomed = 1; return 0; } /* Called when the uretprobe trampoline is hit. */ static void uretprobe_handle_return(struct pt_regs *regs, struct uprobe_task *utask) { unsigned long orig_ret_addr; /* Delay recycling of uproc until end of uprobe_report_signal() */ uprobe_get_process(utask->uproc); utask->state = UPTASK_TRAMPOLINE_HIT; utask->active_probe = &uretprobe_trampoline_dummy_probe; orig_ret_addr = uretprobe_run_handlers(utask, regs, (unsigned long) utask->uproc->uretprobe_trampoline_addr); arch_restore_uret_addr(orig_ret_addr, regs); } int register_uretprobe(struct uretprobe *rp) { if (!rp || !rp->handler) return -EINVAL; rp->u.handler = URETPROBE_HANDLE_ENTRY; return register_uprobe(&rp->u); } EXPORT_SYMBOL_GPL(register_uretprobe); /* * The uretprobe containing u is being unregistered. Its uretprobe_instances * have to hang around 'til their associated instances return (but we can't * run rp's handler). Zap ri->rp for each one to indicate unregistration. * * Runs with uproc write-locked. */ static void zap_uretprobe_instances(struct uprobe *u, struct uprobe_process *uproc) { struct uprobe_task *utask; struct uretprobe *rp = container_of(u, struct uretprobe, u); if (!uproc) return; list_for_each_entry(utask, &uproc->thread_list, list) { struct hlist_node *r; struct uretprobe_instance *ri; hlist_for_each_entry(ri, r, &utask->uretprobe_instances, hlist) if (ri->rp == rp) ri->rp = NULL; } } void unregister_uretprobe(struct uretprobe *rp) { if (!rp) return; unregister_uprobe(&rp->u); } EXPORT_SYMBOL_GPL(unregister_uretprobe); void unmap_uretprobe(struct uretprobe *rp) { if (!rp) return; unmap_uprobe(&rp->u); } EXPORT_SYMBOL_GPL(unmap_uretprobe); /* * uproc->ssol_area has been successfully set up. Establish the * uretprobe trampoline in slot 0. */ static void uretprobe_set_trampoline(struct uprobe_process *uproc, struct task_struct *tsk) { uprobe_opcode_t bp_insn = BREAKPOINT_INSTRUCTION; struct uprobe_ssol_area *area = &uproc->ssol_area; struct uprobe_ssol_slot *slot = &area->slots[0]; if (access_process_vm(tsk, (unsigned long) slot->insn, &bp_insn, BP_INSN_SIZE, 1) == BP_INSN_SIZE) { uproc->uretprobe_trampoline_addr = slot->insn; slot->state = SSOL_RESERVED; area->next_slot = 1; area->nfree--; } else { printk(KERN_ERR "uretprobes disabled for pid %d:" " cannot set uretprobe trampoline at %p\n", uproc->tgid, slot->insn); } } static inline unsigned long lookup_uretprobe(struct hlist_node *r, struct uprobe_process *uproc, unsigned long pc, unsigned long sp) { struct uretprobe_instance *ret_inst; unsigned long trampoline_addr; if (IS_ERR(uproc->uretprobe_trampoline_addr)) return pc; trampoline_addr = (unsigned long)uproc->uretprobe_trampoline_addr; if (pc != trampoline_addr) return pc; hlist_for_each_entry_from(ret_inst, r, hlist) { if (ret_inst->ret_addr == trampoline_addr) continue; /* First handler with a stack pointer lower than the address (or equal) must be the one. */ if (ret_inst->sp == sp || compare_stack_ptrs(ret_inst->sp, sp)) return ret_inst->ret_addr; } printk(KERN_ERR "Original return address for trampoline not found at " "0x%lx pid/tgid=%d/%d\n", sp, current->pid, current->tgid); return 0; } unsigned long uprobe_get_pc(struct uretprobe_instance *ri, unsigned long pc, unsigned long sp) { struct uretprobe *rp; struct uprobe_kimg *uk; struct uprobe_task *utask; struct uprobe_process *uproc; struct hlist_node *r; if (ri == GET_PC_URETPROBE_NONE) { utask = uprobe_find_utask(current); if (!utask) return 0; uproc = utask->uproc; r = utask->uretprobe_instances.first; } else { rp = ri->rp; uk = (struct uprobe_kimg *)rp->u.kdata; if (!uk) return 0; uproc = uk->ppt->uproc; r = &ri->hlist; } return lookup_uretprobe(r, uproc, pc, sp); } EXPORT_SYMBOL_GPL(uprobe_get_pc); unsigned long uprobe_get_pc_task(struct task_struct *task, unsigned long pc, unsigned long sp) { struct uprobe_task *utask; struct uprobe_process *uproc; unsigned long result; utask = uprobe_find_utask(task); if (!utask) { return pc; } else if (current == task && utask->active_probe) { /* everything's locked. */ return uprobe_get_pc(GET_PC_URETPROBE_NONE, pc, sp); } uproc = utask->uproc; down_read(&uproc->rwsem); result = lookup_uretprobe(utask->uretprobe_instances.first, uproc, pc, sp); up_read(&uproc->rwsem); return result; } EXPORT_SYMBOL_GPL(uprobe_get_pc_task); #else /* ! CONFIG_URETPROBES */ static void uretprobe_handle_entry(struct uprobe *u, struct pt_regs *regs, struct uprobe_task *utask) { } static void uretprobe_handle_return(struct pt_regs *regs, struct uprobe_task *utask) { } static void uretprobe_set_trampoline(struct uprobe_process *uproc, struct task_struct *tsk) { } static void zap_uretprobe_instances(struct uprobe *u, struct uprobe_process *uproc) { } #endif /* CONFIG_URETPROBES */ #ifdef NO_ACCESS_PROCESS_VM_EXPORT /* * Some kernel versions export everything that uprobes.ko needs except * access_process_vm, so we copied and pasted it here. Fortunately, * everything it calls is exported. */ #include <linux/pagemap.h> #include <asm/cacheflush.h> static int __access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write) { struct mm_struct *mm; struct vm_area_struct *vma; struct page *page; void *old_buf = buf; mm = get_task_mm(tsk); if (!mm) return 0; down_read(&mm->mmap_sem); /* ignore errors, just check how much was successfully transfered */ while (len) { int bytes, ret, offset; void *maddr; ret = get_user_pages(tsk, mm, addr, 1, write, 1, &page, &vma); if (ret <= 0) break; bytes = len; offset = addr & (PAGE_SIZE-1); if (bytes > PAGE_SIZE-offset) bytes = PAGE_SIZE-offset; maddr = kmap(page); if (write) { copy_to_user_page(vma, page, addr, maddr + offset, buf, bytes); set_page_dirty_lock(page); } else { copy_from_user_page(vma, page, addr, buf, maddr + offset, bytes); } kunmap(page); page_cache_release(page); len -= bytes; buf += bytes; addr += bytes; } up_read(&mm->mmap_sem); mmput(mm); return buf - old_buf; } #endif #include "uprobes_arch.c" MODULE_LICENSE("GPL"); #endif /* uprobes 1 (based on original utrace) */