1#@!#!123s

: / usr / share / systemtap / runtime /
Filename : stp_utrace.c
back
/*
 * utrace infrastructure interface for debugging user processes
 *
 * Copyright (C) 2006-2018 Red Hat, Inc.  All rights reserved.
 *
 * This copyrighted material is made available to anyone wishing to use,
 * modify, copy, or redistribute it subject to the terms and conditions
 * of the GNU General Public License v.2.
 *
 * Heavily based on the original utrace code by Roland McGrath.
 */

#ifndef _STP_UTRACE_C
#define _STP_UTRACE_C

#if (!defined(STAPCONF_UTRACE_VIA_TRACEPOINTS))
#error "STAPCONF_UTRACE_VIA_TRACEPOINTS must be defined."
#endif

#include "stp_utrace.h"
#include <linux/bitmap.h>
#include <linux/list.h>
#include <linux/sched.h>
#include <linux/freezer.h>
#include <linux/slab.h>
#include <trace/events/sched.h>
#include <trace/events/syscalls.h>
#include "stp_task_work.c"
#include "linux/stp_tracepoint.h"

#include "stp_helper_lock.h"

#if defined(__set_task_state)
#define __stp_set_task_state(tsk, state_value)		\
	__set_task_state((tsk), (state_value))
#else
#define __stp_set_task_state(tsk, state_value)		\
	do { (tsk)->state = (state_value); } while (0)
#endif

// For now, disable the task_work_queue on non-RT kernels.
// XXX: Need to decide correct CONFIG_* to check for,
// sticking with the more conservative option for now.
#ifdef CONFIG_PREEMPT_RT_FULL
#define STP_UTRACE_USE_TASK_WORK_QUEUE
#endif
/* If STP_UTRACE_USE_TASK_WORK_QUEUE is defined, reporting work
   will be delayed out of tracepoint context. This must be done
   to prevent illegal operations on realtime kernels. */

/*
 * Per-thread structure private to utrace implementation.
 * If task_struct.utrace_flags is nonzero, task_struct.utrace
 * has always been allocated first.  Once allocated, it is
 * never freed until free_task().
 *
 * The common event reporting loops are done by the task making the
 * report without ever taking any locks.  To facilitate this, the two
 * lists @attached and @attaching work together for smooth asynchronous
 * attaching with low overhead.  Modifying either list requires @lock.
 * The @attaching list can be modified any time while holding @lock.
 * New engines being attached always go on this list.
 *
 * The @attached list is what the task itself uses for its reporting
 * loops.  When the task itself is not quiescent, it can use the
 * @attached list without taking any lock.  Nobody may modify the list
 * when the task is not quiescent.  When it is quiescent, that means
 * that it won't run again without taking @lock itself before using
 * the list.
 *
 * At each place where we know the task is quiescent (or it's current),
 * while holding @lock, we call splice_attaching(), below.  This moves
 * the @attaching list members on to the end of the @attached list.
 * Since this happens at the start of any reporting pass, any new
 * engines attached asynchronously go on the stable @attached list
 * in time to have their callbacks seen.
 */
struct utrace {
	spinlock_t lock;
	struct list_head attached, attaching;

	struct utrace_engine *reporting;

	enum utrace_resume_action resume:UTRACE_RESUME_BITS;
	unsigned int vfork_stop:1; /* need utrace_stop() before vfork wait */
	unsigned int death:1;	/* in utrace_report_death() now */
	unsigned int reap:1;	/* release_task() has run */
	unsigned int pending_attach:1;	/* need splice_attaching() */

	/* We need the '*_work_added' variables to be atomic so they
	 * can be modified without locking the utrace struct. This is
	 * typically done in atomic context where we can't grab the
	 * lock. */
	atomic_t resume_work_added;	/* called task_work_add() on
					 * 'resume_work' */
	atomic_t report_work_added;	/* called task_work_add() on
					 * 'report_work' */

	unsigned long utrace_flags;

	struct hlist_node hlist;       /* task_utrace_table linkage */
	struct task_struct *task;

	struct task_work resume_work;
	struct task_work report_work;
};

#define TASK_UTRACE_HASH_BITS 5
#define TASK_UTRACE_TABLE_SIZE (1 << TASK_UTRACE_HASH_BITS)

static struct hlist_head task_utrace_table[TASK_UTRACE_TABLE_SIZE];
//DEFINE_MUTEX(task_utrace_mutex);      /* Protects task_utrace_table */
static STP_DEFINE_SPINLOCK(task_utrace_lock); /* Protects task_utrace_table */

/* Temporary storage for utrace structures that utrace_exit() has removed
   from task_utrace_table prior to cleaning them up: */
static HLIST_HEAD(utrace_cleanup_list);
static DEFINE_SPINLOCK(utrace_cleanup_lock); /* Protects utrace_cleanup_list */

/* Tracepoint reporting is delayed using task_work structures stored
   in the following linked list: */
static LIST_HEAD(__stp_utrace_task_work_list);
static STP_DEFINE_SPINLOCK(__stp_utrace_task_work_list_lock);
struct __stp_utrace_task_work { /* NB: about 216 bytes, 18 per page: */
	struct list_head list;
	struct task_struct *task;
	struct utrace *utrace;
	union {
		struct pt_regs regs;
		void *data;
	};
	struct task_work work;
};

/* Because kernel-rt does not allow kmalloc(GFP_ATOMIC) in tracepoint
   context, we roll our own nano-allocator.

   TODO: UTRACE_TASK_WORK_POOL_SIZE can be specified on the Systemtap
   command line. Experiment to find the best default value. */
#ifndef UTRACE_TASK_WORK_POOL_SIZE
#define UTRACE_TASK_WORK_POOL_SIZE 288
#endif
static DECLARE_BITMAP(__stp_utrace_task_work_pool_bitmap, UTRACE_TASK_WORK_POOL_SIZE);
static struct __stp_utrace_task_work __stp_utrace_task_work_pool[UTRACE_TASK_WORK_POOL_SIZE];
static STP_DEFINE_SPINLOCK(__stp_utrace_task_work_pool_lock);
/* TODO: It should be possible to stick with kmalloc()/kfree() on
   regular non-realtime kernels. Perhaps control which allocator gets
   used with an #ifdef? */

static struct task_work *
__stp_utrace_alloc_task_work(struct utrace *utrace, void *data)
{
	struct __stp_utrace_task_work *utrace_work = NULL;
	unsigned long flags;
	unsigned long next_fit;

	stp_spin_lock_irqsave(&__stp_utrace_task_work_pool_lock, flags);
	/* Find zero bit in __stp_utrace_task_work_pool_bitmap: */
	next_fit = bitmap_find_next_zero_area
		(__stp_utrace_task_work_pool_bitmap,
		 UTRACE_TASK_WORK_POOL_SIZE, 0, 1, 0);
	if (next_fit < UTRACE_TASK_WORK_POOL_SIZE) {
		bitmap_set(__stp_utrace_task_work_pool_bitmap, next_fit, 1);
		utrace_work = &__stp_utrace_task_work_pool[next_fit];
		memset(utrace_work, 0, sizeof(*utrace_work));
	}
	stp_spin_unlock_irqrestore(&__stp_utrace_task_work_pool_lock, flags);
	if (utrace_work == NULL) {
		_stp_error("Unable to allocate space for task_work");
		return NULL;
	}

	utrace_work->task = current;
	utrace_work->utrace = utrace;
	utrace_work->data = data;

	// Insert new item onto list.
	stp_spin_lock_irqsave(&__stp_utrace_task_work_list_lock, flags);
	list_add(&utrace_work->list, &__stp_utrace_task_work_list);
	stp_spin_unlock_irqrestore(&__stp_utrace_task_work_list_lock, flags);

	return &utrace_work->work;
}

static void __stp_utrace_save_regs(struct task_work *work, struct pt_regs *regs)
{
	struct __stp_utrace_task_work *utrace_work = \
		container_of(work, struct __stp_utrace_task_work, work);
	memcpy(&utrace_work->regs, regs, sizeof(struct pt_regs));
}

static void __stp_utrace_free_task_work_from_pool
(struct __stp_utrace_task_work *utrace_work)
{
	unsigned long flags;
	unsigned long pool_index;

	stp_spin_lock_irqsave(&__stp_utrace_task_work_pool_lock, flags);
	pool_index = /* pointer arithmetic */ utrace_work - __stp_utrace_task_work_pool;
	BUG_ON(pool_index > UTRACE_TASK_WORK_POOL_SIZE);
	bitmap_clear(__stp_utrace_task_work_pool_bitmap, pool_index, 1);
	stp_spin_unlock_irqrestore(&__stp_utrace_task_work_pool_lock, flags);
}

static void __stp_utrace_free_task_work(struct task_work *work)
{
	struct __stp_utrace_task_work *utrace_work, *node;
	unsigned long flags;
	unsigned long pool_index;

	utrace_work = container_of(work, struct __stp_utrace_task_work, work);

	// Remove the item from the list.
	stp_spin_lock_irqsave(&__stp_utrace_task_work_list_lock, flags);
	list_for_each_entry(node, &__stp_utrace_task_work_list, list) {
		if (utrace_work == node) {
			list_del(&utrace_work->list);
			break;
		}
	}
	stp_spin_unlock_irqrestore(&__stp_utrace_task_work_list_lock, flags);

	// Actually free the data.
	__stp_utrace_free_task_work_from_pool(utrace_work);
}

static struct kmem_cache *utrace_cachep;
static struct kmem_cache *utrace_engine_cachep;
static const struct utrace_engine_ops utrace_detached_ops; /* forward decl */

static void utrace_report_clone(void *cb_data __attribute__ ((unused)),
				struct task_struct *task,
				struct task_struct *child);
static void utrace_report_death(void *cb_data __attribute__ ((unused)),
				struct task_struct *task);
static void utrace_report_syscall_entry(void *cb_data __attribute__ ((unused)),
					struct pt_regs *regs, long id);
static void utrace_report_syscall_exit(void *cb_data __attribute__ ((unused)),
				       struct pt_regs *regs, long ret);

static void utrace_report_exec(void *cb_data __attribute__ ((unused)),
			       struct task_struct *task,
			       pid_t old_pid __attribute__((unused)),
			       struct linux_binprm *bprm __attribute__ ((unused)));

#define __UTRACE_UNREGISTERED	0
#define __UTRACE_REGISTERED	1
static atomic_t utrace_state = ATOMIC_INIT(__UTRACE_UNREGISTERED);

// If wake_up_state() is exported, use it.
#if defined(STAPCONF_WAKE_UP_STATE_EXPORTED)
#define stp_wake_up_state wake_up_state
// Otherwise, try to use try_to_wake_up(). The wake_up_state()
// function is just a wrapper around try_to_wake_up().
#elif defined(STAPCONF_TRY_TO_WAKE_UP_EXPORTED)
static inline int stp_wake_up_state(struct task_struct *p, unsigned int state)
{
	return try_to_wake_up(p, state, 0);
}
// Otherwise, we'll have to look up wake_up_state() with kallsyms.
#else
typedef typeof(&wake_up_state) wake_up_state_fn;
#define stp_wake_up_state (* (wake_up_state_fn)kallsyms_wake_up_state)
#endif

#if !defined(STAPCONF_SIGNAL_WAKE_UP_STATE_EXPORTED)
// Sigh. On kernel's without signal_wake_up_state(), there is no
// declaration to use in 'typeof(&signal_wake_up_state)'. So, we'll
// provide one here.
void signal_wake_up_state(struct task_struct *t, unsigned int state);

// First typedef from the original decl, then #define as typecasted call.
typedef typeof(&signal_wake_up_state) signal_wake_up_state_fn;
#define signal_wake_up_state (* (signal_wake_up_state_fn)kallsyms_signal_wake_up_state)
#endif

#if !defined(STAPCONF_SIGNAL_WAKE_UP_EXPORTED)
// First typedef from the original decl, then #define as typecasted call.
typedef typeof(&signal_wake_up) signal_wake_up_fn;
#define signal_wake_up (* (signal_wake_up_fn)kallsyms_signal_wake_up)
#endif

#if !defined(STAPCONF___LOCK_TASK_SIGHAND_EXPORTED)
// First typedef from the original decl, then #define as typecasted call.
typedef typeof(&__lock_task_sighand) __lock_task_sighand_fn;
#define __lock_task_sighand (* (__lock_task_sighand_fn)kallsyms___lock_task_sighand)

/*
 * __lock_task_sighand() is called from the inline function
 * 'lock_task_sighand'. Since the real inline function won't know
 * anything about our '#define' above, we have to have our own version
 * of the inline function.  Sigh.
 */
static inline struct sighand_struct *
stp_lock_task_sighand(struct task_struct *tsk, unsigned long *flags)
{
	struct sighand_struct *ret;

	ret = __lock_task_sighand(tsk, flags);
	(void)__cond_lock(&tsk->sighand->siglock, ret);
	return ret;
}
#else
#define stp_lock_task_sighand lock_task_sighand
#endif


/*
 * Our internal version of signal_wake_up()/signal_wake_up_state()
 * that handles the functions existing and being exported.
 */
static inline void
stp_signal_wake_up(struct task_struct *t, bool resume)
{
#if defined(STAPCONF_SIGNAL_WAKE_UP_STATE_EXPORTED)
    signal_wake_up_state(t, resume ? TASK_WAKEKILL : 0);
#elif defined(STAPCONF_SIGNAL_WAKE_UP_EXPORTED)
    signal_wake_up(t, resume);
#else
    if (kallsyms_signal_wake_up_state) {
	signal_wake_up_state(t, resume ? TASK_WAKEKILL : 0);
    }
    else if (kallsyms_signal_wake_up) {
	signal_wake_up(t, resume);
    }
#endif
}


static int utrace_init(void)
{
	int i;
	int rc = -1;
        static char kmem_cache1_name[50];
        static char kmem_cache2_name[50];

	if (unlikely(stp_task_work_init() != 0))
		goto error;

	/* initialize the list heads */
	for (i = 0; i < TASK_UTRACE_TABLE_SIZE; i++) {
		INIT_HLIST_HEAD(&task_utrace_table[i]);
	}

#if !defined(STAPCONF_TRY_TO_WAKE_UP_EXPORTED) \
    && !defined(STAPCONF_WAKE_UP_STATE_EXPORTED)
	kallsyms_wake_up_state = (void *)kallsyms_lookup_name("wake_up_state");
        if (kallsyms_wake_up_state == NULL) {
		_stp_error("Can't resolve wake_up_state!");
		goto error;
        }
#endif
#if !defined(STAPCONF_SIGNAL_WAKE_UP_STATE_EXPORTED)
	/* The signal_wake_up_state() function (which replaces
	 * signal_wake_up() in newer kernels) isn't exported. Look up
	 * that function address. */
        kallsyms_signal_wake_up_state = (void *)kallsyms_lookup_name("signal_wake_up_state");
#endif
#if !defined(STAPCONF_SIGNAL_WAKE_UP_EXPORTED)
	/* The signal_wake_up() function isn't exported. Look up that
	 * function address. */
        kallsyms_signal_wake_up = (void *)kallsyms_lookup_name("signal_wake_up");
#endif
#if (!defined(STAPCONF_SIGNAL_WAKE_UP_STATE_EXPORTED) \
     && !defined(STAPCONF_SIGNAL_WAKE_UP_EXPORTED))
        if (kallsyms_signal_wake_up_state == NULL
	    && kallsyms_signal_wake_up == NULL) {
		_stp_error("Can't resolve signal_wake_up_state or signal_wake_up!");
		goto error;
        }
#endif
#if !defined(STAPCONF___LOCK_TASK_SIGHAND_EXPORTED)
	/* The __lock_task_sighand() function isn't exported. Look up
	 * that function address. */
        kallsyms___lock_task_sighand = (void *)kallsyms_lookup_name("__lock_task_sighand");
        if (kallsyms___lock_task_sighand == NULL) {
		_stp_error("Can't resolve __lock_task_sighand!");
		goto error;
        }
#endif

        /* PR14781: avoid kmem_cache naming collisions (detected by CONFIG_DEBUG_VM)
           by plopping a non-conflicting token - in this case the address of a 
           locally relevant variable - into the names. */
        snprintf(kmem_cache1_name, sizeof(kmem_cache1_name),
                 "utrace_%lx", (unsigned long) (& utrace_cachep));
	utrace_cachep = kmem_cache_create(kmem_cache1_name, 
                                          sizeof(struct utrace),
                                          0, 0, NULL);
	if (unlikely(!utrace_cachep))
		goto error;

        snprintf(kmem_cache2_name, sizeof(kmem_cache2_name),
                 "utrace_engine_%lx", (unsigned long) (& utrace_engine_cachep));
	utrace_engine_cachep = kmem_cache_create(kmem_cache2_name, 
                                                 sizeof(struct utrace_engine),
                                                 0, 0, NULL);
	if (unlikely(!utrace_engine_cachep))
		goto error;

	rc = STP_TRACE_REGISTER(sched_process_fork, utrace_report_clone);
	if (unlikely(rc != 0)) {
		_stp_error("register_trace_sched_process_fork failed: %d", rc);
		goto error;
	}
	rc = STP_TRACE_REGISTER(sched_process_exit, utrace_report_death);
	if (unlikely(rc != 0)) {
		_stp_error("register_trace_sched_process_exit failed: %d", rc);
		goto error2;
	}
	rc = STP_TRACE_REGISTER(sys_enter, utrace_report_syscall_entry);
	if (unlikely(rc != 0)) {
		_stp_error("register_trace_sys_enter failed: %d", rc);
		goto error3;
	}
	rc = STP_TRACE_REGISTER(sys_exit, utrace_report_syscall_exit);
	if (unlikely(rc != 0)) {
		_stp_error("register_trace_sys_exit failed: %d", rc);
		goto error4;
	}

	rc = STP_TRACE_REGISTER(sched_process_exec, utrace_report_exec);
	if (unlikely(rc != 0)) {
		_stp_error("register_sched_process_exec failed: %d", rc);
		goto error5;
	}

	atomic_set(&utrace_state, __UTRACE_REGISTERED);
	return 0;

error5:
	STP_TRACE_UNREGISTER(sys_exit, utrace_report_syscall_exit);
error4:
	STP_TRACE_UNREGISTER(sys_enter, utrace_report_syscall_entry);
error3:
	STP_TRACE_UNREGISTER(sched_process_exit, utrace_report_death);
error2:
	STP_TRACE_UNREGISTER(sched_process_fork, utrace_report_clone);
	tracepoint_synchronize_unregister();
error:
	if (utrace_cachep) {
		kmem_cache_destroy(utrace_cachep);
		utrace_cachep = NULL;
	}
	if (utrace_engine_cachep) {
		kmem_cache_destroy(utrace_engine_cachep);
		utrace_engine_cachep = NULL;
	}
	return rc;
}

static void utrace_cleanup(struct utrace *utrace);

static int utrace_exit(void)
{
	int i;
	struct utrace *utrace;
	struct hlist_head *head;
	struct hlist_node *node, *node2;
	struct __stp_utrace_task_work *task_node, *task_node2;
	unsigned long flags;

#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d - entry\n", __FUNCTION__, __LINE__);
#endif
	/* Note that we are being paranoid
	 * here. stap_stop_task_finder() should have already called
	 * utrace_shutdown(). */
	utrace_shutdown();
	stp_task_work_exit();

	/* After utrace_shutdown() and stp_task_work_exit() (and the
	 * code in stap_stop_task_finder()), we're *sure* there are no
	 * tracepoint probes or task work items running or scheduled
	 * to be run. So, now would be a great time to actually free
	 * everything. */

	/* Since utrace->lock is an ordinary spinlock, we cannot
           grab it within a task_utrace_lock section. Likewise,
           on kernel-RT we can no longer do memory management in
           a raw_spinlock section. So we dump utrace structures
           to a separate hlist before freeing them: */
#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d - freeing task-specific\n", __FUNCTION__, __LINE__);
#endif
	spin_lock(&utrace_cleanup_lock);
	stp_spin_lock(&task_utrace_lock);
	for (i = 0; i < TASK_UTRACE_TABLE_SIZE; i++) {
		head = &task_utrace_table[i];
		stap_hlist_for_each_entry_safe(utrace, node, node2, head,
					       hlist) {
			hlist_del(&utrace->hlist);
			INIT_HLIST_NODE(&utrace->hlist);
			hlist_add_head(&utrace->hlist, &utrace_cleanup_list);
		}
	}
	stp_spin_unlock(&task_utrace_lock);

	stap_hlist_for_each_entry_safe(utrace, node, node2, &utrace_cleanup_list,
				       hlist) {
		hlist_del(&utrace->hlist);
		utrace_cleanup(utrace);
	}
	spin_unlock(&utrace_cleanup_lock);

	/* Likewise, free any task_work_list item(s). */
	stp_spin_lock_irqsave(&__stp_utrace_task_work_list_lock, flags);
	list_for_each_entry_safe(task_node, task_node2, &__stp_utrace_task_work_list, list) {
		// Remove the item from the list, then free it.
		list_del(&task_node->list);
		__stp_utrace_free_task_work_from_pool(task_node);
	}
	stp_spin_unlock_irqrestore(&__stp_utrace_task_work_list_lock, flags);

	if (utrace_cachep) {
		kmem_cache_destroy(utrace_cachep);
		utrace_cachep = NULL;
	}
	if (utrace_engine_cachep) {
		kmem_cache_destroy(utrace_engine_cachep);
		utrace_engine_cachep = NULL;
	}

#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d - exit\n", __FUNCTION__, __LINE__);
#endif
	return 0;
}

/*
 * stp_task_notify_resume() is our version of
 * set_notify_resume(). When called, the task_work infrastructure will
 * cause utrace_resume() to get called.
 */
static void
stp_task_notify_resume(struct task_struct *target, struct utrace *utrace)
{
	if (atomic_add_unless(&utrace->resume_work_added, 1, 1)) {
		int rc = stp_task_work_add(target, &utrace->resume_work);
		if (rc != 0) {
			atomic_set(&utrace->report_work_added, 0);

			/* stp_task_work_add() returns -ESRCH if the
			 * task has already passed
			 * exit_task_work(). Just ignore this
			 * error. */
			if (rc != -ESRCH) {
				printk(KERN_ERR
				       "%s:%d - task_work_add() returned %d\n",
				       __FUNCTION__, __LINE__, rc);
			}
		}
	}
}

/*
 * Clean up everything associated with @task.utrace.
 *
 * This routine must be called under the utrace_cleanup_lock.
 */
static void utrace_cleanup(struct utrace *utrace)
{
	struct utrace_engine *engine, *next;

	/* We used to require task_utrace_lock here, but none of the
	 * actual cleanup work seems to need it. */
	lockdep_assert_held(&utrace_cleanup_lock);

	/* Free engines associated with the struct utrace, starting
	 * with the 'attached' list then doing the 'attaching' list. */
	spin_lock(&utrace->lock);
	list_for_each_entry_safe(engine, next, &utrace->attached, entry) {
#ifdef STP_TF_DEBUG
	    printk(KERN_ERR "%s:%d - removing engine\n",
		   __FUNCTION__, __LINE__);
#endif
	    list_del_init(&engine->entry);
	    /* FIXME: hmm, should this be utrace_engine_put()? */
	    kmem_cache_free(utrace_engine_cachep, engine);
	}
	list_for_each_entry_safe(engine, next, &utrace->attaching, entry) {
	    list_del(&engine->entry);
	    kmem_cache_free(utrace_engine_cachep, engine);
	}
	spin_unlock(&utrace->lock);

	/* Note that we don't need to worry about
	 * utrace->resume_work_added and utrace->report_work_added,
	 * since those were cancelled by
	 * utrace_cancel_all_task_work(). */

	/* Free the struct utrace itself. */
#ifdef STP_TF_DEBUG
	memset(utrace, 0, sizeof(struct utrace));
#endif
	kmem_cache_free(utrace_cachep, utrace);
#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d exit\n", __FUNCTION__, __LINE__);
#endif
}

static void utrace_resume(struct task_work *work);
static void utrace_report_work(struct task_work *work);
static void utrace_syscall_entry_work(struct task_work *work);
static void utrace_syscall_exit_work(struct task_work *work);
static void utrace_exec_work(struct task_work *work);
static void utrace_clone_work(struct task_work *work);

/* MACROS for utrace_cancel_all_task_work(): */
#ifdef STP_TF_DEBUG
#define CANCEL_IF_WORK(added_field, task_work_func)                     \
        if (atomic_add_unless(&added_field,                             \
                              -1, 0)) {                                 \
                if (stp_task_work_cancel(utrace->task,                  \
                                         &task_work_func) == NULL)      \
                        printk(KERN_ERR "%s:%d - task_work_cancel() failed for %s? task %p, %d, %s\n", \
                               __FUNCTION__, __LINE__, #task_work_func, \
                               utrace->task, utrace->task->tgid,        \
                               (utrace->task->comm                      \
                                ? utrace->task->comm                    \
                                : "UNKNOWN"));                          \
        }
#else
#define CANCEL_IF_WORK(added_field, task_work_func)                     \
        if (atomic_add_unless(&added_field,                             \
                              -1, 0)) {                                 \
                stp_task_work_cancel(utrace->task,                      \
                                     &task_work_func);                  \
        }
#endif

static void utrace_cancel_all_task_work(void)
{
	int i;
	struct utrace *utrace;
	struct hlist_head *head;
	struct hlist_node *utrace_node;
	struct __stp_utrace_task_work *task_node;
	unsigned long flags;

	/* Cancel any pending utrace task_work item(s). */
	stp_spin_lock(&task_utrace_lock);
	for (i = 0; i < TASK_UTRACE_TABLE_SIZE; i++) {
		head = &task_utrace_table[i];
		stap_hlist_for_each_entry(utrace, utrace_node, head, hlist) {
			CANCEL_IF_WORK(utrace->resume_work_added, utrace_resume);
			CANCEL_IF_WORK(utrace->report_work_added, utrace_report_work);
		}
	}
	stp_spin_unlock(&task_utrace_lock);

	/* Cancel any pending task_work_list item(s). */
	stp_spin_lock_irqsave(&__stp_utrace_task_work_list_lock, flags);
	list_for_each_entry(task_node, &__stp_utrace_task_work_list, list) {
		stp_task_work_cancel(task_node->task, task_node->work.func);
	}
	stp_spin_unlock_irqrestore(&__stp_utrace_task_work_list_lock, flags);
}

static void utrace_shutdown(void)
{
	if (atomic_read(&utrace_state) != __UTRACE_REGISTERED)
		return;
	atomic_set(&utrace_state, __UTRACE_UNREGISTERED);

#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d entry\n", __FUNCTION__, __LINE__);
#endif
	/* Unregister all the tracepoint probes. */
	STP_TRACE_UNREGISTER(sched_process_exec, utrace_report_exec);
	STP_TRACE_UNREGISTER(sched_process_fork, utrace_report_clone);
	STP_TRACE_UNREGISTER(sched_process_exit, utrace_report_death);
	STP_TRACE_UNREGISTER(sys_enter, utrace_report_syscall_entry);
	STP_TRACE_UNREGISTER(sys_exit, utrace_report_syscall_exit);

	/* When tracepoint_synchronize_unregister() returns, all
	 * currently executing tracepoint probes will be finished. */
	tracepoint_synchronize_unregister();

	/* We'd like to wait here until all currrently executing
	 * task_work items are finished (by calling
	 * stp_task_work_exit()). But we can't call
	 * stp_task_work_exit() yet, since the task_finder hasn't
	 * canceled its task_work items yet.
	 *
	 * So, we'll just cancel all our task_work items here.
	 */
	utrace_cancel_all_task_work();
}

/*
 * This routine must be called under the task_utrace_lock.
 */
static struct utrace *__task_utrace_struct(struct task_struct *task)
{
	struct hlist_head *head;
	struct hlist_node *node;
	struct utrace *utrace;

	lockdep_assert_held(&task_utrace_lock);
	head = &task_utrace_table[hash_ptr(task, TASK_UTRACE_HASH_BITS)];
	stap_hlist_for_each_entry(utrace, node, head, hlist) {
		if (utrace->task == task)
			return utrace;
	}
	return NULL;
}

/*
 * Set up @task.utrace for the first time.  We can have races
 * between two utrace_attach_task() calls here.  The task_lock()
 * governs installing the new pointer.  If another one got in first,
 * we just punt the new one we allocated.
 *
 * This returns false only in case of a memory allocation failure.
 */
static bool utrace_task_alloc(struct task_struct *task)
{
	struct utrace *utrace = kmem_cache_zalloc(utrace_cachep,
						  STP_ALLOC_FLAGS);
	struct utrace *u;

	if (unlikely(!utrace))
		return false;
	spin_lock_init(&utrace->lock);
	INIT_LIST_HEAD(&utrace->attached);
	INIT_LIST_HEAD(&utrace->attaching);
	utrace->resume = UTRACE_RESUME;
	utrace->task = task;
	stp_init_task_work(&utrace->resume_work, &utrace_resume);
	stp_init_task_work(&utrace->report_work, &utrace_report_work);
	atomic_set(&utrace->resume_work_added, 0);
	atomic_set(&utrace->report_work_added, 0);

	stp_spin_lock(&task_utrace_lock);
	u = __task_utrace_struct(task);
	if (u == NULL) {
		hlist_add_head(&utrace->hlist,
			       &task_utrace_table[hash_ptr(task, TASK_UTRACE_HASH_BITS)]);
	}
	else {
#ifdef STP_TF_DEBUG
		memset(utrace, 0, sizeof(struct utrace));
#endif
		kmem_cache_free(utrace_cachep, utrace);
	}
	stp_spin_unlock(&task_utrace_lock);

	return true;
}

/* macros for utrace_free(): */
#define FREE_IF_WORK(added_field, task_work_func)                       \
        if (atomic_add_unless(&added_field, -1, 0)) {                   \
                if ((stp_task_work_cancel(utrace->task, &task_work_func) \
                     == NULL)                                           \
                    && (utrace->task->flags & ~PF_EXITING)              \
                    && (utrace->task->exit_state == 0))                 \
                        printk(KERN_ERR "%s:%d * task_work_cancel() failed for %s? task %p, %d, %s, 0x%lx 0x%x\n", \
                               __FUNCTION__, __LINE__, #task_work_func, \
                               utrace->task, utrace->task->tgid,        \
                               (utrace->task->comm ? utrace->task->comm \
                                : "UNKNOWN"), utrace->task->state, utrace->task->exit_state); \
        }

/*
 * Correctly free a @utrace structure.
 *
 * Originally, this function was called via tracehook_free_task() from
 * free_task() when @task is being deallocated. But free_task() has no
 * tracepoint we can easily hook.
 */
static void utrace_free(struct utrace *utrace)
{
	if (unlikely(!utrace))
		return;

	/* Remove this utrace from the mapping list of tasks to
	 * struct utrace. */
	stp_spin_lock(&task_utrace_lock);
	hlist_del(&utrace->hlist);
	stp_spin_unlock(&task_utrace_lock);

	/* Free the utrace struct. */
#ifdef STP_TF_DEBUG
	spin_lock(&utrace->lock);
	if (unlikely(utrace->reporting)
	    || unlikely(!list_empty(&utrace->attached))
	    || unlikely(!list_empty(&utrace->attaching)))
		printk(KERN_ERR "%s:%d - reporting? %p, attached empty %d, attaching empty %d\n",
		       __FUNCTION__, __LINE__, utrace->reporting,
		       list_empty(&utrace->attached),
		       list_empty(&utrace->attaching));
	spin_unlock(&utrace->lock);
#endif

	FREE_IF_WORK(utrace->resume_work_added, utrace_resume);
	FREE_IF_WORK(utrace->report_work_added, utrace_report_work);

#ifdef STP_TF_DEBUG
	memset(utrace, 0, sizeof(struct utrace));
#endif
	kmem_cache_free(utrace_cachep, utrace);
}

static struct utrace *task_utrace_struct(struct task_struct *task)
{
	struct utrace *utrace;

	stp_spin_lock(&task_utrace_lock);
	utrace = __task_utrace_struct(task);
	stp_spin_unlock(&task_utrace_lock);
	return utrace;
}

/*
 * This is called when the task is safely quiescent, i.e. it won't consult
 * utrace->attached without the lock.  Move any engines attached
 * asynchronously from @utrace->attaching onto the @utrace->attached list.
 */
static void splice_attaching(struct utrace *utrace)
{
	lockdep_assert_held(&utrace->lock);
	list_splice_tail_init(&utrace->attaching, &utrace->attached);
	utrace->pending_attach = 0;
}

/*
 * This is the exported function used by the utrace_engine_put() inline.
 */
static void __utrace_engine_release(struct kref *kref)
{
	struct utrace_engine *engine = container_of(kref, struct utrace_engine,
						    kref);
	BUG_ON(!list_empty(&engine->entry));
	if (engine->release)
		(*engine->release)(engine->data);
	kmem_cache_free(utrace_engine_cachep, engine);
}

static bool engine_matches(struct utrace_engine *engine, int flags,
			   const struct utrace_engine_ops *ops, void *data)
{
	if ((flags & UTRACE_ATTACH_MATCH_OPS) && engine->ops != ops)
		return false;
	if ((flags & UTRACE_ATTACH_MATCH_DATA) && engine->data != data)
		return false;
	return engine->ops && engine->ops != &utrace_detached_ops;
}

static struct utrace_engine *find_matching_engine(
	struct utrace *utrace, int flags,
	const struct utrace_engine_ops *ops, void *data)
{
	struct utrace_engine *engine;
	list_for_each_entry(engine, &utrace->attached, entry)
		if (engine_matches(engine, flags, ops, data))
			return engine;
	list_for_each_entry(engine, &utrace->attaching, entry)
		if (engine_matches(engine, flags, ops, data))
			return engine;
	return NULL;
}

/*
 * Enqueue @engine, or maybe don't if UTRACE_ATTACH_EXCLUSIVE.
 */
static int utrace_add_engine(struct task_struct *target,
			     struct utrace *utrace,
			     struct utrace_engine *engine,
			     int flags,
			     const struct utrace_engine_ops *ops,
			     void *data)
{
	int ret;

	spin_lock(&utrace->lock);

	ret = -EEXIST;
	if ((flags & UTRACE_ATTACH_EXCLUSIVE) &&
	     unlikely(find_matching_engine(utrace, flags, ops, data)))
		goto unlock;

	/*
	 * In case we had no engines before, make sure that
	 * utrace_flags is not zero. Since we did unlock+lock
	 * at least once after utrace_task_alloc() installed
	 * ->utrace, we have the necessary barrier which pairs
	 * with rmb() in task_utrace_struct().
	 */
	ret = -ESRCH;
	/* FIXME: Hmm, no reap in the brave new world... */
	if (!utrace->utrace_flags) {
		utrace->utrace_flags = UTRACE_EVENT(REAP);
		/*
		 * If we race with tracehook_prepare_release_task()
		 * make sure that either it sees utrace_flags != 0
		 * or we see exit_state == EXIT_DEAD.
		 */
		smp_mb();
		if (unlikely(target->exit_state == EXIT_DEAD)) {
			utrace->utrace_flags = 0;
			goto unlock;
		}
	}

	/*
	 * Put the new engine on the pending ->attaching list.
	 * Make sure it gets onto the ->attached list by the next
	 * time it's examined.  Setting ->pending_attach ensures
	 * that start_report() takes the lock and splices the lists
	 * before the next new reporting pass.
	 *
	 * When target == current, it would be safe just to call
	 * splice_attaching() right here.  But if we're inside a
	 * callback, that would mean the new engine also gets
	 * notified about the event that precipitated its own
	 * creation.  This is not what the user wants.
	 */
	list_add_tail(&engine->entry, &utrace->attaching);
	utrace->pending_attach = 1;
	utrace_engine_get(engine);
	ret = 0;
unlock:
	spin_unlock(&utrace->lock);

	return ret;
}

/**
 * utrace_attach_task - attach new engine, or look up an attached engine
 * @target:	thread to attach to
 * @flags:	flag bits combined with OR, see below
 * @ops:	callback table for new engine
 * @data:	engine private data pointer
 *
 * The caller must ensure that the @target thread does not get freed,
 * i.e. hold a ref or be its parent.  It is always safe to call this
 * on @current, or on the @child pointer in a @report_clone callback.
 *
 * UTRACE_ATTACH_CREATE:
 * Create a new engine.  If %UTRACE_ATTACH_CREATE is not specified, you
 * only look up an existing engine already attached to the thread.
 *
 * *** FIXME: needed??? ***
 * UTRACE_ATTACH_EXCLUSIVE:
 * Attempting to attach a second (matching) engine fails with -%EEXIST.
 *
 * UTRACE_ATTACH_MATCH_OPS: Only consider engines matching @ops.
 * UTRACE_ATTACH_MATCH_DATA: Only consider engines matching @data.
 *
 * *** FIXME: need exclusive processing??? ***
 * Calls with neither %UTRACE_ATTACH_MATCH_OPS nor %UTRACE_ATTACH_MATCH_DATA
 * match the first among any engines attached to @target.  That means that
 * %UTRACE_ATTACH_EXCLUSIVE in such a call fails with -%EEXIST if there
 * are any engines on @target at all.
 */
static struct utrace_engine *utrace_attach_task(
	struct task_struct *target, int flags,
	const struct utrace_engine_ops *ops, void *data)
{
	struct utrace *utrace = task_utrace_struct(target);
	struct utrace_engine *engine;
	int ret;

#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d - target %p, utrace %p\n", __FUNCTION__, __LINE__,
	       target, utrace);
#endif

	if (!(flags & UTRACE_ATTACH_CREATE)) {
		if (unlikely(!utrace))
			return ERR_PTR(-ENOENT);
		spin_lock(&utrace->lock);
		engine = find_matching_engine(utrace, flags, ops, data);
		if (engine)
			utrace_engine_get(engine);
		spin_unlock(&utrace->lock);
		return engine ?: ERR_PTR(-ENOENT);
	}

	if (unlikely(!ops) || unlikely(ops == &utrace_detached_ops))
		return ERR_PTR(-EINVAL);

	if (unlikely(target->flags & PF_KTHREAD))
		/*
		 * Silly kernel, utrace is for users!
		 */
		return ERR_PTR(-EPERM);

	if (!utrace) {
		if (unlikely(!utrace_task_alloc(target)))
			return ERR_PTR(-ENOMEM);
		utrace = task_utrace_struct(target);
	}

	engine = kmem_cache_alloc(utrace_engine_cachep, STP_ALLOC_FLAGS);
	if (unlikely(!engine))
		return ERR_PTR(-ENOMEM);

	/*
	 * Initialize the new engine structure.  It starts out with one ref
	 * to return.  utrace_add_engine() adds another for being attached.
	 */
	kref_init(&engine->kref);
	engine->flags = 0;
	engine->ops = ops;
	engine->data = data;
	engine->release = ops->release;

	ret = utrace_add_engine(target, utrace, engine, flags, ops, data);

	if (unlikely(ret)) {
		kmem_cache_free(utrace_engine_cachep, engine);
		engine = ERR_PTR(ret);
	}

	return engine;
}

/*
 * When an engine is detached, the target thread may still see it and
 * make callbacks until it quiesces.  We install a special ops vector
 * with these two callbacks.  When the target thread quiesces, it can
 * safely free the engine itself.  For any event we will always get
 * the report_quiesce() callback first, so we only need this one
 * pointer to be set.  The only exception is report_reap(), so we
 * supply that callback too.
 */
static u32 utrace_detached_quiesce(u32 action, struct utrace_engine *engine,
				   unsigned long event)
{
	return UTRACE_DETACH;
}

static void utrace_detached_reap(struct utrace_engine *engine,
				 struct task_struct *task)
{
}

static const struct utrace_engine_ops utrace_detached_ops = {
	.report_quiesce = &utrace_detached_quiesce,
	.report_reap = &utrace_detached_reap
};

/*
 * The caller has to hold a ref on the engine.  If the attached flag is
 * true (all but utrace_barrier() calls), the engine is supposed to be
 * attached.  If the attached flag is false (utrace_barrier() only),
 * then return -ERESTARTSYS for an engine marked for detach but not yet
 * fully detached.  The task pointer can be invalid if the engine is
 * detached.
 *
 * Get the utrace lock for the target task.
 * Returns the struct if locked, or ERR_PTR(-errno).
 *
 * This has to be robust against races with:
 *	utrace_control(target, UTRACE_DETACH) calls
 *	UTRACE_DETACH after reports
 *	utrace_report_death
 *	utrace_release_task
 */
static struct utrace *get_utrace_lock(struct task_struct *target,
				      struct utrace_engine *engine,
				      bool attached)
	__acquires(utrace->lock)
{
	struct utrace *utrace;

	rcu_read_lock();

	/*
	 * If this engine was already detached, bail out before we look at
	 * the task_struct pointer at all.  If it's detached after this
	 * check, then RCU is still keeping this task_struct pointer valid.
	 *
	 * The ops pointer is NULL when the engine is fully detached.
	 * It's &utrace_detached_ops when it's marked detached but still
	 * on the list.  In the latter case, utrace_barrier() still works,
	 * since the target might be in the middle of an old callback.
	 */
	if (unlikely(!engine->ops)) {
		rcu_read_unlock();
		return ERR_PTR(-ESRCH);
	}

	if (unlikely(engine->ops == &utrace_detached_ops)) {
		rcu_read_unlock();
		return attached ? ERR_PTR(-ESRCH) : ERR_PTR(-ERESTARTSYS);
	}

	utrace = task_utrace_struct(target);
	spin_lock(&utrace->lock);
	if (unlikely(utrace->reap) || unlikely(!engine->ops) ||
	    unlikely(engine->ops == &utrace_detached_ops)) {
		/*
		 * By the time we got the utrace lock,
		 * it had been reaped or detached already.
		 */
		spin_unlock(&utrace->lock);
		utrace = ERR_PTR(-ESRCH);
		if (!attached && engine->ops == &utrace_detached_ops)
			utrace = ERR_PTR(-ERESTARTSYS);
	}
	rcu_read_unlock();

	return utrace;
}

/*
 * Now that we don't hold any locks, run through any
 * detached engines and free their references.  Each
 * engine had one implicit ref while it was attached.
 */
static void put_detached_list(struct list_head *list)
{
	struct utrace_engine *engine, *next;
	list_for_each_entry_safe(engine, next, list, entry) {
		list_del_init(&engine->entry);
		utrace_engine_put(engine);
	}
}

/*
 * We use an extra bit in utrace_engine.flags past the event bits,
 * to record whether the engine is keeping the target thread stopped.
 *
 * This bit is set in task_struct.utrace_flags whenever it is set in any
 * engine's flags.  Only utrace_reset() resets it in utrace_flags.
 */
#define ENGINE_STOP		(1UL << _UTRACE_NEVENTS)

static void mark_engine_wants_stop(struct utrace *utrace,
				   struct utrace_engine *engine)
{
	engine->flags |= ENGINE_STOP;
	utrace->utrace_flags |= ENGINE_STOP;
}

static void clear_engine_wants_stop(struct utrace_engine *engine)
{
	engine->flags &= ~ENGINE_STOP;
}

static bool engine_wants_stop(struct utrace_engine *engine)
{
	return (engine->flags & ENGINE_STOP) != 0;
}

/**
 * utrace_set_events - choose which event reports a tracing engine gets
 * @target:		thread to affect
 * @engine:		attached engine to affect
 * @events:		new event mask
 *
 * This changes the set of events for which @engine wants callbacks made.
 *
 * This fails with -%EALREADY and does nothing if you try to clear
 * %UTRACE_EVENT(%DEATH) when the @report_death callback may already have
 * begun, or if you try to newly set %UTRACE_EVENT(%DEATH) or
 * %UTRACE_EVENT(%QUIESCE) when @target is already dead or dying.
 *
 * This fails with -%ESRCH if you try to clear %UTRACE_EVENT(%REAP) when
 * the @report_reap callback may already have begun, or when @target has
 * already been detached, including forcible detach on reaping.
 *
 * If @target was stopped before the call, then after a successful call,
 * no event callbacks not requested in @events will be made; if
 * %UTRACE_EVENT(%QUIESCE) is included in @events, then a
 * @report_quiesce callback will be made when @target resumes.
 *
 * If @target was not stopped and @events excludes some bits that were
 * set before, this can return -%EINPROGRESS to indicate that @target
 * may have been making some callback to @engine.  When this returns
 * zero, you can be sure that no event callbacks you've disabled in
 * @events can be made.  If @events only sets new bits that were not set
 * before on @engine, then -%EINPROGRESS will never be returned.
 *
 * To synchronize after an -%EINPROGRESS return, see utrace_barrier().
 *
 * When @target is @current, -%EINPROGRESS is not returned.  But note
 * that a newly-created engine will not receive any callbacks related to
 * an event notification already in progress.  This call enables @events
 * callbacks to be made as soon as @engine becomes eligible for any
 * callbacks, see utrace_attach_task().
 *
 * These rules provide for coherent synchronization based on %UTRACE_STOP,
 * even when %SIGKILL is breaking its normal simple rules.
 */
static int utrace_set_events(struct task_struct *target,
			     struct utrace_engine *engine,
			     unsigned long events)
{
	struct utrace *utrace;
	unsigned long old_flags, old_utrace_flags;
	int ret = -EALREADY;

	/*
	 * We just ignore the internal bit, so callers can use
	 * engine->flags to seed bitwise ops for our argument.
	 */
	events &= ~ENGINE_STOP;

	utrace = get_utrace_lock(target, engine, true);
	if (unlikely(IS_ERR(utrace)))
		return PTR_ERR(utrace);

	old_utrace_flags = utrace->utrace_flags;
	old_flags = engine->flags & ~ENGINE_STOP;

	/*
	 * If utrace_report_death() is already progress now,
	 * it's too late to clear the death event bits.
	 */
	if (target->exit_state &&
	    (((events & ~old_flags) & _UTRACE_DEATH_EVENTS) ||
	     (utrace->death &&
	      ((old_flags & ~events) & _UTRACE_DEATH_EVENTS)) ||
	     (utrace->reap && ((old_flags & ~events) & UTRACE_EVENT(REAP)))))
		goto unlock;

	/*
	 * When setting these flags, it's essential that we really
	 * synchronize with exit_notify().  They cannot be set after
	 * exit_notify() takes the tasklist_lock.  By holding the read
	 * lock here while setting the flags, we ensure that the calls
	 * to tracehook_notify_death() and tracehook_report_death() will
	 * see the new flags.  This ensures that utrace_release_task()
	 * knows positively that utrace_report_death() will be called or
	 * that it won't.
	 */
	if ((events & ~old_flags) & _UTRACE_DEATH_EVENTS) {
		/* FIXME: we can't get the tasklist_lock (since it
		 * isn't exported).  Plus, there is no more tracehook
		 * in exit_notify().  So, we'll ignore this for now
		 * and just assume that the lock on utrace is
		 * enough.  */
		//read_lock(&tasklist_lock);
		if (unlikely(target->exit_state)) {
			//read_unlock(&tasklist_lock);
			goto unlock;
		}
		utrace->utrace_flags |= events;
		//read_unlock(&tasklist_lock);
	}

	engine->flags = events | (engine->flags & ENGINE_STOP);
	utrace->utrace_flags |= events;

	ret = 0;
	if ((old_flags & ~events) && target != current &&
	    !task_is_stopped_or_traced(target) && !target->exit_state) {
		/*
		 * This barrier ensures that our engine->flags changes
		 * have hit before we examine utrace->reporting,
		 * pairing with the barrier in start_callback().  If
		 * @target has not yet hit finish_callback() to clear
		 * utrace->reporting, we might be in the middle of a
		 * callback to @engine.
		 */
		smp_mb();
		if (utrace->reporting == engine)
			ret = -EINPROGRESS;
	}
unlock:
	spin_unlock(&utrace->lock);

	return ret;
}

/*
 * Asynchronously mark an engine as being detached.
 *
 * This must work while the target thread races with us doing
 * start_callback(), defined below.  It uses smp_rmb() between checking
 * @engine->flags and using @engine->ops.  Here we change @engine->ops
 * first, then use smp_wmb() before changing @engine->flags.  This ensures
 * it can check the old flags before using the old ops, or check the old
 * flags before using the new ops, or check the new flags before using the
 * new ops, but can never check the new flags before using the old ops.
 * Hence, utrace_detached_ops might be used with any old flags in place.
 * It has report_quiesce() and report_reap() callbacks to handle all cases.
 */
static void mark_engine_detached(struct utrace_engine *engine)
{
	engine->ops = &utrace_detached_ops;
	smp_wmb();
	engine->flags = UTRACE_EVENT(QUIESCE);
}

/*
 * Get @target to stop and return true if it is already stopped now.
 * If we return false, it will make some event callback soonish.
 * Called with @utrace locked.
 */
static bool utrace_do_stop(struct task_struct *target, struct utrace *utrace)
{
	if (task_is_stopped(target)) {
		/*
		 * Stopped is considered quiescent; when it wakes up, it will
		 * go through utrace_finish_stop() before doing anything else.
		 */
		spin_lock_irq(&target->sighand->siglock);
		if (likely(task_is_stopped(target)))
			__stp_set_task_state(target, TASK_TRACED);
		spin_unlock_irq(&target->sighand->siglock);
	} else if (utrace->resume > UTRACE_REPORT) {
		utrace->resume = UTRACE_REPORT;
		stp_task_notify_resume(target, utrace);
	}

	return task_is_traced(target);
}

/*
 * If the target is not dead it should not be in tracing
 * stop any more.  Wake it unless it's in job control stop.
 */
static void utrace_wakeup(struct task_struct *target, struct utrace *utrace)
{
	lockdep_assert_held(&utrace->lock);
	spin_lock_irq(&target->sighand->siglock);
	if (target->signal->flags & SIGNAL_STOP_STOPPED ||
	    target->signal->group_stop_count)
		target->state = TASK_STOPPED;
	else
		stp_wake_up_state(target, __TASK_TRACED);
	spin_unlock_irq(&target->sighand->siglock);
}

/*
 * This is called when there might be some detached engines on the list or
 * some stale bits in @task->utrace_flags.  Clean them up and recompute the
 * flags.  Returns true if we're now fully detached.
 *
 * Called with @utrace->lock held, returns with it released.
 * After this returns, @utrace might be freed if everything detached.
 */
static bool utrace_reset(struct task_struct *task, struct utrace *utrace)
	__releases(utrace->lock)
{
	struct utrace_engine *engine, *next;
	unsigned long flags = 0;
	LIST_HEAD(detached);

	splice_attaching(utrace);

	/*
	 * Update the set of events of interest from the union
	 * of the interests of the remaining tracing engines.
	 * For any engine marked detached, remove it from the list.
	 * We'll collect them on the detached list.
	 */
	list_for_each_entry_safe(engine, next, &utrace->attached, entry) {
		if (engine->ops == &utrace_detached_ops) {
			engine->ops = NULL;
			list_move(&engine->entry, &detached);
		} else {
			flags |= engine->flags | UTRACE_EVENT(REAP);
		}
	}

	if (task->exit_state) {
		/*
		 * Once it's already dead, we never install any flags
		 * except REAP.  When ->exit_state is set and events
		 * like DEATH are not set, then they never can be set.
		 * This ensures that utrace_release_task() knows
		 * positively that utrace_report_death() can never run.
		 */
		BUG_ON(utrace->death);
		flags &= UTRACE_EVENT(REAP);
	}

	if (!flags) {
		/*
		 * No more engines, cleared out the utrace.
		 */
		utrace->resume = UTRACE_RESUME;
	}

	/*
	 * If no more engines want it stopped, wake it up.
	 */
	if (task_is_traced(task) && !(flags & ENGINE_STOP))
		utrace_wakeup(task, utrace);

	/*
	 * In theory spin_lock() doesn't imply rcu_read_lock().
	 * Once we clear ->utrace_flags this task_struct can go away
	 * because tracehook_prepare_release_task() path does not take
	 * utrace->lock when ->utrace_flags == 0.
	 */
	rcu_read_lock();
	utrace->utrace_flags = flags;
	rcu_read_unlock();
	spin_unlock(&utrace->lock);

	put_detached_list(&detached);

	return !flags;
}

static void utrace_finish_stop(void)
{
	/*
	 * If we were task_is_traced() and then SIGKILL'ed, make
	 * sure we do nothing until the tracer drops utrace->lock.
	 */
	if (unlikely(__fatal_signal_pending(current))) {
		struct utrace *utrace = task_utrace_struct(current);
		spin_lock(&utrace->lock);
		spin_unlock(&utrace->lock);
	}
}

/*
 * Perform %UTRACE_STOP, i.e. block in TASK_TRACED until woken up.
 * @task == current, @utrace == current->utrace, which is not locked.
 * Return true if we were woken up by SIGKILL even though some utrace
 * engine may still want us to stay stopped.
 */
static void utrace_stop(struct task_struct *task, struct utrace *utrace,
			enum utrace_resume_action action)
{
relock:
	spin_lock(&utrace->lock);

	if (action < utrace->resume) {
		/*
		 * Ensure a reporting pass when we're resumed.
		 */
		utrace->resume = action;
		stp_task_notify_resume(task, utrace);
		if (action == UTRACE_INTERRUPT)
			set_thread_flag(TIF_SIGPENDING);
	}

	/*
	 * If the ENGINE_STOP bit is clear in utrace_flags, that means
	 * utrace_reset() ran after we processed some UTRACE_STOP return
	 * values from callbacks to get here.  If all engines have detached
	 * or resumed us, we don't stop.  This check doesn't require
	 * siglock, but it should follow the interrupt/report bookkeeping
	 * steps (this can matter for UTRACE_RESUME but not UTRACE_DETACH).
	 */
	if (unlikely(!(utrace->utrace_flags & ENGINE_STOP))) {
		utrace_reset(task, utrace);
		if (utrace->utrace_flags & ENGINE_STOP)
			goto relock;
		return;
	}

	/*
	 * The siglock protects us against signals.  As well as SIGKILL
	 * waking us up, we must synchronize with the signal bookkeeping
	 * for stop signals and SIGCONT.
	 */
	spin_lock_irq(&task->sighand->siglock);

	if (unlikely(__fatal_signal_pending(task))) {
		spin_unlock_irq(&task->sighand->siglock);
		spin_unlock(&utrace->lock);
		return;
	}

	__set_current_state(TASK_TRACED);

	/*
	 * If there is a group stop in progress,
	 * we must participate in the bookkeeping.
	 */
	if (unlikely(task->signal->group_stop_count) &&
			!--task->signal->group_stop_count)
		task->signal->flags = SIGNAL_STOP_STOPPED;

	spin_unlock_irq(&task->sighand->siglock);
	spin_unlock(&utrace->lock);

	schedule();

	utrace_finish_stop();

	/*
	 * While in TASK_TRACED, we were considered "frozen enough".
	 * Now that we woke up, it's crucial if we're supposed to be
	 * frozen that we freeze now before running anything substantial.
	 */
	try_to_freeze();

	/*
	 * While we were in TASK_TRACED, complete_signal() considered
	 * us "uninterested" in signal wakeups.  Now make sure our
	 * TIF_SIGPENDING state is correct for normal running.
	 */
	spin_lock_irq(&task->sighand->siglock);
	recalc_sigpending();
	spin_unlock_irq(&task->sighand->siglock);
}

/*
 * Called by release_task() with @reap set to true.
 * Called by utrace_report_death() with @reap set to false.
 * On reap, make report_reap callbacks and clean out @utrace
 * unless still making callbacks.  On death, update bookkeeping
 * and handle the reap work if release_task() came in first.
 */
static void utrace_maybe_reap(struct task_struct *target, struct utrace *utrace,
			      bool reap)
{
	struct utrace_engine *engine, *next;
	struct list_head attached;

	spin_lock(&utrace->lock);

	if (reap) {
		/*
		 * If the target will do some final callbacks but hasn't
		 * finished them yet, we know because it clears these event
		 * bits after it's done.  Instead of cleaning up here and
		 * requiring utrace_report_death() to cope with it, we
		 * delay the REAP report and the teardown until after the
		 * target finishes its death reports.
		 */
		utrace->reap = 1;

		if (utrace->utrace_flags & _UTRACE_DEATH_EVENTS) {
			spin_unlock(&utrace->lock);
			return;
		}
	} else {
		/*
		 * After we unlock with this flag clear, any competing
		 * utrace_control/utrace_set_events calls know that we've
		 * finished our callbacks and any detach bookkeeping.
		 */
		utrace->death = 0;

		if (!utrace->reap) {
			/*
			 * We're just dead, not reaped yet.  This will
			 * reset @target->utrace_flags so the later call
			 * with @reap set won't hit the check above.
			 */
			utrace_reset(target, utrace);
			return;
		}
	}

	/*
	 * utrace_add_engine() checks ->utrace_flags != 0.  Since
	 * @utrace->reap is set, nobody can set or clear UTRACE_EVENT(REAP)
	 * in @engine->flags or change @engine->ops and nobody can change
	 * @utrace->attached after we drop the lock.
	 */
	utrace->utrace_flags = 0;

	/*
	 * We clear out @utrace->attached before we drop the lock so
	 * that find_matching_engine() can't come across any old engine
	 * while we are busy tearing it down.
	 */
	list_replace_init(&utrace->attached, &attached);
	list_splice_tail_init(&utrace->attaching, &attached);

	spin_unlock(&utrace->lock);

	list_for_each_entry_safe(engine, next, &attached, entry) {
		if (engine->flags & UTRACE_EVENT(REAP))
			engine->ops->report_reap(engine, target);

		engine->ops = NULL;
		engine->flags = 0;
		list_del_init(&engine->entry);

		utrace_engine_put(engine);
	}
}

/*
 * You can't do anything to a dead task but detach it.
 * If release_task() has been called, you can't do that.
 *
 * On the exit path, DEATH and QUIESCE event bits are set only
 * before utrace_report_death() has taken the lock.  At that point,
 * the death report will come soon, so disallow detach until it's
 * done.  This prevents us from racing with it detaching itself.
 *
 * Called only when @target->exit_state is nonzero.
 */
static inline int utrace_control_dead(struct task_struct *target,
				      struct utrace *utrace,
				      enum utrace_resume_action action)
{
	lockdep_assert_held(&utrace->lock);

	if (action != UTRACE_DETACH || unlikely(utrace->reap))
		return -ESRCH;

	if (unlikely(utrace->death))
		/*
		 * We have already started the death report.  We can't
		 * prevent the report_death and report_reap callbacks,
		 * so tell the caller they will happen.
		 */
		return -EALREADY;

	return 0;
}

/**
 * utrace_control - control a thread being traced by a tracing engine
 * @target:		thread to affect
 * @engine:		attached engine to affect
 * @action:		&enum utrace_resume_action for thread to do
 *
 * This is how a tracing engine asks a traced thread to do something.
 * This call is controlled by the @action argument, which has the
 * same meaning as the &enum utrace_resume_action value returned by
 * event reporting callbacks.
 *
 * If @target is already dead (@target->exit_state nonzero),
 * all actions except %UTRACE_DETACH fail with -%ESRCH.
 *
 * The following sections describe each option for the @action argument.
 *
 * UTRACE_DETACH:
 *
 * After this, the @engine data structure is no longer accessible,
 * and the thread might be reaped.  The thread will start running
 * again if it was stopped and no longer has any attached engines
 * that want it stopped.
 *
 * If the @report_reap callback may already have begun, this fails
 * with -%ESRCH.  If the @report_death callback may already have
 * begun, this fails with -%EALREADY.
 *
 * If @target is not already stopped, then a callback to this engine
 * might be in progress or about to start on another CPU.  If so,
 * then this returns -%EINPROGRESS; the detach happens as soon as
 * the pending callback is finished.  To synchronize after an
 * -%EINPROGRESS return, see utrace_barrier().
 *
 * If @target is properly stopped before utrace_control() is called,
 * then after successful return it's guaranteed that no more callbacks
 * to the @engine->ops vector will be made.
 *
 * The only exception is %SIGKILL (and exec or group-exit by another
 * thread in the group), which can cause asynchronous @report_death
 * and/or @report_reap callbacks even when %UTRACE_STOP was used.
 * (In that event, this fails with -%ESRCH or -%EALREADY, see above.)
 *
 * UTRACE_STOP:
 *
 * This asks that @target stop running.  This returns 0 only if
 * @target is already stopped, either for tracing or for job
 * control.  Then @target will remain stopped until another
 * utrace_control() call is made on @engine; @target can be woken
 * only by %SIGKILL (or equivalent, such as exec or termination by
 * another thread in the same thread group).
 *
 * This returns -%EINPROGRESS if @target is not already stopped.
 * Then the effect is like %UTRACE_REPORT.  A @report_quiesce
 * callback will be made soon.  Your callback can
 * then return %UTRACE_STOP to keep @target stopped.
 *
 * This does not interrupt system calls in progress, including ones
 * that sleep for a long time.
 *
 * UTRACE_RESUME:
 *
 * Just let @target continue running normally, reversing the effect
 * of a previous %UTRACE_STOP.  If another engine is keeping @target
 * stopped, then it remains stopped until all engines let it resume.
 * If @target was not stopped, this has no effect.
 *
 * UTRACE_REPORT:
 *
 * This is like %UTRACE_RESUME, but also ensures that there will be
 * a @report_quiesce callback made soon.  If
 * @target had been stopped, then there will be a callback before it
 * resumes running normally.  If another engine is keeping @target
 * stopped, then there might be no callbacks until all engines let
 * it resume.
 *
 * Since this is meaningless unless @report_quiesce callbacks will
 * be made, it returns -%EINVAL if @engine lacks %UTRACE_EVENT(%QUIESCE).
 *
 * UTRACE_INTERRUPT:
 *
 * This is like %UTRACE_REPORT, but ensures that @target will make a
 * callback before it resumes or delivers signals.  If @target was in
 * a system call or about to enter one, work in progress will be
 * interrupted as if by %SIGSTOP.  If another engine is keeping
 * @target stopped, then there might be no callbacks until all engines
 * let it resume.
 */
static int utrace_control(struct task_struct *target,
			  struct utrace_engine *engine,
			  enum utrace_resume_action action)
{
	struct utrace *utrace;
	bool reset;
	int ret;

	if (unlikely(action >= UTRACE_RESUME_MAX)) {
		WARN(1, "invalid action argument to utrace_control()!");
		return -EINVAL;
	}

	/*
	 * This is a sanity check for a programming error in the caller.
	 * Their request can only work properly in all cases by relying on
	 * a follow-up callback, but they didn't set one up!  This check
	 * doesn't do locking, but it shouldn't matter.  The caller has to
	 * be synchronously sure the callback is set up to be operating the
	 * interface properly.
	 */
	if (action >= UTRACE_REPORT && action < UTRACE_RESUME &&
	    unlikely(!(engine->flags & UTRACE_EVENT(QUIESCE)))) {
		WARN(1, "utrace_control() with no QUIESCE callback in place!");
		return -EINVAL;
	}

	utrace = get_utrace_lock(target, engine, true);
	if (unlikely(IS_ERR(utrace)))
		return PTR_ERR(utrace);

	reset = task_is_traced(target);
	ret = 0;

	/*
	 * ->exit_state can change under us, this doesn't matter.
	 * We do not care about ->exit_state in fact, but we do
	 * care about ->reap and ->death. If either flag is set,
	 * we must also see ->exit_state != 0.
	 */
	if (unlikely(target->exit_state)) {
		ret = utrace_control_dead(target, utrace, action);
		if (ret) {
			spin_unlock(&utrace->lock);
			return ret;
		}
		reset = true;
	}

	switch (action) {
	case UTRACE_STOP:
		mark_engine_wants_stop(utrace, engine);
		if (!reset && !utrace_do_stop(target, utrace))
			ret = -EINPROGRESS;
		reset = false;
		break;

	case UTRACE_DETACH:
		if (engine_wants_stop(engine))
			utrace->utrace_flags &= ~ENGINE_STOP;
		mark_engine_detached(engine);
		reset = reset || utrace_do_stop(target, utrace);
		if (!reset) {
			/*
			 * As in utrace_set_events(), this barrier ensures
			 * that our engine->flags changes have hit before we
			 * examine utrace->reporting, pairing with the barrier
			 * in start_callback().  If @target has not yet hit
			 * finish_callback() to clear utrace->reporting, we
			 * might be in the middle of a callback to @engine.
			 */
			smp_mb();
			if (utrace->reporting == engine)
				ret = -EINPROGRESS;
		}
		break;

	case UTRACE_RESUME:
		clear_engine_wants_stop(engine);
		break;

	case UTRACE_REPORT:
		/*
		 * Make the thread call tracehook_notify_resume() soon.
		 * But don't bother if it's already been interrupted.
		 * In that case, utrace_get_signal() will be reporting soon.
		 */
		clear_engine_wants_stop(engine);
		if (action < utrace->resume) {
			utrace->resume = action;
			stp_task_notify_resume(target, utrace);
		}
		break;

	case UTRACE_INTERRUPT:
		/*
		 * Make the thread call tracehook_get_signal() soon.
		 */
		clear_engine_wants_stop(engine);
		if (utrace->resume == UTRACE_INTERRUPT)
			break;
		utrace->resume = UTRACE_INTERRUPT;

		/*
		 * If it's not already stopped, interrupt it now.  We need
		 * the siglock here in case it calls recalc_sigpending()
		 * and clears its own TIF_SIGPENDING.  By taking the lock,
		 * we've serialized any later recalc_sigpending() after our
		 * setting of utrace->resume to force it on.
		 */
		stp_task_notify_resume(target, utrace);
		if (reset) {
			/*
			 * This is really just to keep the invariant that
			 * TIF_SIGPENDING is set with UTRACE_INTERRUPT.
			 * When it's stopped, we know it's always going
			 * through utrace_get_signal() and will recalculate.
			 */
			set_tsk_thread_flag(target, TIF_SIGPENDING);
		} else {
			struct sighand_struct *sighand;
			unsigned long irqflags;
			sighand = stp_lock_task_sighand(target, &irqflags);
			if (likely(sighand)) {
				stp_signal_wake_up(target, 0);
				unlock_task_sighand(target, &irqflags);
			}
		}
		break;

	default:
		BUG();		/* We checked it on entry.  */
	}

	/*
	 * Let the thread resume running.  If it's not stopped now,
	 * there is nothing more we need to do.
	 */
	if (reset)
		utrace_reset(target, utrace);
	else
		spin_unlock(&utrace->lock);

	return ret;
}

/**
 * utrace_barrier - synchronize with simultaneous tracing callbacks
 * @target:		thread to affect
 * @engine:		engine to affect (can be detached)
 *
 * This blocks while @target might be in the midst of making a callback to
 * @engine.  It can be interrupted by signals and will return -%ERESTARTSYS.
 * A return value of zero means no callback from @target to @engine was
 * in progress.  Any effect of its return value (such as %UTRACE_STOP) has
 * already been applied to @engine.
 *
 * It's not necessary to keep the @target pointer alive for this call.
 * It's only necessary to hold a ref on @engine.  This will return
 * safely even if @target has been reaped and has no task refs.
 *
 * A successful return from utrace_barrier() guarantees its ordering
 * with respect to utrace_set_events() and utrace_control() calls.  If
 * @target was not properly stopped, event callbacks just disabled might
 * still be in progress; utrace_barrier() waits until there is no chance
 * an unwanted callback can be in progress.
 */
static int utrace_barrier(struct task_struct *target,
			  struct utrace_engine *engine)
{
	struct utrace *utrace;
	int ret = -ERESTARTSYS;

	if (unlikely(target == current))
		return 0;

	/* If we get here, we might call
	 * schedule_timeout_interruptible(), which sleeps. */
	might_sleep();
	do {
		utrace = get_utrace_lock(target, engine, false);
		if (unlikely(IS_ERR(utrace))) {
			ret = PTR_ERR(utrace);
			if (ret != -ERESTARTSYS)
				break;
		} else {
			/*
			 * All engine state changes are done while
			 * holding the lock, i.e. before we get here.
			 * Since we have the lock, we only need to
			 * worry about @target making a callback.
			 * When it has entered start_callback() but
			 * not yet gotten to finish_callback(), we
			 * will see utrace->reporting == @engine.
			 * When @target doesn't take the lock, it uses
			 * barriers to order setting utrace->reporting
			 * before it examines the engine state.
			 */
			if (utrace->reporting != engine)
				ret = 0;
			spin_unlock(&utrace->lock);
			if (!ret)
				break;
		}
		schedule_timeout_interruptible(1);
	} while (!signal_pending(current));

	return ret;
}

/*
 * This is local state used for reporting loops, perhaps optimized away.
 */
struct utrace_report {
	u32 result;
	enum utrace_resume_action action;
	enum utrace_resume_action resume_action;
	bool detaches;
	bool spurious;
};

#define INIT_REPORT(var)			\
	struct utrace_report var = {		\
		.action = UTRACE_RESUME,	\
		.resume_action = UTRACE_RESUME,	\
		.spurious = true 		\
	}

/*
 * We are now making the report, so clear the flag saying we need one.
 * When there is a new attach, ->pending_attach is set just so we will
 * know to do splice_attaching() here before the callback loop.
 */
static enum utrace_resume_action start_report(struct utrace *utrace)
{
	enum utrace_resume_action resume = utrace->resume;
	if (utrace->pending_attach ||
	    (resume > UTRACE_STOP && resume < UTRACE_RESUME)) {
		spin_lock(&utrace->lock);
		splice_attaching(utrace);
		resume = utrace->resume;
		if (resume > UTRACE_STOP)
			utrace->resume = UTRACE_RESUME;
		spin_unlock(&utrace->lock);
	}
	return resume;
}

static inline void finish_report_reset(struct task_struct *task,
				       struct utrace *utrace,
				       struct utrace_report *report)
{
	if (unlikely(report->spurious || report->detaches)) {
		spin_lock(&utrace->lock);
		if (utrace_reset(task, utrace))
			report->action = UTRACE_RESUME;
	}
}

/*
 * Complete a normal reporting pass, pairing with a start_report()
 * call.  This handles any UTRACE_DETACH or UTRACE_REPORT returns from
 * engine callbacks.  If @will_not_stop is true and any engine's last
 * callback used UTRACE_STOP, we do UTRACE_REPORT here to ensure we
 * stop before user mode.  If there were no callbacks made, it will
 * recompute @task->utrace_flags to avoid another false-positive.
 */
static void finish_report(struct task_struct *task, struct utrace *utrace,
			  struct utrace_report *report, bool will_not_stop)
{
	enum utrace_resume_action resume = report->action;

	if (resume == UTRACE_STOP)
		resume = will_not_stop ? UTRACE_REPORT : UTRACE_RESUME;

	if (resume < utrace->resume) {
		spin_lock(&utrace->lock);
		utrace->resume = resume;
		stp_task_notify_resume(task, utrace);
		if (resume == UTRACE_INTERRUPT)
			set_tsk_thread_flag(task, TIF_SIGPENDING);
		spin_unlock(&utrace->lock);
	}

	finish_report_reset(task, utrace, report);
}

static void finish_callback_report(struct task_struct *task,
				   struct utrace *utrace,
				   struct utrace_report *report,
				   struct utrace_engine *engine,
				   enum utrace_resume_action action)
{
	if (action == UTRACE_DETACH) {
		/*
		 * By holding the lock here, we make sure that
		 * utrace_barrier() (really get_utrace_lock()) sees the
		 * effect of this detach.  Otherwise utrace_barrier() could
		 * return 0 after this callback had returned UTRACE_DETACH.
		 * This way, a 0 return is an unambiguous indicator that any
		 * callback returning UTRACE_DETACH has indeed caused detach.
		 */
		spin_lock(&utrace->lock);
		engine->ops = &utrace_detached_ops;
		spin_unlock(&utrace->lock);
	}

	/*
	 * If utrace_control() was used, treat that like UTRACE_DETACH here.
	 */
	if (engine->ops == &utrace_detached_ops) {
		report->detaches = true;
		return;
	}

	if (action < report->action)
		report->action = action;

	if (action != UTRACE_STOP) {
		if (action < report->resume_action)
			report->resume_action = action;

		if (engine_wants_stop(engine)) {
			spin_lock(&utrace->lock);
			clear_engine_wants_stop(engine);
			spin_unlock(&utrace->lock);
		}

		return;
	}

	if (!engine_wants_stop(engine)) {
		spin_lock(&utrace->lock);
		/*
		 * If utrace_control() came in and detached us
		 * before we got the lock, we must not stop now.
		 */
		if (unlikely(engine->ops == &utrace_detached_ops))
			report->detaches = true;
		else
			mark_engine_wants_stop(utrace, engine);
		spin_unlock(&utrace->lock);
	}
}

/*
 * Apply the return value of one engine callback to @report.
 * Returns true if @engine detached and should not get any more callbacks.
 */
static bool finish_callback(struct task_struct *task, struct utrace *utrace,
			    struct utrace_report *report,
			    struct utrace_engine *engine,
			    u32 ret)
{
	report->result = ret & ~UTRACE_RESUME_MASK;
	finish_callback_report(task, utrace, report, engine,
			       utrace_resume_action(ret));

	/*
	 * Now that we have applied the effect of the return value,
	 * clear this so that utrace_barrier() can stop waiting.
	 * A subsequent utrace_control() can stop or resume @engine
	 * and know this was ordered after its callback's action.
	 *
	 * We don't need any barriers here because utrace_barrier()
	 * takes utrace->lock.  If we touched engine->flags above,
	 * the lock guaranteed this change was before utrace_barrier()
	 * examined utrace->reporting.
	 */
	utrace->reporting = NULL;

	/*
	 * We've just done an engine callback.  These are *not*
	 * allowed to sleep, unlike the original utrace (since
	 * tracepiont handlers aren't allowed to sleep).
	 */

	return engine->ops == &utrace_detached_ops;
}

/*
 * Start the callbacks for @engine to consider @event (a bit mask).
 * This makes the report_quiesce() callback first.  If @engine wants
 * a specific callback for @event, we return the ops vector to use.
 * If not, we return NULL.  The return value from the ops->callback
 * function called should be passed to finish_callback().
 */
static const struct utrace_engine_ops *start_callback(
	struct utrace *utrace, struct utrace_report *report,
	struct utrace_engine *engine, struct task_struct *task,
	unsigned long event)
{
	const struct utrace_engine_ops *ops;
	unsigned long want;

#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d - utrace %p, report %p, engine %p, task %p, event %ld\n",
	       __FUNCTION__, __LINE__, utrace, report, engine, task, event);
#endif

	/*
	 * This barrier ensures that we've set utrace->reporting before
	 * we examine engine->flags or engine->ops.  utrace_barrier()
	 * relies on this ordering to indicate that the effect of any
	 * utrace_control() and utrace_set_events() calls is in place
	 * by the time utrace->reporting can be seen to be NULL.
	 */
	utrace->reporting = engine;
	smp_mb();

	/*
	 * This pairs with the barrier in mark_engine_detached().
	 * It makes sure that we never see the old ops vector with
	 * the new flags, in case the original vector had no report_quiesce.
	 */
	want = engine->flags;
	smp_rmb();
	ops = engine->ops;

	if ((want & UTRACE_EVENT(QUIESCE)) || ops == &utrace_detached_ops) {
#ifdef STP_TF_DEBUG
		printk(KERN_ERR "%s:%d - quiescing, ops %p, ops->report_quiesce %p\n",
		       __FUNCTION__, __LINE__, ops,
		       (ops == NULL ? 0 : ops->report_quiesce));
#endif
		if (finish_callback(task, utrace, report, engine,
				    (*ops->report_quiesce)(report->action,
							   engine, event)))
			return NULL;

		if (!event) {
			/* We only got here to report QUIESCE */
			report->spurious = false;
			return NULL;
		}

		/*
		 * finish_callback() reset utrace->reporting after the
		 * quiesce callback.  Now we set it again (as above)
		 * before re-examining engine->flags, which could have
		 * been changed synchronously by ->report_quiesce or
		 * asynchronously by utrace_control() or utrace_set_events().
		 */
		utrace->reporting = engine;
		smp_mb();
		want = engine->flags;
	}

	if (want & ENGINE_STOP)
		report->action = UTRACE_STOP;

	if (want & event) {
		report->spurious = false;
		return ops;
	}

	utrace->reporting = NULL;
	return NULL;
}

/*
 * Do a normal reporting pass for engines interested in @event.
 * @callback is the name of the member in the ops vector, and remaining
 * args are the extras it takes after the standard three args.
 */
#define REPORT_CALLBACKS(rev, task, utrace, report, event, callback, ...)     \
	do {								      \
		struct utrace_engine *engine;				      \
		const struct utrace_engine_ops *ops;			      \
		list_for_each_entry##rev(engine, &utrace->attached, entry) {  \
			ops = start_callback(utrace, report, engine, task,    \
					     event);			      \
			if (!ops)					      \
				continue;				      \
			finish_callback(task, utrace, report, engine,	      \
					(*ops->callback)(__VA_ARGS__));	      \
		}							      \
	} while (0)
#define REPORT(task, utrace, report, event, callback, ...)		      \
	do {								      \
		start_report(utrace);					      \
		REPORT_CALLBACKS(, task, utrace, report, event, callback,     \
				 (report)->action, engine, ## __VA_ARGS__);   \
		finish_report(task, utrace, report, true);		      \
	} while (0)

/*
 * Called iff UTRACE_EVENT(EXEC) flag is set.
 */
static void utrace_report_exec(void *cb_data __attribute__ ((unused)),
			       struct task_struct *task,
			       pid_t old_pid __attribute__((unused)),
			       struct linux_binprm *bprm __attribute__ ((unused)))
{
#ifdef STP_UTRACE_USE_TASK_WORK_QUEUE
	// This is the version that's deferred with task_work_list.
	struct task_work *work;
	struct utrace *utrace;
	int rc;

	if (atomic_read(&utrace_state) != __UTRACE_REGISTERED)
		return;
	utrace = task_utrace_struct(task);

	if (!utrace || !(utrace->utrace_flags & UTRACE_EVENT(EXEC)))
		return;

#if 0
	if (!in_atomic() && !irqs_disabled())
		printk(KERN_ERR
		       "%s:%d - utrace_report_exec() in nonatomic context (%d, %d)\n",
		       __FUNCTION__, __LINE__, in_atomic(), irqs_disabled());
#endif

	/* Defer the report_exec work so it doesn't happen in atomic context: */
	work = __stp_utrace_alloc_task_work(utrace, (void *)NULL);
	if (work == NULL) {
		_stp_error("Unable to allocate space for task_work");
		return;
	}
	stp_init_task_work(work, &utrace_exec_work);
	rc = stp_task_work_add(task, work);
	// stp_task_work_add() returns -ESRCH if the task has already
	// passed exit_task_work(). Just ignore this error.
	if (rc != 0 && rc != -ESRCH) {
		printk(KERN_ERR "%s:%d - stp_task_work_add() returned %d\n",
		       __FUNCTION__, __LINE__, rc);
	}
#else
	// This is the original version.
	struct utrace *utrace;

	if (atomic_read(&utrace_state) != __UTRACE_REGISTERED)
		return;
	utrace = task_utrace_struct(task);

	if (utrace && utrace->utrace_flags & UTRACE_EVENT(EXEC)) {
		INIT_REPORT(report);

		/* FIXME: Hmm, can we get regs another way? */
		REPORT(task, utrace, &report, UTRACE_EVENT(EXEC),
		       report_exec, NULL, NULL, NULL /* regs */);
	}
#endif
}

static void utrace_exec_work(struct task_work *work)
{
	struct __stp_utrace_task_work *utrace_work = \
		container_of(work, struct __stp_utrace_task_work, work);
	struct utrace *utrace = utrace_work->utrace;
	struct task_struct *task = current;

	INIT_REPORT(report);

	might_sleep();

	/* FIXME: Hmm, can we get regs another way? */
	REPORT(task, utrace, &report, UTRACE_EVENT(EXEC),
	       report_exec, NULL, NULL, NULL /* regs */);
	__stp_utrace_free_task_work(work);

	/* Remember that this task_work_func is finished. */
	stp_task_work_func_done();
}

#if 0
static u32 do_report_syscall_entry(struct pt_regs *regs,
				   struct task_struct *task,
				   struct utrace *utrace,
				   struct utrace_report *report,
				   u32 resume_report)
{
	start_report(utrace);
	REPORT_CALLBACKS(_reverse, task, utrace, report,
			 UTRACE_EVENT(SYSCALL_ENTRY), report_syscall_entry,
			 resume_report | report->result | report->action,
			 engine, regs);
	finish_report(task, utrace, report, false);

	if (report->action != UTRACE_STOP)
		return 0;

	utrace_stop(task, utrace, report->resume_action);

	if (fatal_signal_pending(task)) {
		/*
		 * We are continuing despite UTRACE_STOP because of a
		 * SIGKILL.  Don't let the system call actually proceed.
		 */
		report->result = UTRACE_SYSCALL_ABORT;
	} else if (utrace->resume <= UTRACE_REPORT) {
		/*
		 * If we've been asked for another report after our stop,
		 * go back to report (and maybe stop) again before we run
		 * the system call.  The second (and later) reports are
		 * marked with the UTRACE_SYSCALL_RESUMED flag so that
		 * engines know this is a second report at the same
		 * entry.  This gives them the chance to examine the
		 * registers anew after they might have been changed
		 * while we were stopped.
		 */
		report->detaches = false;
		report->spurious = true;
		report->action = report->resume_action = UTRACE_RESUME;
		return UTRACE_SYSCALL_RESUMED;
	}

	return 0;
}
#endif

/*
 * Called iff UTRACE_EVENT(SYSCALL_ENTRY) flag is set.
 * Return true to prevent the system call.
 */
static void utrace_report_syscall_entry(void *cb_data __attribute__ ((unused)),
					struct pt_regs *regs, long id)
{
#ifdef STP_UTRACE_USE_TASK_WORK_QUEUE
	// This is the version that's deferred with task_work_list.
	struct task_struct *task = current;
	struct task_work *work;
	struct utrace *utrace;
	int rc;

	if (atomic_read(&utrace_state) != __UTRACE_REGISTERED)
		return;
	utrace = task_utrace_struct(task);

	/* FIXME: Is this 100% correct? */
	if (!utrace
	    || !(utrace->utrace_flags & (UTRACE_EVENT(SYSCALL_ENTRY)|ENGINE_STOP)))
		return;

#if 0
	if (!in_atomic() && !irqs_disabled())
		printk(KERN_ERR
		       "%s:%d - utrace_report_syscall_entry() in nonatomic context (%d, %d)\n",
		       __FUNCTION__, __LINE__, in_atomic(), irqs_disabled());
#endif

	/* Defer the report_syscall_entry work so it doesn't happen in atomic context: */
	work = __stp_utrace_alloc_task_work(utrace, NULL);
	__stp_utrace_save_regs(work, regs);
	if (work == NULL) {
		_stp_error("Unable to allocate space for task_work");
		return;
	}
	stp_init_task_work(work, &utrace_syscall_entry_work);
	rc = stp_task_work_add(task, work);
	// stp_task_work_add() returns -ESRCH if the task has already
	// passed exit_task_work(). Just ignore this error.
	if (rc != 0 && rc != -ESRCH) {
		printk(KERN_ERR "%s:%d - stp_task_work_add() returned %d\n",
		       __FUNCTION__, __LINE__, rc);
	}
#else
	// This is the original version.
	struct task_struct *task = current;
	struct utrace *utrace;

	if (atomic_read(&utrace_state) != __UTRACE_REGISTERED)
		return;
	utrace = task_utrace_struct(task);

	/* FIXME: Is this 100% correct? */
	if (utrace
	    && utrace->utrace_flags & (UTRACE_EVENT(SYSCALL_ENTRY)|ENGINE_STOP)) {
		INIT_REPORT(report);


		/* FIXME: Hmm, original utrace called probes in reverse
		 * order.  Needed here? */
		REPORT(task, utrace, &report, UTRACE_EVENT(SYSCALL_ENTRY),
		       report_syscall_entry, regs);
	}

#if 0
	INIT_REPORT(report);
	u32 resume_report = 0;

	do {
		resume_report = do_report_syscall_entry(regs, task, utrace,
							&report, resume_report);
	} while (resume_report);

	return utrace_syscall_action(report.result) == UTRACE_SYSCALL_ABORT;
#endif
#endif
}

static void utrace_syscall_entry_work(struct task_work *work)
{
	struct __stp_utrace_task_work *utrace_work = \
		container_of(work, struct __stp_utrace_task_work, work);
	struct utrace *utrace = utrace_work->utrace;
	struct task_struct *task = current;
	struct pt_regs *regs = &utrace_work->regs;

	INIT_REPORT(report);

	might_sleep();

	/* FIXME: Hmm, original utrace called probes in reverse
	 * order.  Needed here? */
	REPORT(task, utrace, &report, UTRACE_EVENT(SYSCALL_ENTRY),
	       report_syscall_entry, regs);
	__stp_utrace_free_task_work(work);

	/* Remember that this task_work_func is finished. */
	stp_task_work_func_done();
}

/*
 * Called iff UTRACE_EVENT(SYSCALL_EXIT) flag is set.
 */
static void utrace_report_syscall_exit(void *cb_data __attribute__ ((unused)),
				       struct pt_regs *regs, long ret)
{
#ifdef STP_UTRACE_USE_TASK_WORK_QUEUE
	// This is the version that's deferred with task_work_list.
	struct task_struct *task = current;
	struct task_work *work;
	struct utrace *utrace;
	int rc;

	if (atomic_read(&utrace_state) != __UTRACE_REGISTERED)
		return;
	utrace = task_utrace_struct(task);

	/* FIXME: Is this 100% correct? */
	if (!utrace
	    || !(utrace->utrace_flags & (UTRACE_EVENT(SYSCALL_EXIT)|ENGINE_STOP)))
		return;

#if 0
	if (!in_atomic() && !irqs_disabled())
		printk(KERN_ERR
		       "%s:%d - utrace_report_syscall_exit() in nonatomic context (%d, %d)\n",
		       __FUNCTION__, __LINE__, in_atomic(), irqs_disabled());
#endif

	/* Defer the report_syscall_exit work so it doesn't happen in atomic context: */
	work = __stp_utrace_alloc_task_work(utrace, NULL);
	__stp_utrace_save_regs(work, regs);
	if (work == NULL) {
		_stp_error("Unable to allocate space for task_work");
		return;
	}
	stp_init_task_work(work, &utrace_syscall_exit_work);
	rc = stp_task_work_add(task, work);
	// stp_task_work_add() returns -ESRCH if the task has already
	// passed exit_task_work(). Just ignore this error.
	if (rc != 0 && rc != -ESRCH) {
		printk(KERN_ERR "%s:%d - stp_task_work_add() returned %d\n",
		       __FUNCTION__, __LINE__, rc);
	}
#else
	// This is the original version.
	struct task_struct *task = current;
	struct utrace *utrace;

	if (atomic_read(&utrace_state) != __UTRACE_REGISTERED)
		return;
	utrace = task_utrace_struct(task);

	/* FIXME: Is this 100% correct? */
	if (utrace
	    && utrace->utrace_flags & (UTRACE_EVENT(SYSCALL_EXIT)|ENGINE_STOP)) {
		INIT_REPORT(report);

#ifdef STP_TF_DEBUG
		printk(KERN_ERR "%s:%d - task %p, utrace %p, utrace_flags 0x%lx\n",
		       __FUNCTION__, __LINE__, task, utrace,
		       utrace->utrace_flags);
#endif
		REPORT(task, utrace, &report, UTRACE_EVENT(SYSCALL_EXIT),
		       report_syscall_exit, regs);
	}
#endif
}

static void utrace_syscall_exit_work(struct task_work *work)
{
	struct __stp_utrace_task_work *utrace_work = \
		container_of(work, struct __stp_utrace_task_work, work);
	struct utrace *utrace = utrace_work->utrace;
	struct task_struct *task = current;
	struct pt_regs *regs = &utrace_work->regs;

	INIT_REPORT(report);

#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d - task %p, utrace %p, utrace_flags 0x%lx\n",
	       __FUNCTION__, __LINE__, task, utrace,
	       utrace->utrace_flags);
#endif
	might_sleep();
	__stp_utrace_free_task_work(work);

#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d - task %p, utrace %p, utrace_flags 0x%lx\n",
	       __FUNCTION__, __LINE__, task, utrace,
	       utrace->utrace_flags);
#endif

	REPORT(task, utrace, &report, UTRACE_EVENT(SYSCALL_EXIT),
	       report_syscall_exit, regs);

	/* Remember that this task_work_func is finished. */
	stp_task_work_func_done();
}

/*
 * Called iff UTRACE_EVENT(CLONE) flag is set.
 * This notification call blocks the wake_up_new_task call on the child.
 * So we must not quiesce here.  tracehook_report_clone_complete will do
 * a quiescence check momentarily.
 *
 * TODO: Need to review how tracehook_report_clone_complete() interacts
 * with the task_work_list report delaying.
 */
static void utrace_report_clone(void *cb_data __attribute__ ((unused)),
				struct task_struct *task,
				struct task_struct *child)
{
#ifdef STP_UTRACE_USE_TASK_WORK_QUEUE
	// This is the version that's deferred with task_work_list.
	struct task_work *work;
	struct utrace *utrace;
	int rc;

	if (atomic_read(&utrace_state) != __UTRACE_REGISTERED)
		return;
	utrace = task_utrace_struct(task);

#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d - parent %p, child %p, current %p\n",
	       __FUNCTION__, __LINE__, task, child, current);
#endif

	if (!utrace
	    || !(utrace->utrace_flags & UTRACE_EVENT(CLONE)))
		return;

	// TODO: Need to double-check the lifetime of struct child.

	/* Defer the report_clone work so it doesn't happen in atomic context: */
	work = __stp_utrace_alloc_task_work(utrace, (void *)child);
	if (work == NULL) {
		_stp_error("Unable to allocate space for task_work");
		return;
	}
	stp_init_task_work(work, &utrace_clone_work);
	rc = stp_task_work_add(task, work);
	// stp_task_work_add() returns -ESRCH if the task has already
	// passed exit_task_work(). Just ignore this error.
	if (rc != 0 && rc != -ESRCH) {
		printk(KERN_ERR "%s:%d - stp_task_work_add() returned %d\n",
		       __FUNCTION__, __LINE__, rc);
	}
#else
	// This is the original version.
	struct utrace *utrace;

	if (atomic_read(&utrace_state) != __UTRACE_REGISTERED)
		return;
	utrace = task_utrace_struct(task);

#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d - parent %p, child %p, current %p\n",
	       __FUNCTION__, __LINE__, task, child, current);
#endif

	if (utrace && utrace->utrace_flags & UTRACE_EVENT(CLONE)) {
		unsigned long clone_flags = 0;
		INIT_REPORT(report);

		/* FIXME: Figure out what the clone_flags were. For
		 * task_finder's purposes, all we need is CLONE_THREAD. */
		if (task->mm == child->mm)
			clone_flags |= CLONE_VM;
		if (task->fs == child->fs)
			clone_flags |= CLONE_FS;
		if (task->files == child->files)
			clone_flags |= CLONE_FILES;
		if (task->sighand == child->sighand)
			clone_flags |= CLONE_SIGHAND;

#if 0
#define CLONE_PTRACE	0x00002000	/* set if we want to let tracing continue on the child too */
#define CLONE_VFORK	0x00004000	/* set if the parent wants the child to wake it up on mm_release */
#define CLONE_PARENT	0x00008000	/* set if we want to have the same parent as the cloner */
#endif
		if (! thread_group_leader(child)) /* Same thread group? */
			clone_flags |= CLONE_THREAD;

#if 0
#define CLONE_NEWNS	0x00020000	/* New namespace group? */
#define CLONE_SYSVSEM	0x00040000	/* share system V SEM_UNDO semantics */
#define CLONE_SETTLS	0x00080000	/* create a new TLS for the child */
#define CLONE_PARENT_SETTID	0x00100000	/* set the TID in the parent */
#define CLONE_CHILD_CLEARTID	0x00200000	/* clear the TID in the child */
#define CLONE_DETACHED		0x00400000	/* Unused, ignored */
#define CLONE_UNTRACED		0x00800000	/* set if the tracing process can't force CLONE_PTRACE on this clone */
#define CLONE_CHILD_SETTID	0x01000000	/* set the TID in the child */
/* 0x02000000 was previously the unused CLONE_STOPPED (Start in stopped state)
   and is now available for re-use. */
#define CLONE_NEWUTS		0x04000000	/* New utsname group? */
#define CLONE_NEWIPC		0x08000000	/* New ipcs */
#define CLONE_NEWUSER		0x10000000	/* New user namespace */
#define CLONE_NEWPID		0x20000000	/* New pid namespace */
#define CLONE_NEWNET		0x40000000	/* New network namespace */
#define CLONE_IO		0x80000000	/* Clone io context */
#endif


		REPORT(task, utrace, &report, UTRACE_EVENT(CLONE),
		       report_clone, clone_flags, child);


#if 0
		/*
		 * For a vfork, we will go into an uninterruptible
		 * block waiting for the child.  We need UTRACE_STOP
		 * to happen before this, not after.  For CLONE_VFORK,
		 * utrace_finish_vfork() will be called.
		 */
		if (report.action == UTRACE_STOP
		    && (clone_flags & CLONE_VFORK)) {
			spin_lock(&utrace->lock);
			utrace->vfork_stop = 1;
			spin_unlock(&utrace->lock);
		}
#endif
	}
#endif
}

static void utrace_clone_work(struct task_work *work)
{
	struct __stp_utrace_task_work *utrace_work = \
		container_of(work, struct __stp_utrace_task_work, work);
	struct utrace *utrace = utrace_work->utrace;
	struct task_struct *task = current;
	struct task_struct *child = (struct task_struct *)utrace_work->data;

	unsigned long clone_flags = 0;
	INIT_REPORT(report);

	might_sleep();
	__stp_utrace_free_task_work(work);

	/* FIXME: Figure out what the clone_flags were. For
	 * task_finder's purposes, all we need is CLONE_THREAD. */
	if (task->mm == child->mm)
		clone_flags |= CLONE_VM;
	if (task->fs == child->fs)
		clone_flags |= CLONE_FS;
	if (task->files == child->files)
		clone_flags |= CLONE_FILES;
	if (task->sighand == child->sighand)
		clone_flags |= CLONE_SIGHAND;

#if 0
#define CLONE_PTRACE	0x00002000	/* set if we want to let tracing continue on the child too */
#define CLONE_VFORK	0x00004000	/* set if the parent wants the child to wake it up on mm_release */
#define CLONE_PARENT	0x00008000	/* set if we want to have the same parent as the cloner */
#endif
	if (! thread_group_leader(child)) /* Same thread group? */
		clone_flags |= CLONE_THREAD;

#if 0
#define CLONE_NEWNS	0x00020000	/* New namespace group? */
#define CLONE_SYSVSEM	0x00040000	/* share system V SEM_UNDO semantics */
#define CLONE_SETTLS	0x00080000	/* create a new TLS for the child */
#define CLONE_PARENT_SETTID	0x00100000	/* set the TID in the parent */
#define CLONE_CHILD_CLEARTID	0x00200000	/* clear the TID in the child */
#define CLONE_DETACHED		0x00400000	/* Unused, ignored */
#define CLONE_UNTRACED		0x00800000	/* set if the tracing process can't force CLONE_PTRACE on this clone */
#define CLONE_CHILD_SETTID	0x01000000	/* set the TID in the child */
/* 0x02000000 was previously the unused CLONE_STOPPED (Start in stopped state)
   and is now available for re-use. */
#define CLONE_NEWUTS		0x04000000	/* New utsname group? */
#define CLONE_NEWIPC		0x08000000	/* New ipcs */
#define CLONE_NEWUSER		0x10000000	/* New user namespace */
#define CLONE_NEWPID		0x20000000	/* New pid namespace */
#define CLONE_NEWNET		0x40000000	/* New network namespace */
#define CLONE_IO		0x80000000	/* Clone io context */
#endif


	REPORT(task, utrace, &report, UTRACE_EVENT(CLONE),
	       report_clone, clone_flags, child);


#if 0
	/*
	 * For a vfork, we will go into an uninterruptible
	 * block waiting for the child.  We need UTRACE_STOP
	 * to happen before this, not after.  For CLONE_VFORK,
	 * utrace_finish_vfork() will be called.
	 */
	if (report.action == UTRACE_STOP
	    && (clone_flags & CLONE_VFORK)) {
		spin_lock(&utrace->lock);
		utrace->vfork_stop = 1;
		spin_unlock(&utrace->lock);
	}
#endif

	/* Remember that this task_work_func is finished. */
	stp_task_work_func_done();
}

/*
 * We're called after utrace_report_clone() for a CLONE_VFORK.
 * If UTRACE_STOP was left from the clone report, we stop here.
 * After this, we'll enter the uninterruptible wait_for_completion()
 * waiting for the child.
 *
 * TODO: Is this an orphaned function? If it were actually called
 * anywhere, its timing relative to utrace_clone_work() would be
 * important.
 */
static void utrace_finish_vfork(struct task_struct *task)
{
	struct utrace *utrace = task_utrace_struct(task);

	if (utrace->vfork_stop) {
		spin_lock(&utrace->lock);
		utrace->vfork_stop = 0;
		spin_unlock(&utrace->lock);
		utrace_stop(task, utrace, UTRACE_RESUME); /* XXX */
	}
}

/*
 * Called iff UTRACE_EVENT(DEATH) or UTRACE_EVENT(QUIESCE) flag is set.
 *
 * It is always possible that we are racing with utrace_release_task here.
 * For this reason, utrace_release_task checks for the event bits that get
 * us here, and delays its cleanup for us to do.
 */
static void utrace_report_death(void *cb_data __attribute__ ((unused)),
				struct task_struct *task)
{
	struct utrace *utrace;
	INIT_REPORT(report);

	if (atomic_read(&utrace_state) != __UTRACE_REGISTERED)
		return;
	utrace = task_utrace_struct(task);

#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d - task %p, utrace %p, flags %lx\n", __FUNCTION__, __LINE__, task, utrace, utrace ? utrace->utrace_flags : 0);
#endif
	if (!utrace || !(utrace->utrace_flags & UTRACE_EVENT(DEATH)))
		return;

	/* This code is called from the 'sched_process_exit'
	 * tracepoint, which really corresponds more to UTRACE_EXIT
	 * (thread exit in progress) than to UTRACE_DEATH (thread has
	 * died).  But utrace_report_death() calls
	 * utrace_maybe_reap(), which does cleanup that we need.
	 *
	 * Because of this, 'exit_state' won't be set yet (as it would
	 * have been when the original utrace hit this code).
	 *
	 * BUG_ON(!task->exit_state);
	 */

	/*
	 * We are presently considered "quiescent"--which is accurate
	 * inasmuch as we won't run any more user instructions ever again.
	 * But for utrace_control and utrace_set_events to be robust, they
	 * must be sure whether or not we will run any more callbacks.  If
	 * a call comes in before we do, taking the lock here synchronizes
	 * us so we don't run any callbacks just disabled.  Calls that come
	 * in while we're running the callbacks will see the exit.death
	 * flag and know that we are not yet fully quiescent for purposes
	 * of detach bookkeeping.
	 */
	if (in_atomic() || irqs_disabled()) {
		if (atomic_add_unless(&utrace->report_work_added, 1, 1)) {
			int rc;
#ifdef STP_TF_DEBUG
			printk(KERN_ERR "%s:%d - adding task_work\n",
			       __FUNCTION__, __LINE__);
#endif
			rc = stp_task_work_add(task,
					       &utrace->report_work);
			if (rc != 0) {
				atomic_set(&utrace->report_work_added, 0);
				/* stp_task_work_add() returns -ESRCH
				 * if the task has already passed
				 * exit_task_work(). Just ignore this
				 * error. */
				if (rc != -ESRCH) {
					printk(KERN_ERR
					       "%s:%d - task_work_add() returned %d\n",
					       __FUNCTION__, __LINE__, rc);
				}
			}
		}
	}
	else {
		spin_lock(&utrace->lock);
		BUG_ON(utrace->death);
		utrace->death = 1;
		utrace->resume = UTRACE_RESUME;
		splice_attaching(utrace);
		spin_unlock(&utrace->lock);

		REPORT_CALLBACKS(, task, utrace, &report, UTRACE_EVENT(DEATH),
				 report_death, engine, -1/*group_dead*/,
				 -1/*signal*/);

		utrace_maybe_reap(task, utrace, false);
		utrace_free(utrace);
	}
}

/*
 * Finish the last reporting pass before returning to user mode.
 */
static void finish_resume_report(struct task_struct *task,
				 struct utrace *utrace,
				 struct utrace_report *report)
{
	finish_report_reset(task, utrace, report);

	switch (report->action) {
	case UTRACE_STOP:
		utrace_stop(task, utrace, report->resume_action);
		break;

	case UTRACE_INTERRUPT:
		if (!signal_pending(task)) {
			stp_task_notify_resume(task, utrace);
			set_tsk_thread_flag(task, TIF_SIGPENDING);
		}
		break;

	case UTRACE_REPORT:
	case UTRACE_RESUME:
	default:
		break;
	}
}

/*
 * This is called when TIF_NOTIFY_RESUME had been set (and is now clear).
 * We are close to user mode, and this is the place to report or stop.
 * When we return, we're going to user mode or into the signals code.
 */
static void utrace_resume(struct task_work *work)
{
	/*
	 * We could also do 'task_utrace_struct()' here to find the
	 * task's 'struct utrace', but 'container_of()' should be
	 * instantaneous (where 'task_utrace_struct()' has to do a
	 * hash lookup).
	 */
	struct utrace *utrace = container_of(work, struct utrace, resume_work);
	struct task_struct *task = current;
	INIT_REPORT(report);
	struct utrace_engine *engine;

	might_sleep();
	atomic_set(&utrace->resume_work_added, 0);

	/* Make sure the task isn't exiting. */
	if (task->flags & PF_EXITING) {
		/* Remember that this task_work_func is finished. */
		stp_task_work_func_done();
		return;
	}

	/*
	 * Some machines get here with interrupts disabled.  The same arch
	 * code path leads to calling into get_signal_to_deliver(), which
	 * implicitly reenables them by virtue of spin_unlock_irq.
	 */
	local_irq_enable();

	/*
	 * Update our bookkeeping even if there are no callbacks made here.
	 */
	report.action = start_report(utrace);

	switch (report.action) {
	case UTRACE_RESUME:
		/*
		 * Anything we might have done was already handled by
		 * utrace_get_signal(), or this is an entirely spurious
		 * call.  (The arch might use TIF_NOTIFY_RESUME for other
		 * purposes as well as calling us.)
		 */

		/* Remember that this task_work_func is finished. */
		stp_task_work_func_done();
		return;
	case UTRACE_INTERRUPT:
		/*
		 * Note that UTRACE_INTERRUPT reporting was handled by
		 * utrace_get_signal() in original utrace. In this
		 * utrace version, we'll handle it here like UTRACE_REPORT.
		 *
		 * Fallthrough...
		 */
	case UTRACE_REPORT:
		if (unlikely(!(utrace->utrace_flags & UTRACE_EVENT(QUIESCE))))
			break;
		/*
		 * Do a simple reporting pass, with no specific
		 * callback after report_quiesce.
		 */
		report.action = UTRACE_RESUME;
		list_for_each_entry(engine, &utrace->attached, entry)
			start_callback(utrace, &report, engine, task, 0);
		break;
	default:
		/*
		 * Even if this report was truly spurious, there is no need
		 * for utrace_reset() now.  TIF_NOTIFY_RESUME was already
		 * cleared--it doesn't stay spuriously set.
		 */
		report.spurious = false;
		break;
	}

	/*
	 * Finish the report and either stop or get ready to resume.
	 * If utrace->resume was not UTRACE_REPORT, this applies its
	 * effect now (i.e. step or interrupt).
	 */
	finish_resume_report(task, utrace, &report);

	/* Remember that this task_work_func is finished. */
	stp_task_work_func_done();
}


static void utrace_report_work(struct task_work *work)
{
	/*
	 * We could also do 'task_utrace_struct()' here to find the
	 * task's 'struct utrace', but 'container_of()' should be
	 * instantaneous (where 'task_utrace_struct()' has to do a
	 * hash lookup).
	 */
	struct utrace *utrace = container_of(work, struct utrace, report_work);
	struct task_struct *task = current;
	INIT_REPORT(report);
	struct utrace_engine *engine;
	unsigned long clone_flags;

#ifdef STP_TF_DEBUG
	printk(KERN_ERR "%s:%d - atomic %d, irqs_disabled %d\n",
	       __FUNCTION__, __LINE__, in_atomic(), irqs_disabled());
#endif
	might_sleep();

	atomic_set(&utrace->report_work_added, 0);
	spin_lock(&utrace->lock);
	BUG_ON(utrace->death);
	utrace->death = 1;
	utrace->resume = UTRACE_RESUME;
	splice_attaching(utrace);
	spin_unlock(&utrace->lock);

	REPORT_CALLBACKS(, task, utrace, &report, UTRACE_EVENT(DEATH),
			 report_death, engine, -1/*group_dead*/,
			 -1/*signal*/);

	utrace_maybe_reap(task, utrace, false);
	utrace_free(utrace);

	/* Remember that this task_work_func is finished. */
	stp_task_work_func_done();
}

#endif	/* _STP_UTRACE_C */