327 lines
9.1 KiB
C
327 lines
9.1 KiB
C
|
// SPDX-License-Identifier: GPL-2.0
|
||
|
|
|
||
|
|
#define pr_fmt(fmt) "rethook: " fmt
|
||
|
|
|
||
|
|
#include <linux/bug.h>
|
||
|
|
#include <linux/kallsyms.h>
|
||
|
|
#include <linux/kprobes.h>
|
||
|
|
#include <linux/preempt.h>
|
||
|
|
#include <linux/rethook.h>
|
||
|
|
#include <linux/slab.h>
|
||
|
|
#include <linux/sort.h>
|
||
|
|
|
||
|
|
/* Return hook list (shadow stack by list) */
|
||
|
|
|
||
|
|
/*
|
||
|
|
* This function is called from delayed_put_task_struct() when a task is
|
||
|
|
* dead and cleaned up to recycle any kretprobe instances associated with
|
||
|
|
* this task. These left over instances represent probed functions that
|
||
|
|
* have been called but will never return.
|
||
|
|
*/
|
||
|
|
void rethook_flush_task(struct task_struct *tk)
|
||
|
|
{
|
||
|
|
struct rethook_node *rhn;
|
||
|
|
struct llist_node *node;
|
||
|
|
|
||
|
|
node = __llist_del_all(&tk->rethooks);
|
||
|
|
while (node) {
|
||
|
|
rhn = container_of(node, struct rethook_node, llist);
|
||
|
|
node = node->next;
|
||
|
|
preempt_disable();
|
||
|
|
rethook_recycle(rhn);
|
||
|
|
preempt_enable();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static void rethook_free_rcu(struct rcu_head *head)
|
||
|
|
{
|
||
|
|
struct rethook *rh = container_of(head, struct rethook, rcu);
|
||
|
|
struct rethook_node *rhn;
|
||
|
|
struct freelist_node *node;
|
||
|
|
int count = 1;
|
||
|
|
|
||
|
|
node = rh->pool.head;
|
||
|
|
while (node) {
|
||
|
|
rhn = container_of(node, struct rethook_node, freelist);
|
||
|
|
node = node->next;
|
||
|
|
kfree(rhn);
|
||
|
|
count++;
|
||
|
|
}
|
||
|
|
|
||
|
|
/* The rh->ref is the number of pooled node + 1 */
|
||
|
|
if (refcount_sub_and_test(count, &rh->ref))
|
||
|
|
kfree(rh);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* rethook_free() - Free struct rethook.
|
||
|
|
* @rh: the struct rethook to be freed.
|
||
|
|
*
|
||
|
|
* Free the rethook. Before calling this function, user must ensure the
|
||
|
|
* @rh::data is cleaned if needed (or, the handler can access it after
|
||
|
|
* calling this function.) This function will set the @rh to be freed
|
||
|
|
* after all rethook_node are freed (not soon). And the caller must
|
||
|
|
* not touch @rh after calling this.
|
||
|
|
*/
|
||
|
|
void rethook_free(struct rethook *rh)
|
||
|
|
{
|
||
|
|
WRITE_ONCE(rh->handler, NULL);
|
||
|
|
|
||
|
|
call_rcu(&rh->rcu, rethook_free_rcu);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* rethook_alloc() - Allocate struct rethook.
|
||
|
|
* @data: a data to pass the @handler when hooking the return.
|
||
|
|
* @handler: the return hook callback function.
|
||
|
|
*
|
||
|
|
* Allocate and initialize a new rethook with @data and @handler.
|
||
|
|
* Return NULL if memory allocation fails or @handler is NULL.
|
||
|
|
* Note that @handler == NULL means this rethook is going to be freed.
|
||
|
|
*/
|
||
|
|
struct rethook *rethook_alloc(void *data, rethook_handler_t handler)
|
||
|
|
{
|
||
|
|
struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
|
||
|
|
|
||
|
|
if (!rh || !handler)
|
||
|
|
return NULL;
|
||
|
|
|
||
|
|
rh->data = data;
|
||
|
|
rh->handler = handler;
|
||
|
|
rh->pool.head = NULL;
|
||
|
|
refcount_set(&rh->ref, 1);
|
||
|
|
|
||
|
|
return rh;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* rethook_add_node() - Add a new node to the rethook.
|
||
|
|
* @rh: the struct rethook.
|
||
|
|
* @node: the struct rethook_node to be added.
|
||
|
|
*
|
||
|
|
* Add @node to @rh. User must allocate @node (as a part of user's
|
||
|
|
* data structure.) The @node fields are initialized in this function.
|
||
|
|
*/
|
||
|
|
void rethook_add_node(struct rethook *rh, struct rethook_node *node)
|
||
|
|
{
|
||
|
|
node->rethook = rh;
|
||
|
|
freelist_add(&node->freelist, &rh->pool);
|
||
|
|
refcount_inc(&rh->ref);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void free_rethook_node_rcu(struct rcu_head *head)
|
||
|
|
{
|
||
|
|
struct rethook_node *node = container_of(head, struct rethook_node, rcu);
|
||
|
|
|
||
|
|
if (refcount_dec_and_test(&node->rethook->ref))
|
||
|
|
kfree(node->rethook);
|
||
|
|
kfree(node);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* rethook_recycle() - return the node to rethook.
|
||
|
|
* @node: The struct rethook_node to be returned.
|
||
|
|
*
|
||
|
|
* Return back the @node to @node::rethook. If the @node::rethook is already
|
||
|
|
* marked as freed, this will free the @node.
|
||
|
|
*/
|
||
|
|
void rethook_recycle(struct rethook_node *node)
|
||
|
|
{
|
||
|
|
lockdep_assert_preemption_disabled();
|
||
|
|
|
||
|
|
if (likely(READ_ONCE(node->rethook->handler)))
|
||
|
|
freelist_add(&node->freelist, &node->rethook->pool);
|
||
|
|
else
|
||
|
|
call_rcu(&node->rcu, free_rethook_node_rcu);
|
||
|
|
}
|
||
|
|
NOKPROBE_SYMBOL(rethook_recycle);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* rethook_try_get() - get an unused rethook node.
|
||
|
|
* @rh: The struct rethook which pools the nodes.
|
||
|
|
*
|
||
|
|
* Get an unused rethook node from @rh. If the node pool is empty, this
|
||
|
|
* will return NULL. Caller must disable preemption.
|
||
|
|
*/
|
||
|
|
struct rethook_node *rethook_try_get(struct rethook *rh)
|
||
|
|
{
|
||
|
|
rethook_handler_t handler = READ_ONCE(rh->handler);
|
||
|
|
struct freelist_node *fn;
|
||
|
|
|
||
|
|
lockdep_assert_preemption_disabled();
|
||
|
|
|
||
|
|
/* Check whether @rh is going to be freed. */
|
||
|
|
if (unlikely(!handler))
|
||
|
|
return NULL;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* This expects the caller will set up a rethook on a function entry.
|
||
|
|
* When the function returns, the rethook will eventually be reclaimed
|
||
|
|
* or released in the rethook_recycle() with call_rcu().
|
||
|
|
* This means the caller must be run in the RCU-availabe context.
|
||
|
|
*/
|
||
|
|
if (unlikely(!rcu_is_watching()))
|
||
|
|
return NULL;
|
||
|
|
|
||
|
|
fn = freelist_try_get(&rh->pool);
|
||
|
|
if (!fn)
|
||
|
|
return NULL;
|
||
|
|
|
||
|
|
return container_of(fn, struct rethook_node, freelist);
|
||
|
|
}
|
||
|
|
NOKPROBE_SYMBOL(rethook_try_get);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* rethook_hook() - Hook the current function return.
|
||
|
|
* @node: The struct rethook node to hook the function return.
|
||
|
|
* @regs: The struct pt_regs for the function entry.
|
||
|
|
* @mcount: True if this is called from mcount(ftrace) context.
|
||
|
|
*
|
||
|
|
* Hook the current running function return. This must be called when the
|
||
|
|
* function entry (or at least @regs must be the registers of the function
|
||
|
|
* entry.) @mcount is used for identifying the context. If this is called
|
||
|
|
* from ftrace (mcount) callback, @mcount must be set true. If this is called
|
||
|
|
* from the real function entry (e.g. kprobes) @mcount must be set false.
|
||
|
|
* This is because the way to hook the function return depends on the context.
|
||
|
|
*/
|
||
|
|
void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
|
||
|
|
{
|
||
|
|
arch_rethook_prepare(node, regs, mcount);
|
||
|
|
__llist_add(&node->llist, ¤t->rethooks);
|
||
|
|
}
|
||
|
|
NOKPROBE_SYMBOL(rethook_hook);
|
||
|
|
|
||
|
|
/* This assumes the 'tsk' is the current task or is not running. */
|
||
|
|
static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
|
||
|
|
struct llist_node **cur)
|
||
|
|
{
|
||
|
|
struct rethook_node *rh = NULL;
|
||
|
|
struct llist_node *node = *cur;
|
||
|
|
|
||
|
|
if (!node)
|
||
|
|
node = tsk->rethooks.first;
|
||
|
|
else
|
||
|
|
node = node->next;
|
||
|
|
|
||
|
|
while (node) {
|
||
|
|
rh = container_of(node, struct rethook_node, llist);
|
||
|
|
if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) {
|
||
|
|
*cur = node;
|
||
|
|
return rh->ret_addr;
|
||
|
|
}
|
||
|
|
node = node->next;
|
||
|
|
}
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
NOKPROBE_SYMBOL(__rethook_find_ret_addr);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* rethook_find_ret_addr -- Find correct return address modified by rethook
|
||
|
|
* @tsk: Target task
|
||
|
|
* @frame: A frame pointer
|
||
|
|
* @cur: a storage of the loop cursor llist_node pointer for next call
|
||
|
|
*
|
||
|
|
* Find the correct return address modified by a rethook on @tsk in unsigned
|
||
|
|
* long type.
|
||
|
|
* The @tsk must be 'current' or a task which is not running. @frame is a hint
|
||
|
|
* to get the currect return address - which is compared with the
|
||
|
|
* rethook::frame field. The @cur is a loop cursor for searching the
|
||
|
|
* kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
|
||
|
|
* first call, but '@cur' itself must NOT NULL.
|
||
|
|
*
|
||
|
|
* Returns found address value or zero if not found.
|
||
|
|
*/
|
||
|
|
unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
|
||
|
|
struct llist_node **cur)
|
||
|
|
{
|
||
|
|
struct rethook_node *rhn = NULL;
|
||
|
|
unsigned long ret;
|
||
|
|
|
||
|
|
if (WARN_ON_ONCE(!cur))
|
||
|
|
return 0;
|
||
|
|
|
||
|
|
if (WARN_ON_ONCE(tsk != current && task_is_running(tsk)))
|
||
|
|
return 0;
|
||
|
|
|
||
|
|
do {
|
||
|
|
ret = __rethook_find_ret_addr(tsk, cur);
|
||
|
|
if (!ret)
|
||
|
|
break;
|
||
|
|
rhn = container_of(*cur, struct rethook_node, llist);
|
||
|
|
} while (rhn->frame != frame);
|
||
|
|
|
||
|
|
return ret;
|
||
|
|
}
|
||
|
|
NOKPROBE_SYMBOL(rethook_find_ret_addr);
|
||
|
|
|
||
|
|
void __weak arch_rethook_fixup_return(struct pt_regs *regs,
|
||
|
|
unsigned long correct_ret_addr)
|
||
|
|
{
|
||
|
|
/*
|
||
|
|
* Do nothing by default. If the architecture which uses a
|
||
|
|
* frame pointer to record real return address on the stack,
|
||
|
|
* it should fill this function to fixup the return address
|
||
|
|
* so that stacktrace works from the rethook handler.
|
||
|
|
*/
|
||
|
|
}
|
||
|
|
|
||
|
|
/* This function will be called from each arch-defined trampoline. */
|
||
|
|
unsigned long rethook_trampoline_handler(struct pt_regs *regs,
|
||
|
|
unsigned long frame)
|
||
|
|
{
|
||
|
|
struct llist_node *first, *node = NULL;
|
||
|
|
unsigned long correct_ret_addr;
|
||
|
|
rethook_handler_t handler;
|
||
|
|
struct rethook_node *rhn;
|
||
|
|
|
||
|
|
correct_ret_addr = __rethook_find_ret_addr(current, &node);
|
||
|
|
if (!correct_ret_addr) {
|
||
|
|
pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n");
|
||
|
|
BUG_ON(1);
|
||
|
|
}
|
||
|
|
|
||
|
|
instruction_pointer_set(regs, correct_ret_addr);
|
||
|
|
|
||
|
|
/*
|
||
|
|
* These loops must be protected from rethook_free_rcu() because those
|
||
|
|
* are accessing 'rhn->rethook'.
|
||
|
|
*/
|
||
|
|
preempt_disable();
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Run the handler on the shadow stack. Do not unlink the list here because
|
||
|
|
* stackdump inside the handlers needs to decode it.
|
||
|
|
*/
|
||
|
|
first = current->rethooks.first;
|
||
|
|
while (first) {
|
||
|
|
rhn = container_of(first, struct rethook_node, llist);
|
||
|
|
if (WARN_ON_ONCE(rhn->frame != frame))
|
||
|
|
break;
|
||
|
|
handler = READ_ONCE(rhn->rethook->handler);
|
||
|
|
if (handler)
|
||
|
|
handler(rhn, rhn->rethook->data, regs);
|
||
|
|
|
||
|
|
if (first == node)
|
||
|
|
break;
|
||
|
|
first = first->next;
|
||
|
|
}
|
||
|
|
|
||
|
|
/* Fixup registers for returning to correct address. */
|
||
|
|
arch_rethook_fixup_return(regs, correct_ret_addr);
|
||
|
|
|
||
|
|
/* Unlink used shadow stack */
|
||
|
|
first = current->rethooks.first;
|
||
|
|
current->rethooks.first = node->next;
|
||
|
|
node->next = NULL;
|
||
|
|
|
||
|
|
while (first) {
|
||
|
|
rhn = container_of(first, struct rethook_node, llist);
|
||
|
|
first = first->next;
|
||
|
|
rethook_recycle(rhn);
|
||
|
|
}
|
||
|
|
preempt_enable();
|
||
|
|
|
||
|
|
return correct_ret_addr;
|
||
|
|
}
|
||
|
|
NOKPROBE_SYMBOL(rethook_trampoline_handler);
|