// SPDX-License-Identifier: GPL-2.0
#define pr_fmt(fmt) "kcov: " fmt
#define DISABLE_BRANCH_PROFILING
#include <linux/atomic.h>
#include <linux/compiler.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/types.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/hashtable.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/preempt.h>
#include <linux/printk.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/vmalloc.h>
#include <linux/debugfs.h>
#include <linux/uaccess.h>
#include <linux/kcov.h>
#include <linux/refcount.h>
#include <linux/log2.h>
#include <asm/setup.h>
#define kcov_debug(fmt, ...) pr_debug("%s: " fmt, __func__, ##__VA_ARGS__)
/* Number of 64-bit words written per one comparison: */
#define KCOV_WORDS_PER_CMP 4
/*
* kcov descriptor (one per opened debugfs file).
* State transitions of the descriptor:
* - initial state after open()
* - then there must be a single ioctl(KCOV_INIT_TRACE) call
* - then, mmap() call (several calls are allowed but not useful)
* - then, ioctl(KCOV_ENABLE, arg), where arg is
* KCOV_TRACE_PC - to trace only the PCs
* or
* KCOV_TRACE_CMP - to trace only the comparison operands
* - then, ioctl(KCOV_DISABLE) to disable the task.
* Enabling/disabling ioctls can be repeated (only one task a time allowed).
*/
struct kcov {
/*
* Reference counter. We keep one for:
* - opened file descriptor
* - task with enabled coverage (we can't unwire it from another task)
* - each code section for remote coverage collection
*/
refcount_t refcount;
/* The lock protects mode, size, area and t. */
spinlock_t lock;
enum kcov_mode mode;
/* Size of arena (in long's). */
unsigned int size;
/* Coverage buffer shared with user space. */
void *area;
/* Task for which we collect coverage, or NULL. */
struct task_struct *t;
/* Collecting coverage from remote (background) threads. */
bool remote;
/* Size of remote area (in long's). */
unsigned int remote_size;
/*
* Sequence is incremented each time kcov is reenabled, used by
* kcov_remote_stop(), see the comment there.
*/
int sequence;
};
struct kcov_remote_area {
struct list_head list;
unsigned int size;
};
struct kcov_remote {
u64 handle;
struct kcov *kcov;
struct hlist_node hnode;
};
static DEFINE_SPINLOCK(kcov_remote_lock);
static DEFINE_HASHTABLE(kcov_remote_map, 4);
static struct list_head kcov_remote_areas = LIST_HEAD_INIT(kcov_remote_areas);
struct kcov_percpu_data {
void *irq_area;
unsigned int saved_mode;
unsigned int saved_size;
void *saved_area;
struct kcov *saved_kcov;
int saved_sequence;
};
DEFINE_PER_CPU(struct kcov_percpu_data, kcov_percpu_data);
/* Must be called with kcov_remote_lock locked. */
static struct kcov_remote *kcov_remote_find(u64 handle)
{
struct kcov_remote *remote;
hash_for_each_possible(kcov_remote_map, remote, hnode, handle) {
if (remote->handle == handle)
return remote;
}
return NULL;
}
/* Must be called with kcov_remote_lock locked. */
static struct kcov_remote *kcov_remote_add(struct kcov *kcov, u64 handle)
{
struct kcov_remote *remote;
if (kcov_remote_find(handle))
return ERR_PTR(-EEXIST);
remote = kmalloc(sizeof(*remote), GFP_ATOMIC);
if (!remote)
return ERR_PTR(-ENOMEM);
remote->handle = handle;
remote->kcov = kcov;
hash_add(kcov_remote_map, &remote->hnode, handle);
return remote;
}
/* Must be called with kcov_remote_lock locked. */
static struct kcov_remote_area *kcov_remote_area_get(unsigned int size)
{
struct kcov_remote_area *area;
struct list_head *pos;
list_for_each(pos, &kcov_remote_areas) {
area = list_entry(pos, struct kcov_remote_area, list);
if (area->size == size) {
list_del(&area->list);
return area;
}
}
return NULL;
}
/* Must be called with kcov_remote_lock locked. */
static void kcov_remote_area_put(struct kcov_remote_area *area,
unsigned int size)
{
INIT_LIST_HEAD(&area->list);
area->size = size;
list_add(&area->list, &kcov_remote_areas);
}
static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t)
{
unsigned int mode;
/*
* We are interested in code coverage as a function of a syscall inputs,
* so we ignore code executed in interrupts, unless we are in a remote
* coverage collection section in a softirq.
*/
if (!in_task() && !(in_serving_softirq() && t->kcov_softirq))
return false;
mode = READ_ONCE(t->kcov_mode);
/*
* There is some code that runs in interrupts but for which
* in_interrupt() returns false (e.g. preempt_schedule_irq()).
* READ_ONCE()/barrier() effectively provides load-acquire wrt
* interrupts, there are paired barrier()/WRITE_ONCE() in
* kcov_start().
*/
barrier();
return mode ==