#include <linux/errno.h>
#include <linux/numa.h>
#include <linux/slab.h>
#include <linux/rculist.h>
#include <linux/threads.h>
#include <linux/preempt.h>
#include <linux/irqflags.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/device-mapper.h>
#include "dm.h"
#include "dm-stats.h"
#define DM_MSG_PREFIX "stats"
static int dm_stat_need_rcu_barrier;
/*
* Using 64-bit values to avoid overflow (which is a
* problem that block/genhd.c's IO accounting has).
*/
struct dm_stat_percpu {
unsigned long long sectors[2];
unsigned long long ios[2];
unsigned long long merges[2];
unsigned long long ticks[2];
unsigned long long io_ticks[2];
unsigned long long io_ticks_total;
unsigned long long time_in_queue;
};
struct dm_stat_shared {
atomic_t in_flight[2];
unsigned long stamp;
struct dm_stat_percpu tmp;
};
struct dm_stat {
struct list_head list_entry;
int id;
size_t n_entries;
sector_t start;
sector_t end;
sector_t step;
const char *program_id;
const char *aux_data;
struct rcu_head rcu_head;
size_t shared_alloc_size;
size_t percpu_alloc_size;
struct dm_stat_percpu *stat_percpu[NR_CPUS];
struct dm_stat_shared stat_shared[0];
};
struct dm_stats_last_position {
sector_t last_sector;
unsigned last_rw;
};
/*
* A typo on the command line could possibly make the kernel run out of memory
* and crash. To prevent the crash we account all used memory. We fail if we
* exhaust 1/4 of all memory or 1/2 of vmalloc space.
*/
#define DM_STATS_MEMORY_FACTOR 4
#define DM_STATS_VMALLOC_FACTOR 2
static DEFINE_SPINLOCK(shared_memory_lock);
static unsigned long shared_memory_amount;
static bool __check_shared_memory(size_t alloc_size)
{
size_t a;
a = shared_memory_amount + alloc_size;
if (a < shared_memory_amount)
return false;
if (a >> PAGE_SHIFT > totalram_pages / DM_STATS_MEMORY_FACTOR)
return false;
#ifdef CONFIG_MMU
if (a > (VMALLOC_END - VMALLOC_START) / DM_STATS_VMALLOC_FACTOR)
return false;
#endif
return true;
}
static bool check_shared_memory(size_t alloc_size)
{
bool ret;
spin_lock_irq(&shared_memory_lock);
ret = __check_shared_memory(alloc_size);
spin_unlock_irq(&shared_memory_lock);
return ret;
}
static bool claim_shared_memory(size_t alloc_size)
{
spin_lock_irq(&shared_memory_lock);
if (!__check_shared_memory(alloc_size)) {
spin_unlock_irq(&shared_memory_lock);
return false;
}
shared_memory_amount += alloc_size;
spin_unlock_irq(&shared_memory_lock);
return true;
}
static void free_shared_memory(size_t alloc_size)
{
unsigned long flags;
spin_lock_irqsave(&shared_memory_lock, flags);
if (WARN_ON_ONCE(shared_memory_amount < alloc_size)) {
spin_unlock_irqrestore(&shared_memory_lock, flags);
DMCRIT("Memory usage accounting bug.");
return;
}
shared_memory_amount -= alloc_size;
spin_unlock_irqrestore(&shared_memory_lock, flags);
}
static void *dm_kvzalloc(size_t alloc_size, int node)
{
void *p;
if (!claim_shared_memory(alloc_size))
return NULL;
if (alloc_size <= KMALLOC_MAX_SIZE) {
p = kzalloc_node(alloc_size, GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN, node);
if (p)
return p;
}
p = vzalloc_node(alloc_size, node);
if (p)
return p;
free_shared_memory(alloc_size);
return NULL;
}
static void dm_kvfree(void *ptr, size_t alloc_size)
{
if (!ptr)
return;
free_shared_memory(alloc_size);
kvfree(ptr);
}
static void dm_stat_free(struct rcu_head *head)
{
int cpu;
struct dm_stat *s = container_of(head, struct dm_stat, rcu_head);
kfree(s->program_id);
kfree(s->aux_data);
for_each_possible_cpu(cpu)
dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size);
dm_kvfree(s, s->shared_alloc_size);
}
static int dm_stat_in_flight(struct dm_stat_shared *shared)
{
return atomic_read(&shared->in_flight[READ]) +
atomic_read(