// SPDX-License-Identifier: GPL-2.0-or-later
/*
* cgroups support for the BFQ I/O scheduler.
*/
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/blkdev.h>
#include <linux/cgroup.h>
#include <linux/elevator.h>
#include <linux/ktime.h>
#include <linux/rbtree.h>
#include <linux/ioprio.h>
#include <linux/sbitmap.h>
#include <linux/delay.h>
#include "bfq-iosched.h"
#ifdef CONFIG_BFQ_CGROUP_DEBUG
static int bfq_stat_init(struct bfq_stat *stat, gfp_t gfp)
{
int ret;
ret = percpu_counter_init(&stat->cpu_cnt, 0, gfp);
if (ret)
return ret;
atomic64_set(&stat->aux_cnt, 0);
return 0;
}
static void bfq_stat_exit(struct bfq_stat *stat)
{
percpu_counter_destroy(&stat->cpu_cnt);
}
/**
* bfq_stat_add - add a value to a bfq_stat
* @stat: target bfq_stat
* @val: value to add
*
* Add @val to @stat. The caller must ensure that IRQ on the same CPU
* don't re-enter this function for the same counter.
*/
static inline void bfq_stat_add(struct bfq_stat *stat, uint64_t val)
{
percpu_counter_add_batch(&stat->cpu_cnt, val, BLKG_STAT_CPU_BATCH);
}
/**
* bfq_stat_read - read the current value of a bfq_stat
* @stat: bfq_stat to read
*/
static inline uint64_t bfq_stat_read(struct bfq_stat *stat)
{
return percpu_counter_sum_positive(&stat->cpu_cnt);
}
/**
* bfq_stat_reset - reset a bfq_stat
* @stat: bfq_stat to reset
*/
static inline void bfq_stat_reset(struct bfq_stat *stat)
{
percpu_counter_set(&stat->cpu_cnt, 0);
atomic64_set(&stat->aux_cnt, 0);
}
/**
* bfq_stat_add_aux - add a bfq_stat into another's aux count
* @to: the destination bfq_stat
* @from: the source
*
* Add @from's count including the aux one to @to's aux count.
*/
static inline void bfq_stat_add_aux(struct bfq_stat *to,
struct bfq_stat *from)
{
atomic64_add(bfq_stat_read(from) + atomic64_read(&from->aux_cnt),
&to->aux_cnt);
}
/**
* blkg_prfill_stat - prfill callback for bfq_stat
* @sf: seq_file to print to
* @pd: policy private data of interest
* @off: offset to the bfq_stat in @pd
*
* prfill callback for printing a bfq_stat.
*/
static u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd,
int off)
{
return __blkg_prfill_u64(sf, pd, bfq_stat_read((void *)pd + off));
}
/* bfqg stats flags */
enum bfqg_stats_flags {
BFQG_stats_waiting = 0,
BFQG_stats_idling,
BFQG_stats_empty