/*
* cgroups support for the BFQ I/O scheduler.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/blkdev.h>
#include <linux/cgroup.h>
#include <linux/elevator.h>
#include <linux/ktime.h>
#include <linux/rbtree.h>
#include <linux/ioprio.h>
#include <linux/sbitmap.h>
#include <linux/delay.h>
#include "bfq-iosched.h"
#ifdef CONFIG_BFQ_GROUP_IOSCHED
/* bfqg stats flags */
enum bfqg_stats_flags {
BFQG_stats_waiting = 0,
BFQG_stats_idling,
BFQG_stats_empty,
};
#define BFQG_FLAG_FNS(name) \
static void bfqg_stats_mark_##name(struct bfqg_stats *stats) \
{ \
stats->flags |= (1 << BFQG_stats_##name); \
} \
static void bfqg_stats_clear_##name(struct bfqg_stats *stats) \
{ \
stats->flags &= ~(1 << BFQG_stats_##name); \
} \
static int bfqg_stats_##name(struct bfqg_stats *stats) \
{ \
return (stats->flags & (1 << BFQG_stats_##name)) != 0; \
} \
BFQG_FLAG_FNS(waiting)
BFQG_FLAG_FNS(idling)
BFQG_FLAG_FNS(empty)
#undef BFQG_FLAG_FNS
/* This should be called with the scheduler lock held. */
static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats)
{
unsigned long long now;
if (!bfqg_stats_waiting(stats))
return;
now = sched_clock();
if (time_after64(now, stats->start_group_wait_time))
blkg_stat_add(&stats->group_wait_time,
now - stats->start_group_wait_time);
bfqg_stats_clear_waiting(stats);
}
/* This should be called with the scheduler lock held. */
static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,
struct bfq_group *curr_bfqg)
{
struct bfqg_stats *stats = &bfqg->stats;
if (bfqg_stats_waiting(stats))
return;
if (bfqg == curr_bfqg)
return;
stats->start_group_wait_time = sched_clock();
bfqg_stats_mark_waiting(stats);
}
/* This should be called with the scheduler lock held. */
static void bfqg_stats_end_empty_time(struct bfqg_stats *stats)
{
unsigned long long now;
if (!bfqg_stats_empty(stats))
return;
now = sched_clock();
if (time_after64(now, stats->start_empty_time))
blkg_stat_add(&stats->empty_time,
now - stats->start_empty_time);
bfqg_stats_clear_empty(stats);
}
void bfqg_stats_update_dequeue(struct bfq_group *bfqg)
{
blkg_stat_add(&bfqg->stats.dequeue, 1);
}
void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)
{
struct bfqg_stats *stats = &bfqg->stats;
if (blkg_rwstat_total(&stats->queued))
return;
/*
* group is already marked empty. This can happen if bfqq got new
* request in parent group and moved to this group while being added
* to service tree. Just ignore the event and move on.
*/
if (bfqg_stats_empty(stats))
return;
stats->start_empty_time = sched_clock();
bfqg_stats_mark_empty(stats);
}
void bfqg_stats_update_idle_time(struct bfq_group *bfqg)
{
struct bfqg_stats *stats = &bfqg->stats;
if (bfqg_stats_idling(stats)) {
unsigned long long now = sched_clock();
if (time_after64(now, stats->start_idle_time))
blkg_stat_add(&stats->idle_time,
now - stats->start_idle_time