// SPDX-License-Identifier: GPL-2.0
/* net/sched/sch_taprio.c Time Aware Priority Scheduler
*
* Authors: Vinicius Costa Gomes <vinicius.gomes@intel.com>
*
*/
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/list.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/math64.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/rcupdate.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/sch_generic.h>
static LIST_HEAD(taprio_list);
static DEFINE_SPINLOCK(taprio_list_lock);
#define TAPRIO_ALL_GATES_OPEN -1
struct sched_entry {
struct list_head list;
/* The instant that this entry "closes" and the next one
* should open, the qdisc will make some effort so that no
* packet leaves after this time.
*/
ktime_t close_time;
atomic_t budget;
int index;
u32 gate_mask;
u32 interval;
u8 command;
};
struct sched_gate_list {
struct rcu_head rcu;
struct list_head entries;
size_t num_entries;
ktime_t cycle_close_time;
s64 cycle_time;
s64 base_time;
};
struct taprio_sched {
struct Qdisc **qdiscs;
struct Qdisc *root;
int clockid;
atomic64_t picos_per_byte; /* Using picoseconds because for 10Gbps+
* speeds it's sub-nanoseconds per byte
*/
/* Protects the update side of the RCU protected current_entry */
spinlock_t current_entry_lock;
struct sched_entry __rcu *current_entry;
struct sched_gate_list __rcu *oper_sched;
struct sched_gate_list __rcu *admin_sched;
ktime_t (*get_time)(void);
struct hrtimer advance_timer;
struct list_head taprio_list;
};
static ktime_t sched_base_time(const struct sched_gate_list *sched)
{
if (!sched)
return KTIME_MAX;
return ns_to_ktime(sched->base_time);
}
static void taprio_free_sched_cb(struct rcu_head *head)
{
struct sched_gate_list *sched = container_of(head, struct sched_gate_list, rcu);
struct sched_entry *entry, *n;
if (!sched)
return;
list_for_each_entry_safe(entry, n, &sched->entries, list) {
list_del(&entry->list);
kfree(entry);
}
kfree(sched);
}
static void switch_schedules(struct taprio_sched *q,
struct sched_gate_list **admin,
struct sched_gate_list **oper)
{
rcu_assign_pointer(q->oper_sched, *admin);
rcu_assign_pointer(q->admin_sched, NULL);
if (*oper)
call_rcu(&(*oper)->rcu, taprio_free_sched_cb);
*oper = *admin;
*admin = NULL;
}
static ktime_t get_cycle_time(struct sched_gate_list *sched)
{
struct sched_entry *entry;
ktime_t cycle = 0;
if (sched->cycle_time != 0)
return sched->cycle_time;
list_for_each_entry(entry, &sched->entries, list)
cycle = ktime_add_ns(cycle,