// SPDX-License-Identifier: GPL-2.0-or-later
/*
* SN Platform GRU Driver
*
* DRIVER TABLE MANAGER + GRU CONTEXT LOAD/UNLOAD
*
* Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
*/
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/spinlock.h>
#include <linux/sched.h>
#include <linux/device.h>
#include <linux/list.h>
#include <linux/err.h>
#include <linux/prefetch.h>
#include <asm/uv/uv_hub.h>
#include "gru.h"
#include "grutables.h"
#include "gruhandles.h"
unsigned long gru_options __read_mostly;
static struct device_driver gru_driver = {
.name = "gru"
};
static struct device gru_device = {
.init_name = "",
.driver = &gru_driver,
};
struct device *grudev = &gru_device;
/*
* Select a gru fault map to be used by the current cpu. Note that
* multiple cpus may be using the same map.
* ZZZ should be inline but did not work on emulator
*/
int gru_cpu_fault_map_id(void)
{
#ifdef CONFIG_IA64
return uv_blade_processor_id() % GRU_NUM_TFM;
#else
int cpu = smp_processor_id();
int id, core;
core = uv_cpu_core_number(cpu);
id = core + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu);
return id;
#endif
}
/*--------- ASID Management -------------------------------------------
*
* Initially, assign asids sequentially from MIN_ASID .. MAX_ASID.
* Once MAX is reached, flush the TLB & start over. However,
* some asids may still be in use. There won't be many (percentage wise) still
* in use. Search active contexts & determine the value of the first
* asid in use ("x"s below). Set "limit" to this value.
* This defines a block of assignable asids.
*
* When "limit" is reached, search forward from limit+1 and determine the
* next block of assignable asids.
*
* Repeat until MAX_ASID is reached, then start over again.
*
* Each time MAX_ASID is reached, increment the asid generation. Since
* the search for in-use asids only checks contexts with GRUs currently
* assigned, asids in some contexts will be missed. Prior to loading
* a context, the asid generation of the GTS asid is rechecked. If it
* doesn't match the current generation, a new asid will be assigned.
*
* 0---------------x------------x---------------------x----|
* ^-next ^-limit ^-MAX_ASID
*
* All asid manipulation & context loading/unloading is protected by the
* gs_lock.
*/
/* Hit the asid limit. Start over */
static int gru_wrap_asid(struct gru_state *gru)
{
gru_dbg(grudev, "gid %d\n", gru->gs_gid);
STAT(asid_wrap);
gru->gs_asid_gen++;
return MIN_ASID;
}
/* Find the next chunk of unused asids */
static int gru_reset_asid_limit(struct gru_state *gru, int asid)
{
int i, gid, inuse_asid, limit;
gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);
STAT(asid_next);
limit = MAX_ASID;
if (asid >= limit)
asid = gru_wrap_asid(gru);
gru_flush_all_tlb(gru);
gid = gru->gs_gid;
again:
for (i = 0; i < GRU_NUM_CCH; i++) {
if (!gru->gs_gts[i] || is_kernel_context(gru->gs_gts[i]))
continue;
inuse_asid = gru->gs_gts[i]->ts_gms->ms_asids[gid].mt_asid;
gru_dbg(grudev, "gid %d, gts %p, gms %p, inuse 0x%x, cxt %d\n",
gru->gs_gid, gru->gs_gts[i], gru->gs_gts[i]->ts_gms,
inuse_asid, i);
if (inuse_asid == asid) {
asid += ASID_INC;
if (asid >= limit) {
/*
* empty range: reset the range limit and
* start over
*/
limit = MAX_ASID;
if (asid >= MAX_ASID)
asid = gru_wrap_asid(gru);
goto again;
}
}
if ((inuse_asid > asid) && (inuse_asid < limit))
limit = inuse_asid;
}
gru->gs_asid_limit = limit;
gru->gs_asid = asid;
gru_dbg(grudev, "gid %d, new asid 0x%x, new_limit 0x%x\n", gru->gs_gid,
asid, limit);
return asid;
}
/* Assign a new ASID to a thread context. */
static int gru_assign_asid(struct gru_state *gru)
{
int asid;
gru->gs_asid += ASID_INC;
asid = gru->gs_asid;
if (asid >= gru->gs_asid_limit)
asid = gru_reset_asid_limit(gru, asid);
gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);
return asid;
}
/*
* Clear n bits in a word. Return a word indicating the bits that were cleared.
* Optionally, build an array of chars that contain the bit numbers allocated.
*/
static unsigned long reserve_resources(unsigned long *p, int n, int mmax,
char *idx)
{
unsigned long bits = 0;
int i;
while (n--) {
i = find_first_bit(p, mmax);
if (i == mmax)
BUG();
__clear_bit(i, p);
__set_bit(i, &bits);
if (idx)
*idx++ = i;
}
return bits;
}
unsigned long gru_reserve_cb_resources(struct gru_state