path: root/drivers/misc/sgi-gru/grutables.h

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * SN Platform GRU Driver
 *
 *            GRU DRIVER TABLES, MACROS, externs, etc
 *
 *  Copyright (c) 2008 Silicon Graphics, Inc.  All Rights Reserved.
 */

#ifndef __GRUTABLES_H__
#define __GRUTABLES_H__

/*
 * GRU Chiplet:
 *   The GRU is a user addressible memory accelerator. It provides
 *   several forms of load, store, memset, bcopy instructions. In addition, it
 *   contains special instructions for AMOs, sending messages to message
 *   queues, etc.
 *
 *   The GRU is an integral part of the node controller. It connects
 *   directly to the cpu socket. In its current implementation, there are 2
 *   GRU chiplets in the node controller on each blade (~node).
 *
 *   The entire GRU memory space is fully coherent and cacheable by the cpus.
 *
 *   Each GRU chiplet has a physical memory map that looks like the following:
 *
 *   	+-----------------+
 *   	|/////////////////|
 *   	|/////////////////|
 *   	|/////////////////|
 *   	|/////////////////|
 *   	|/////////////////|
 *   	|/////////////////|
 *   	|/////////////////|
 *   	|/////////////////|
 *   	+-----------------+
 *   	|  system control |
 *   	+-----------------+        _______ +-------------+
 *   	|/////////////////|       /        |             |
 *   	|/////////////////|      /         |             |
 *   	|/////////////////|     /          | instructions|
 *   	|/////////////////|    /           |             |
 *   	|/////////////////|   /            |             |
 *   	|/////////////////|  /             |-------------|
 *   	|/////////////////| /              |             |
 *   	+-----------------+                |             |
 *   	|   context 15    |                |  data       |
 *   	+-----------------+                |             |
 *   	|    ......       | \              |             |
 *   	+-----------------+  \____________ +-------------+
 *   	|   context 1     |
 *   	+-----------------+
 *   	|   context 0     |
 *   	+-----------------+
 *
 *   Each of the "contexts" is a chunk of memory that can be mmaped into user
 *   space. The context consists of 2 parts:
 *
 *  	- an instruction space that can be directly accessed by the user
 *  	  to issue GRU instructions and to check instruction status.
 *
 *  	- a data area that acts as normal RAM.
 *
 *   User instructions contain virtual addresses of data to be accessed by the
 *   GRU. The GRU contains a TLB that is used to convert these user virtual
 *   addresses to physical addresses.
 *
 *   The "system control" area of the GRU chiplet is used by the kernel driver
 *   to manage user contexts and to perform functions such as TLB dropin and
 *   purging.
 *
 *   One context may be reserved for the kernel and used for cross-partition
 *   communication. The GRU will also be used to asynchronously zero out
 *   large blocks of memory (not currently implemented).
 *
 *
 * Tables:
 *
 * 	VDATA-VMA Data		- Holds a few parameters. Head of linked list of
 * 				  GTS tables for threads using the GSEG
 * 	GTS - Gru Thread State  - contains info for managing a GSEG context. A
 * 				  GTS is allocated for each thread accessing a
 * 				  GSEG.
 *     	GTD - GRU Thread Data   - contains shadow copy of GRU data when GSEG is
 *     				  not loaded into a GRU
 *	GMS - GRU Memory Struct - Used to manage TLB shootdowns. Tracks GRUs
 *				  where a GSEG has been loaded. Similar to
 *				  an mm_struct but for GRU.
 *
 *	GS  - GRU State 	- Used to manage the state of a GRU chiplet
 *	BS  - Blade State	- Used to manage state of all GRU chiplets
 *				  on a blade
 *
 *
 *  Normal task tables for task using GRU.
 *  		- 2 threads in process
 *  		- 2 GSEGs open in process
 *  		- GSEG1 is being used by both threads
 *  		- GSEG2 is used only by thread 2
 *
 *       task -->|
 *       task ---+---> mm ->------ (notifier) -------+-> gms
 *                     |                             |
 *                     |--> vma -> vdata ---> gts--->|		GSEG1 (thread1)
 *                     |                  |          |
 *                     |                  +-> gts--->|		GSEG1 (thread2)
 *                     |                             |
 *                     |--> vma -> vdata ---> gts--->|		GSEG2 (thread2)
 *                     .
 *                     .
 *
 *  GSEGs are marked DONTCOPY on fork
 *
 * At open
 * 	file.private_data -> NULL
 *
 * At mmap,
 * 	vma -> vdata
 *
 * After gseg reference
 * 	vma -> vdata ->gts
 *
 * After fork
 *   parent
 * 	vma -> vdata -> gts
 *   child
 * 	(vma is not copied)
 *
 */

#include <linux/rmap.h>
#include <linux/interrupt.h>
#include <linux/mutex.h>
#include <linux/wait.h>
#include <linux/mmu_notifier.h>
#include <linux/mm_types.h>
#include "gru.h"
#include "grulib.h"
#include "gruhandles.h"

extern struct gru_stats_s gru_stats;
extern struct gru_blade_state *gru_base[];
extern unsigned long gru_start_paddr, gru_end_paddr;
extern void *gru_start_vaddr;
extern unsigned int gru_max_gids;

#define GRU_MAX_BLADES		MAX_NUMNODES
#define GRU_MAX_GRUS		(GRU_MAX_BLADES * GRU_CHIPLETS_PER_BLADE)

#define GRU_DRIVER_ID_STR	"SGI GRU Device Driver"
#define GRU_DRIVER_VERSION_STR	"0.85"

/*
 * GRU statistics.
 */
struct gru_stats_s {
	atomic_long_t vdata_alloc;
	atomic_long_t vdata_free;
	atomic_long_t gts_alloc;
	atomic_long_t gts_free;
	atomic_long_t gms_alloc;
	atomic_long_t gms_free;
	atomic_long_t gts_double_allocate;
	atomic_long_t assign_context;
	atomic_long_t assign_context_failed;
	atomic_long_t free_context;
	atomic_long_t load_user_context;
	atomic_long_t load_kernel_context;
	atomic_long_t lock_kernel_context;
	atomic_long_t unlock_kernel_context;
	atomic_long_t steal_user_context;
	atomic_long_t steal_kernel_context;
	atomic_long_t steal_context_failed;
	atomic_long_t nopfn;
	atomic_long_t asid_new;
	atomic_long_t asid_next;
	atomic_long_t asid_wrap;
	atomic_long_t asid_reuse;
	atomic_long_t intr;
	atomic_long_t intr_cbr;
	atomic_long_t intr_tfh;
	atomic_long_t intr_spurious;
	atomic_long_t intr_mm_lock_failed;
	atomic_long_t call_os;
	atomic_long_t call_os_wait_queue;
	atomic_long_t user_flush_tlb;
	atomic_long_t user_unload_context;
	atomic_long_t user_exception;
	atomic_long_t set_context_option;
	atomic_long_t check_context_retarget_intr;
	atomic_long_t check_context_unload;
	atomic_long_t tlb_dropin;
	atomic_long_t tlb_preload_page;
	atomic_long_t tlb_dropin_fail_no_asid;
	atomic_long_t tlb_dropin_fail_upm;
	atomic_long_t tlb_dropin_fail_invalid;
	atomic_long_t tlb_dropin_fail_range_active;
	atomic_long_t tlb_dropin_fail_idle;
	atomic_long_t tlb_dropin_fail_fmm;
	atomic_long_t tlb_dropin_fail_no_exception;
	atomic_long_t tfh_stale_on_fault;
	atomic_long_t mmu_invalidate_range;
	atomic_long_t mmu_invalidate_page;
	atomic_long_t flush_tlb;
	atomic_long_t flush_tlb_gru;
	atomic_long_t flush_tlb_gru_tgh;
	atomic_long_t flush_tlb_gru_zero_asid;

	atomic_long_t copy_gpa;
	atomic_long_t read_gpa;

	atomic_long_t mesq_receive;
	atomic_long_t mesq_receive_none;
	atomic_long_t mesq_send;
	atomic_long_t mesq_send_failed;
	atomic_long_t mesq_noop;
	atomic_long_t mesq_send_unexpected_error;
	atomic_long_t mesq_send_lb_overflow;
	atomic_long_t mesq_send_qlimit_reached;
	atomic_long_t mesq_send_amo_nacked;
	atomic_long_t mesq_send_put_nacked;
	atomic_long_t mesq_page_overflow;
	atomic_long_t mesq_qf_locked;
	atomic_long_t mesq_qf_noop_not_full;
	atomic_long_t mesq_qf_switch_head_failed;
	atomic_long_t mesq_qf_unexpected_error;
	atomic_long_t mesq_noop_unexpected_error;
	atomic_long_t mesq_noop_lb_overflow;
	atomic_long_t mesq_noop_qlimit_reached;
	atomic_long_t mesq_noop_amo_nacked;
	atomic_long_t mesq_noop_put_nacked;
	atomic_long_t mesq_noop_page_overflow;

};

enum mcs_op {cchop_allocate, cchop_start, cchop_interrupt, cchop_interrupt_sync,
	cchop_deallocate, tfhop_write_only, tfhop_write_restart,
	tghop_invalidate, mcsop_last};

struct mcs_op_statistic {
	atomic_long_t	count;
	atomic_long_t	total;
	unsigned long	max;
};

extern struct mcs_op_statistic mcs_op_statistics[mcsop_last];

#define OPT_DPRINT		1
#define OPT_STATS		2


#define IRQ_GRU			110	/* Starting IRQ number for interrupts */

/* Delay in jiffies between attempts to assign a GRU context */
#define GRU_ASSIGN_DELAY	((HZ * 20) / 1000)

/*
 * If a process has it's context stolen, min delay in jiffies before trying to
 * steal a context from another process.
 */
#define GRU_STEAL_DELAY		((HZ * 200) / 1000)

#define STAT(id)	do {						\
				if (gru_options & OPT_STATS)		\
					atomic_long_inc(&gru_stats.id);	\
			} while (0)

#ifdef CONFIG_SGI_GRU_DEBUG
#define gru_dbg(dev, fmt, x...)						\
	do {								\
		if (gru_options & OPT_DPRINT)				\
			printk(KERN_DEBUG "GRU:%d %s: " fmt, smp_processor_id(), __func__, x);\
	} while (0)
#else
#define gru_dbg(x...)
#endif

/*-----------------------------------------------------------------------------
 * ASID management
 */
#define MAX_ASID	0xfffff0
#define MIN_ASID	8
#define ASID_INC	8	/* number of regions */

/* Generate a GRU asid value from a GRU base asid & a virtual address. */
#define VADDR_HI_BIT		64
#define GRUREGION(addr)		((addr) >> (VADDR_HI_BIT - 3) & 3)
#define GRUASID(asid, addr)	((asid) + GRUREGION(addr))

/*------------------------------------------------------------------------------
 *  File & VMS Tables
 */

struct gru_state;

/*
 * This structure is pointed to from the mmstruct via the notifier pointer.
 * There is one of these per address space.
 */
struct gru_mm_tracker {				/* pack to reduce size */
	unsigned int		mt_asid_gen:24;	/* ASID wrap count */
	unsigned int		mt_asid:24;	/* current base ASID for gru */
	unsigned short		mt_ctxbitmap:16;/* bitmap of contexts using
						   asid */
} __attribute__ ((packed));

struct gru_mm_struct {
	struct mmu_notifier	ms_notifier;
	spinlock_t		ms_asid_lock;	/* protects ASID assignment */
	atomic_t		ms_range_active;/* num range_invals active */
	wait_queue_head_t	ms_wait_queue;
	DECLARE_BITMAP(ms_asidmap, GRU_MAX_GRUS);
	struct gru_mm_tracker	ms_asids[GRU_MAX_GRUS];
};

/*
 * One of these structures is allocated when a GSEG is mmaped. The
 * structure is pointed to by the vma->vm_private_data field in the vma struct.
 */
struct gru_vma_data {
	spinlock_t		vd_lock;	/* Serialize access to vma */
	struct list_head	vd_head;	/* head of linked list of gts */
	long			vd_user_options;/* misc user option flags */
	int			vd_cbr_au_count;
	int			vd_dsr_au_count;
	unsigned char		vd_tlb_preload_count;
};

/*
 * One of these is allocated for each thread accessing a mmaped GRU. A linked
 * list of these structure is hung off the struct gru_vma_data in the mm_struct.
 */
struct gru_thread_state {
	struct list_head	ts_next;	/* list - head at vma-private */
	struct mutex		ts_ctxlock;	/* load/unload CTX lock */
	struct mm_struct	*ts_mm;		/* mm currently mapped to
						   context */
	struct vm_area_struct	*ts_vma;	/* vma of GRU context */
	struct gru_state	*ts_gru;	/* GRU where the context is
						   loaded */
	struct gru_mm_struct	*ts_gms;	/* asid & ioproc struct */
	unsigned char		ts_tlb_preload_count; /* TLB preload pages */
	unsigned long		ts_cbr_map;	/* map of allocated CBRs */
	unsigned long		ts_dsr_map;	/* map of allocated DATA
						   resources */
	unsigned long		ts_steal_jiffies;/* jiffies when context last
						    stolen */
	long			ts_user_options;/* misc user option flags */
	pid_t			ts_tgid_owner;	/* task that is using the
						   context - for migration */
	short			ts_user_blade_id;/* user selected blade */
	char			ts_user_chiplet_id;/* user selected