diff options
Diffstat (limited to 'drivers/irqchip')
-rw-r--r-- | drivers/irqchip/Kconfig | 12 | ||||
-rw-r--r-- | drivers/irqchip/Makefile | 3 | ||||
-rw-r--r-- | drivers/irqchip/irq-gic-v2m.c | 333 | ||||
-rw-r--r-- | drivers/irqchip/irq-gic-v3-its.c | 1425 | ||||
-rw-r--r-- | drivers/irqchip/irq-gic-v3.c | 156 | ||||
-rw-r--r-- | drivers/irqchip/irq-gic.c | 81 | ||||
-rw-r--r-- | drivers/irqchip/irq-mtk-sysirq.c | 163 |
7 files changed, 2109 insertions, 64 deletions
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index e12cb23d786c..cc79d2a5a8c2 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -5,8 +5,15 @@ config IRQCHIP config ARM_GIC bool select IRQ_DOMAIN + select IRQ_DOMAIN_HIERARCHY select MULTI_IRQ_HANDLER +config ARM_GIC_V2M + bool + depends on ARM_GIC + depends on PCI && PCI_MSI + select PCI_MSI_IRQ_DOMAIN + config GIC_NON_BANKED bool @@ -14,6 +21,11 @@ config ARM_GIC_V3 bool select IRQ_DOMAIN select MULTI_IRQ_HANDLER + select IRQ_DOMAIN_HIERARCHY + +config ARM_GIC_V3_ITS + bool + select PCI_MSI_IRQ_DOMAIN config ARM_NVIC bool diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 4954a314c31e..9516a324be6d 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -19,7 +19,9 @@ obj-$(CONFIG_ARCH_SUNXI) += irq-sun4i.o obj-$(CONFIG_ARCH_SUNXI) += irq-sunxi-nmi.o obj-$(CONFIG_ARCH_SPEAR3XX) += spear-shirq.o obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o +obj-$(CONFIG_ARM_GIC_V2M) += irq-gic-v2m.o obj-$(CONFIG_ARM_GIC_V3) += irq-gic-v3.o irq-gic-common.o +obj-$(CONFIG_ARM_GIC_V3_ITS) += irq-gic-v3-its.o obj-$(CONFIG_ARM_NVIC) += irq-nvic.o obj-$(CONFIG_ARM_VIC) += irq-vic.o obj-$(CONFIG_ATMEL_AIC_IRQ) += irq-atmel-aic-common.o irq-atmel-aic.o @@ -39,3 +41,4 @@ obj-$(CONFIG_BCM7120_L2_IRQ) += irq-bcm7120-l2.o obj-$(CONFIG_BRCMSTB_L2_IRQ) += irq-brcmstb-l2.o obj-$(CONFIG_KEYSTONE_IRQ) += irq-keystone.o obj-$(CONFIG_MIPS_GIC) += irq-mips-gic.o +obj-$(CONFIG_ARCH_MEDIATEK) += irq-mtk-sysirq.o diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c new file mode 100644 index 000000000000..fdf706555d72 --- /dev/null +++ b/drivers/irqchip/irq-gic-v2m.c @@ -0,0 +1,333 @@ +/* + * ARM GIC v2m MSI(-X) support + * Support for Message Signaled Interrupts for systems that + * implement ARM Generic Interrupt Controller: GICv2m. + * + * Copyright (C) 2014 Advanced Micro Devices, Inc. + * Authors: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> + * Harish Kasiviswanathan <harish.kasiviswanathan@amd.com> + * Brandon Anderson <brandon.anderson@amd.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#define pr_fmt(fmt) "GICv2m: " fmt + +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/slab.h> +#include <linux/spinlock.h> + +/* +* MSI_TYPER: +* [31:26] Reserved +* [25:16] lowest SPI assigned to MSI +* [15:10] Reserved +* [9:0] Numer of SPIs assigned to MSI +*/ +#define V2M_MSI_TYPER 0x008 +#define V2M_MSI_TYPER_BASE_SHIFT 16 +#define V2M_MSI_TYPER_BASE_MASK 0x3FF +#define V2M_MSI_TYPER_NUM_MASK 0x3FF +#define V2M_MSI_SETSPI_NS 0x040 +#define V2M_MIN_SPI 32 +#define V2M_MAX_SPI 1019 + +#define V2M_MSI_TYPER_BASE_SPI(x) \ + (((x) >> V2M_MSI_TYPER_BASE_SHIFT) & V2M_MSI_TYPER_BASE_MASK) + +#define V2M_MSI_TYPER_NUM_SPI(x) ((x) & V2M_MSI_TYPER_NUM_MASK) + +struct v2m_data { + spinlock_t msi_cnt_lock; + struct msi_controller mchip; + struct resource res; /* GICv2m resource */ + void __iomem *base; /* GICv2m virt address */ + u32 spi_start; /* The SPI number that MSIs start */ + u32 nr_spis; /* The number of SPIs for MSIs */ + unsigned long *bm; /* MSI vector bitmap */ + struct irq_domain *domain; +}; + +static void gicv2m_mask_msi_irq(struct irq_data *d) +{ + pci_msi_mask_irq(d); + irq_chip_mask_parent(d); +} + +static void gicv2m_unmask_msi_irq(struct irq_data *d) +{ + pci_msi_unmask_irq(d); + irq_chip_unmask_parent(d); +} + +static struct irq_chip gicv2m_msi_irq_chip = { + .name = "MSI", + .irq_mask = gicv2m_mask_msi_irq, + .irq_unmask = gicv2m_unmask_msi_irq, + .irq_eoi = irq_chip_eoi_parent, + .irq_write_msi_msg = pci_msi_domain_write_msg, +}; + +static struct msi_domain_info gicv2m_msi_domain_info = { + .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSIX), + .chip = &gicv2m_msi_irq_chip, +}; + +static int gicv2m_set_affinity(struct irq_data *irq_data, + const struct cpumask *mask, bool force) +{ + int ret; + + ret = irq_chip_set_affinity_parent(irq_data, mask, force); + if (ret == IRQ_SET_MASK_OK) + ret = IRQ_SET_MASK_OK_DONE; + + return ret; +} + +static void gicv2m_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct v2m_data *v2m = irq_data_get_irq_chip_data(data); + phys_addr_t addr = v2m->res.start + V2M_MSI_SETSPI_NS; + + msg->address_hi = (u32) (addr >> 32); + msg->address_lo = (u32) (addr); + msg->data = data->hwirq; +} + +static struct irq_chip gicv2m_irq_chip = { + .name = "GICv2m", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = gicv2m_set_affinity, + .irq_compose_msi_msg = gicv2m_compose_msi_msg, +}; + +static int gicv2m_irq_gic_domain_alloc(struct irq_domain *domain, + unsigned int virq, + irq_hw_number_t hwirq) +{ + struct of_phandle_args args; + struct irq_data *d; + int err; + + args.np = domain->parent->of_node; + args.args_count = 3; + args.args[0] = 0; + args.args[1] = hwirq - 32; + args.args[2] = IRQ_TYPE_EDGE_RISING; + + err = irq_domain_alloc_irqs_parent(domain, virq, 1, &args); + if (err) + return err; + + /* Configure the interrupt line to be edge */ + d = irq_domain_get_irq_data(domain->parent, virq); + d->chip->irq_set_type(d, IRQ_TYPE_EDGE_RISING); + return 0; +} + +static void gicv2m_unalloc_msi(struct v2m_data *v2m, unsigned int hwirq) +{ + int pos; + + pos = hwirq - v2m->spi_start; + if (pos < 0 || pos >= v2m->nr_spis) { + pr_err("Failed to teardown msi. Invalid hwirq %d\n", hwirq); + return; + } + + spin_lock(&v2m->msi_cnt_lock); + __clear_bit(pos, v2m->bm); + spin_unlock(&v2m->msi_cnt_lock); +} + +static int gicv2m_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct v2m_data *v2m = domain->host_data; + int hwirq, offset, err = 0; + + spin_lock(&v2m->msi_cnt_lock); + offset = find_first_zero_bit(v2m->bm, v2m->nr_spis); + if (offset < v2m->nr_spis) + __set_bit(offset, v2m->bm); + else + err = -ENOSPC; + spin_unlock(&v2m->msi_cnt_lock); + + if (err) + return err; + + hwirq = v2m->spi_start + offset; + + err = gicv2m_irq_gic_domain_alloc(domain, virq, hwirq); + if (err) { + gicv2m_unalloc_msi(v2m, hwirq); + return err; + } + + irq_domain_set_hwirq_and_chip(domain, virq, hwirq, + &gicv2m_irq_chip, v2m); + + return 0; +} + +static void gicv2m_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct v2m_data *v2m = irq_data_get_irq_chip_data(d); + + BUG_ON(nr_irqs != 1); + gicv2m_unalloc_msi(v2m, d->hwirq); + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static const struct irq_domain_ops gicv2m_domain_ops = { + .alloc = gicv2m_irq_domain_alloc, + .free = gicv2m_irq_domain_free, +}; + +static bool is_msi_spi_valid(u32 base, u32 num) +{ + if (base < V2M_MIN_SPI) { + pr_err("Invalid MSI base SPI (base:%u)\n", base); + return false; + } + + if ((num == 0) || (base + num > V2M_MAX_SPI)) { + pr_err("Number of SPIs (%u) exceed maximum (%u)\n", + num, V2M_MAX_SPI - V2M_MIN_SPI + 1); + return false; + } + + return true; +} + +static int __init gicv2m_init_one(struct device_node *node, + struct irq_domain *parent) +{ + int ret; + struct v2m_data *v2m; + + v2m = kzalloc(sizeof(struct v2m_data), GFP_KERNEL); + if (!v2m) { + pr_err("Failed to allocate struct v2m_data.\n"); + return -ENOMEM; + } + + ret = of_address_to_resource(node, 0, &v2m->res); + if (ret) { + pr_err("Failed to allocate v2m resource.\n"); + goto err_free_v2m; + } + + v2m->base = ioremap(v2m->res.start, resource_size(&v2m->res)); + if (!v2m->base) { + pr_err("Failed to map GICv2m resource\n"); + ret = -ENOMEM; + goto err_free_v2m; + } + + if (!of_property_read_u32(node, "arm,msi-base-spi", &v2m->spi_start) && + !of_property_read_u32(node, "arm,msi-num-spis", &v2m->nr_spis)) { + pr_info("Overriding V2M MSI_TYPER (base:%u, num:%u)\n", + v2m->spi_start, v2m->nr_spis); + } else { + u32 typer = readl_relaxed(v2m->base + V2M_MSI_TYPER); + + v2m->spi_start = V2M_MSI_TYPER_BASE_SPI(typer); + v2m->nr_spis = V2M_MSI_TYPER_NUM_SPI(typer); + } + + if (!is_msi_spi_valid(v2m->spi_start, v2m->nr_spis)) { + ret = -EINVAL; + goto err_iounmap; + } + + v2m->bm = kzalloc(sizeof(long) * BITS_TO_LONGS(v2m->nr_spis), + GFP_KERNEL); + if (!v2m->bm) { + ret = -ENOMEM; + goto err_iounmap; + } + + v2m->domain = irq_domain_add_tree(NULL, &gicv2m_domain_ops, v2m); + if (!v2m->domain) { + pr_err("Failed to create GICv2m domain\n"); + ret = -ENOMEM; + goto err_free_bm; + } + + v2m->domain->parent = parent; + v2m->mchip.of_node = node; + v2m->mchip.domain = pci_msi_create_irq_domain(node, + &gicv2m_msi_domain_info, + v2m->domain); + if (!v2m->mchip.domain) { + pr_err("Failed to create MSI domain\n"); + ret = -ENOMEM; + goto err_free_domains; + } + + spin_lock_init(&v2m->msi_cnt_lock); + + ret = of_pci_msi_chip_add(&v2m->mchip); + if (ret) { + pr_err("Failed to add msi_chip.\n"); + goto err_free_domains; + } + + pr_info("Node %s: range[%#lx:%#lx], SPI[%d:%d]\n", node->name, + (unsigned long)v2m->res.start, (unsigned long)v2m->res.end, + v2m->spi_start, (v2m->spi_start + v2m->nr_spis)); + + return 0; + +err_free_domains: + if (v2m->mchip.domain) + irq_domain_remove(v2m->mchip.domain); + if (v2m->domain) + irq_domain_remove(v2m->domain); +err_free_bm: + kfree(v2m->bm); +err_iounmap: + iounmap(v2m->base); +err_free_v2m: + kfree(v2m); + return ret; +} + +static struct of_device_id gicv2m_device_id[] = { + { .compatible = "arm,gic-v2m-frame", }, + {}, +}; + +int __init gicv2m_of_init(struct device_node *node, struct irq_domain *parent) +{ + int ret = 0; + struct device_node *child; + + for (child = of_find_matching_node(node, gicv2m_device_id); child; + child = of_find_matching_node(child, gicv2m_device_id)) { + if (!of_find_property(child, "msi-controller", NULL)) + continue; + + ret = gicv2m_init_one(child, parent); + if (ret) { + of_node_put(node); + break; + } + } + + return ret; +} diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c new file mode 100644 index 000000000000..86e4684adeb1 --- /dev/null +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -0,0 +1,1425 @@ +/* + * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved. + * Author: Marc Zyngier <marc.zyngier@arm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/bitmap.h> +#include <linux/cpu.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/log2.h> +#include <linux/mm.h> +#include <linux/msi.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/percpu.h> +#include <linux/slab.h> + +#include <linux/irqchip/arm-gic-v3.h> + +#include <asm/cacheflush.h> +#include <asm/cputype.h> +#include <asm/exception.h> + +#include "irqchip.h" + +#define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1 << 0) + +#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) + +/* + * Collection structure - just an ID, and a redistributor address to + * ping. We use one per CPU as a bag of interrupts assigned to this + * CPU. + */ +struct its_collection { + u64 target_address; + u16 col_id; +}; + +/* + * The ITS structure - contains most of the infrastructure, with the + * msi_controller, the command queue, the collections, and the list of + * devices writing to it. + */ +struct its_node { + raw_spinlock_t lock; + struct list_head entry; + struct msi_controller msi_chip; + struct irq_domain *domain; + void __iomem *base; + unsigned long phys_base; + struct its_cmd_block *cmd_base; + struct its_cmd_block *cmd_write; + void *tables[GITS_BASER_NR_REGS]; + struct its_collection *collections; + struct list_head its_device_list; + u64 flags; + u32 ite_size; +}; + +#define ITS_ITT_ALIGN SZ_256 + +/* + * The ITS view of a device - belongs to an ITS, a collection, owns an + * interrupt translation table, and a list of interrupts. + */ +struct its_device { + struct list_head entry; + struct its_node *its; + struct its_collection *collection; + void *itt; + unsigned long *lpi_map; + irq_hw_number_t lpi_base; + int nr_lpis; + u32 nr_ites; + u32 device_id; +}; + +static LIST_HEAD(its_nodes); +static DEFINE_SPINLOCK(its_lock); +static struct device_node *gic_root_node; +static struct rdists *gic_rdists; + +#define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist)) +#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) + +/* + * ITS command descriptors - parameters to be encoded in a command + * block. + */ +struct its_cmd_desc { + union { + struct { + struct its_device *dev; + u32 event_id; + } its_inv_cmd; + + struct { + struct its_device *dev; + u32 event_id; + } its_int_cmd; + + struct { + struct its_device *dev; + int valid; + } its_mapd_cmd; + + struct { + struct its_collection *col; + int valid; + } its_mapc_cmd; + + struct { + struct its_device *dev; + u32 phys_id; + u32 event_id; + } its_mapvi_cmd; + + struct { + struct its_device *dev; + struct its_collection *col; + u32 id; + } its_movi_cmd; + + struct { + struct its_device *dev; + u32 event_id; + } its_discard_cmd; + + struct { + struct its_collection *col; + } its_invall_cmd; + }; +}; + +/* + * The ITS command block, which is what the ITS actually parses. + */ +struct its_cmd_block { + u64 raw_cmd[4]; +}; + +#define ITS_CMD_QUEUE_SZ SZ_64K +#define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) + +typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *, + struct its_cmd_desc *); + +static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) +{ + cmd->raw_cmd[0] &= ~0xffUL; + cmd->raw_cmd[0] |= cmd_nr; +} + +static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) +{ + cmd->raw_cmd[0] &= ~(0xffffUL << 32); + cmd->raw_cmd[0] |= ((u64)devid) << 32; +} + +static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) +{ + cmd->raw_cmd[1] &= ~0xffffffffUL; + cmd->raw_cmd[1] |= id; +} + +static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) +{ + cmd->raw_cmd[1] &= 0xffffffffUL; + cmd->raw_cmd[1] |= ((u64)phys_id) << 32; +} + +static void its_encode_size(struct its_cmd_block *cmd, u8 size) +{ + cmd->raw_cmd[1] &= ~0x1fUL; + cmd->raw_cmd[1] |= size & 0x1f; +} + +static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) +{ + cmd->raw_cmd[2] &= ~0xffffffffffffUL; + cmd->raw_cmd[2] |= itt_addr & 0xffffffffff00UL; +} + +static void its_encode_valid(struct its_cmd_block *cmd, int valid) +{ + cmd->raw_cmd[2] &= ~(1UL << 63); + cmd->raw_cmd[2] |= ((u64)!!valid) << 63; +} + +static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) +{ + cmd->raw_cmd[2] &= ~(0xffffffffUL << 16); + cmd->raw_cmd[2] |= (target_addr & (0xffffffffUL << 16)); +} + +static void its_encode_collection(struct its_cmd_block *cmd, u16 col) +{ + cmd->raw_cmd[2] &= ~0xffffUL; + cmd->raw_cmd[2] |= col; +} + +static inline void its_fixup_cmd(struct its_cmd_block *cmd) +{ + /* Let's fixup BE commands */ + cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]); + cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]); + cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]); + cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]); +} + +static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + unsigned long itt_addr; + u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); + + itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); + itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); + + its_encode_cmd(cmd, GITS_CMD_MAPD); + its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); + its_encode_size(cmd, size - 1); + its_encode_itt(cmd, itt_addr); + its_encode_valid(cmd, desc->its_mapd_cmd.valid); + + its_fixup_cmd(cmd); + + return desc->its_mapd_cmd.dev->collection; +} + +static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_MAPC); + its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); + its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); + its_encode_valid(cmd, desc->its_mapc_cmd.valid); + + its_fixup_cmd(cmd); + + return desc->its_mapc_cmd.col; +} + +static struct its_collection *its_build_mapvi_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_MAPVI); + its_encode_devid(cmd, desc->its_mapvi_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_mapvi_cmd.event_id); + its_encode_phys_id(cmd, desc->its_mapvi_cmd.phys_id); + its_encode_collection(cmd, desc->its_mapvi_cmd.dev->collection->col_id); + + its_fixup_cmd(cmd); + + return desc->its_mapvi_cmd.dev->collection; +} + +static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_MOVI); + its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_movi_cmd.id); + its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); + + its_fixup_cmd(cmd); + + return desc->its_movi_cmd.dev->collection; +} + +static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_DISCARD); + its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_discard_cmd.event_id); + + its_fixup_cmd(cmd); + + return desc->its_discard_cmd.dev->collection; +} + +static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_INV); + its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_inv_cmd.event_id); + + its_fixup_cmd(cmd); + + return desc->its_inv_cmd.dev->collection; +} + +static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_INVALL); + its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); + + its_fixup_cmd(cmd); + + return NULL; +} + +static u64 its_cmd_ptr_to_offset(struct its_node *its, + struct its_cmd_block *ptr) +{ + return (ptr - its->cmd_base) * sizeof(*ptr); +} + +static int its_queue_full(struct its_node *its) +{ + int widx; + int ridx; + + widx = its->cmd_write - its->cmd_base; + ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); + + /* This is incredibly unlikely to happen, unless the ITS locks up. */ + if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx) + return 1; + + return 0; +} + +static struct its_cmd_block *its_allocate_entry(struct its_node *its) +{ + struct its_cmd_block *cmd; + u32 count = 1000000; /* 1s! */ + + while (its_queue_full(its)) { + count--; + if (!count) { + pr_err_ratelimited("ITS queue not draining\n"); + return NULL; + } + cpu_relax(); + udelay(1); + } + + cmd = its->cmd_write++; + + /* Handle queue wrapping */ + if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) + its->cmd_write = its->cmd_base; + + return cmd; +} + +static struct its_cmd_block *its_post_commands(struct its_node *its) +{ + u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); + + writel_relaxed(wr, its->base + GITS_CWRITER); + + return its->cmd_write; +} + +static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) +{ + /* + * Make sure the commands written to memory are observable by + * the ITS. + */ + if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) + __flush_dcache_area(cmd, sizeof(*cmd)); + else + dsb(ishst); +} + +static void its_wait_for_range_completion(struct its_node *its, + struct its_cmd_block *from, + struct its_cmd_block *to) +{ + u64 rd_idx, from_idx, to_idx; + u32 count = 1000000; /* 1s! */ + + from_idx = its_cmd_ptr_to_offset(its, from); + to_idx = its_cmd_ptr_to_offset(its, to); + + while (1) { + rd_idx = readl_relaxed(its->base + GITS_CREADR); + if (rd_idx >= to_idx || rd_idx < from_idx) + break; + + count--; + if (!count) { + pr_err_ratelimited("ITS queue timeout\n"); + return; + } + cpu_relax(); + udelay(1); + } +} + +static void its_send_single_command(struct its_node *its, + its_cmd_builder_t builder, + struct its_cmd_desc *desc) +{ + struct its_cmd_block *cmd, *sync_cmd, *next_cmd; + struct its_collection *sync_col; + + raw_spin_lock(&its->lock); + + cmd = its_allocate_entry(its); + if (!cmd) { /* We're soooooo screewed... */ + pr_err_ratelimited("ITS can't allocate, dropping command\n"); + raw_spin_unlock(&its->lock); + return; + } + sync_col = builder(cmd, desc); + its_flush_cmd(its, cmd); + + if (sync_col) { + sync_cmd = its_allocate_entry(its); + if (!sync_cmd) { + pr_err_ratelimited("ITS can't SYNC, skipping\n"); + goto post; + } + its_encode_cmd(sync_cmd, GITS_CMD_SYNC); + its_encode_target(sync_cmd, sync_col->target_address); + its_fixup_cmd(sync_cmd); + its_flush_cmd(its, sync_cmd); + } + +post: + next_cmd = its_post_commands(its); + raw_spin_unlock(&its->lock); + + its_wait_for_range_completion(its, cmd, next_cmd); +} + +static void its_send_inv(struct its_device *dev, u32 event_id) +{ + struct its_cmd_desc desc; + + desc.its_inv_cmd.dev = dev; + desc.its_inv_cmd.event_id = event_id; + + its_send_single_command(dev->its, its_build_inv_cmd, &desc); +} + +static void its_send_mapd(struct its_device *dev, int valid) +{ + struct its_cmd_desc desc; + + desc.its_mapd_cmd.dev = dev; + desc.its_mapd_cmd.valid = !!valid; + + its_send_single_command(dev->its, its_build_mapd_cmd, &desc); +} + +static void its_send_mapc(struct its_node *its, struct its_collection *col, + int valid) +{ + struct its_cmd_desc desc; + + desc.its_mapc_cmd.col = col; + desc.its_mapc_cmd.valid = !!valid; + + its_send_single_command(its, its_build_mapc_cmd, &desc); +} + +static void its_send_mapvi(struct its_device *dev, u32 irq_id, u32 id) +{ + struct its_cmd_desc desc; + + desc.its_mapvi_cmd.dev = dev; + desc.its_mapvi_cmd.phys_id = irq_id; + desc.its_mapvi_cmd.event_id = id; + + its_send_single_command(dev->its, its_build_mapvi_cmd, &desc); +} + +static void its_send_movi(struct its_device *dev, + struct its_collection *col, u32 id) +{ + struct its_cmd_desc desc; + + desc.its_movi_cmd.dev = dev; + desc.its_movi_cmd.col = col; + desc.its_movi_cmd.id = id; + + its_send_single_command(dev->its, its_build_movi_cmd, &desc); +} + +static void its_send_discard(struct its_device *dev, u32 id) +{ + struct its_cmd_desc desc; + + desc.its_discard_cmd.dev = dev; + desc.its_discard_cmd.event_id = id; + + its_send_single_command(dev->its, its_build_discard_cmd, &desc); +} + +static void its_send_invall(struct its_node *its, struct its_collection *col) +{ + struct its_cmd_desc desc; + + desc.its_invall_cmd.col = col; + + its_send_single_command(its, its_build_invall_cmd, &desc); +} + +/* + * irqchip functions - assumes MSI, mostly. + */ + +static inline u32 its_get_event_id(struct irq_data *d) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + return d->hwirq - its_dev->lpi_base; +} + +static void lpi_set_config(struct irq_data *d, bool enable) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + irq_hw_number_t hwirq = d->hwirq; + u32 id = its_get_event_id(d); + u8 *cfg = page_address(gic_rdists->prop_page) + hwirq - 8192; + + if (enable) + *cfg |= LPI_PROP_ENABLED; + else + *cfg &= ~LPI_PROP_ENABLED; + + /* + * Make the above write visible to the redistributors. + * And yes, we're flushing exactly: One. Single. Byte. + * Humpf... + */ + if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING) + __flush_dcache_area(cfg, sizeof(*cfg)); + else + dsb(ishst); + its_send_inv(its_dev, id); +} + +static void its_mask_irq(struct irq_data *d) +{ + lpi_set_config(d, false); +} + +static void its_unmask_irq(struct irq_data *d) +{ + lpi_set_config(d, true); +} + +static void its_eoi_irq(struct irq_data *d) +{ + gic_write_eoir(d->hwirq); +} + +static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, + bool force) +{ + unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask); + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + struct its_collection *target_col; + u32 id = its_get_event_id(d); + + if (cpu >= nr_cpu_ids) + return -EINVAL; + + target_col = &its_dev->its->collections[cpu]; + its_send_movi(its_dev, target_col, id); + its_dev->collection = target_col; + + return IRQ_SET_MASK_OK_DONE; +} + +static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + struct its_node *its; + u64 addr; + + its = its_dev->its; + addr = its->phys_base + GITS_TRANSLATER; + + msg->address_lo = addr & ((1UL << 32) - 1); + msg->address_hi = addr >> 32; + msg->data = its_get_event_id(d); +} + +static struct irq_chip its_irq_chip = { + .name = "ITS", + .irq_mask = its_mask_irq, + .irq_unmask = its_unmask_irq, + .irq_eoi = its_eoi_irq, + .irq_set_affinity = its_set_affinity, + .irq_compose_msi_msg = its_irq_compose_msi_msg, +}; + +static void its_mask_msi_irq(struct irq_data *d) +{ + pci_msi_mask_irq(d); + irq_chip_mask_parent(d); +} + +static void its_unmask_msi_irq(struct irq_data *d) +{ + pci_msi_unmask_irq(d); + irq_chip_unmask_parent(d); +} + +static struct irq_chip its_msi_irq_chip = { + .name = "ITS-MSI", + .irq_unmask = its_unmask_msi_irq, + .irq_mask = its_mask_msi_irq, + .irq_eoi = irq_chip_eoi_parent, + .irq_write_msi_msg = pci_msi_domain_write_msg, +}; + +/* + * How we allocate LPIs: + * + * The GIC has id_bits bits for interrupt identifiers. From there, we + * must subtract 8192 which are reserved for SGIs/PPIs/SPIs. Then, as + * we allocate LPIs by chunks of 32, we can shift the whole thing by 5 + * bits to the right. + * + * This gives us (((1UL << id_bits) - 8192) >> 5) possible allocations. + */ +#define IRQS_PER_CHUNK_SHIFT 5 +#define IRQS_PER_CHUNK (1 << IRQS_PER_CHUNK_SHIFT) + +static unsigned long *lpi_bitmap; +static u32 lpi_chunks; +static DEFINE_SPINLOCK(lpi_lock); + +static int its_lpi_to_chunk(int lpi) +{ + return (lpi - 8192) >> IRQS_PER_CHUNK_SHIFT; +} + +static int its_chunk_to_lpi(int chunk) +{ + return (chunk << IRQS_PER_CHUNK_SHIFT) + 8192; +} + +static int its_lpi_init(u32 id_bits) +{ + lpi_chunks = its_lpi_to_chunk(1UL << id_bits); + + lpi_bitmap = kzalloc(BITS_TO_LONGS(lpi_chunks) * sizeof(long), + GFP_KERNEL); + if (!lpi_bitmap) { + lpi_chunks = 0; + return -ENOMEM; + } + + pr_info("ITS: Allocated %d chunks for LPIs\n", (int)lpi_chunks); + return 0; +} + +static unsigned long *its_lpi_alloc_chunks(int nr_irqs, int *base, int *nr_ids) +{ + unsigned long *bitmap = NULL; + int chunk_id; + int nr_chunks; + int i; + + nr_chunks = DIV_ROUND_UP(nr_irqs, IRQS_PER_CHUNK); + + spin_lock(&lpi_lock); + + do { + chunk_id = bitmap_find_next_zero_area(lpi_bitmap, lpi_chunks, + 0, nr_chunks, 0); + if (chunk_id < lpi_chunks) + break; + + nr_chunks--; + } while (nr_chunks > 0); + + if (!nr_chunks) + goto out; + + bitmap = kzalloc(BITS_TO_LONGS(nr_chunks * IRQS_PER_CHUNK) * sizeof (long), + GFP_ATOMIC); + if (!bitmap) + goto out; + + for (i = 0; i < nr_chunks; i++) + set_bit(chunk_id + i, lpi_bitmap); + + *base = its_chunk_to_lpi(chunk_id); + *nr_ids = nr_chunks * IRQS_PER_CHUNK; + +out: + spin_unlock(&lpi_lock); + + return bitmap; +} + +static void its_lpi_free(unsigned long *bitmap, int base, int nr_ids) +{ + int lpi; + + spin_lock(&lpi_lock); + + for (lpi = base; lpi < (base + nr_ids); lpi += IRQS_PER_CHUNK) { + int chunk = its_lpi_to_chunk(lpi); + BUG_ON(chunk > lpi_chunks); + if (test_bit(chunk, lpi_bitmap)) { + clear_bit(chunk, lpi_bitmap); + } else { + pr_err("Bad LPI chunk %d\n", chunk); + } + } + + spin_unlock(&lpi_lock); + + kfree(bitmap); +} + +/* + * We allocate 64kB for PROPBASE. That gives us at most 64K LPIs to + * deal with (one configuration byte per interrupt). PENDBASE has to + * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI). + */ +#define LPI_PROPBASE_SZ SZ_64K +#define LPI_PENDBASE_SZ (LPI_PROPBASE_SZ / 8 + SZ_1K) + +/* + * This is how many bits of ID we need, including the useless ones. + */ +#define LPI_NRBITS ilog2(LPI_PROPBASE_SZ + SZ_8K) + +#define LPI_PROP_DEFAULT_PRIO 0xa0 + +static int __init its_alloc_lpi_tables(void) +{ + phys_addr_t paddr; + + gic_rdists->prop_page = alloc_pages(GFP_NOWAIT, + get_order(LPI_PROPBASE_SZ)); + if (!gic_rdists->prop_page) { + pr_err("Failed to allocate PROPBASE\n"); + return -ENOMEM; + } + + paddr = page_to_phys(gic_rdists->prop_page); + pr_info("GIC: using LPI property table @%pa\n", &paddr); + + /* Priority 0xa0, Group-1, disabled */ + memset(page_address(gic_rdists->prop_page), + LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, + LPI_PROPBASE_SZ); + + /* Make sure the GIC will observe the written configuration */ |