diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2017-11-13 17:33:11 -0800 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2017-11-13 17:33:11 -0800 |
commit | 670310dfbae0eefe7318ff6a61e29e67a7a7bbce (patch) | |
tree | eb3ce3aa3e6786a64fec93d410bb6f0b9a56be77 /drivers/irqchip | |
parent | 43ff2f4db9d0f76452b77cfa645f02b471143b24 (diff) | |
parent | ffc661c99f621152d5fdcf53f9df0d48c326318b (diff) |
Merge branch 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull irq core updates from Thomas Gleixner:
"A rather large update for the interrupt core code and the irq chip drivers:
- Add a new bitmap matrix allocator and supporting changes, which is
used to replace the x86 vector allocator which comes with separate
pull request. This allows to replace the convoluted nested loop
allocation function in x86 with a facility which supports the
recently added property of managed interrupts proper and allows to
switch to a best effort vector reservation scheme, which addresses
problems with vector exhaustion.
- A large update to the ARM GIC-V3-ITS driver adding support for
range selectors.
- New interrupt controllers:
- Meson and Meson8 GPIO
- BCM7271 L2
- Socionext EXIU
If you expected that this will stop at some point, I have to
disappoint you. There are new ones posted already. Sigh!
- STM32 interrupt controller support for new platforms.
- A pile of fixes, cleanups and updates to the MIPS GIC driver
- The usual small fixes, cleanups and updates all over the place.
Most visible one is to move the irq chip drivers Kconfig switches
into a separate Kconfig menu"
* 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (70 commits)
genirq: Fix type of shifting literal 1 in __setup_irq()
irqdomain: Drop pointless NULL check in virq_debug_show_one
genirq/proc: Return proper error code when irq_set_affinity() fails
irq/work: Use llist_for_each_entry_safe
irqchip: mips-gic: Print warning if inherited GIC base is used
irqchip/mips-gic: Add pr_fmt and reword pr_* messages
irqchip/stm32: Move the wakeup on interrupt mask
irqchip/stm32: Fix initial values
irqchip/stm32: Add stm32h7 support
dt-bindings/interrupt-controllers: Add compatible string for stm32h7
irqchip/stm32: Add multi-bank management
irqchip/stm32: Select GENERIC_IRQ_CHIP
irqchip/exiu: Add support for Socionext Synquacer EXIU controller
dt-bindings: Add description of Socionext EXIU interrupt controller
irqchip/gic-v3-its: Fix VPE activate callback return value
irqchip: mips-gic: Make IPI bitmaps static
irqchip: mips-gic: Share register writes in gic_set_type()
irqchip: mips-gic: Remove gic_vpes variable
irqchip: mips-gic: Use num_possible_cpus() to reserve IPIs
irqchip: mips-gic: Configure EIC when CPUs come online
...
Diffstat (limited to 'drivers/irqchip')
-rw-r--r-- | drivers/irqchip/Kconfig | 13 | ||||
-rw-r--r-- | drivers/irqchip/Makefile | 2 | ||||
-rw-r--r-- | drivers/irqchip/irq-aspeed-i2c-ic.c | 4 | ||||
-rw-r--r-- | drivers/irqchip/irq-brcmstb-l2.c | 171 | ||||
-rw-r--r-- | drivers/irqchip/irq-gic-common.c | 5 | ||||
-rw-r--r-- | drivers/irqchip/irq-gic-common.h | 2 | ||||
-rw-r--r-- | drivers/irqchip/irq-gic-v3-its.c | 390 | ||||
-rw-r--r-- | drivers/irqchip/irq-gic-v3.c | 50 | ||||
-rw-r--r-- | drivers/irqchip/irq-gic.c | 71 | ||||
-rw-r--r-- | drivers/irqchip/irq-meson-gpio.c | 419 | ||||
-rw-r--r-- | drivers/irqchip/irq-mips-gic.c | 226 | ||||
-rw-r--r-- | drivers/irqchip/irq-omap-intc.c | 16 | ||||
-rw-r--r-- | drivers/irqchip/irq-renesas-intc-irqpin.c | 9 | ||||
-rw-r--r-- | drivers/irqchip/irq-sni-exiu.c | 227 | ||||
-rw-r--r-- | drivers/irqchip/irq-stm32-exti.c | 206 |
15 files changed, 1489 insertions, 322 deletions
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index a2ca82f6c2dd..53380bd72ea4 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -1,3 +1,5 @@ +menu "IRQ chip support" + config IRQCHIP def_bool y depends on OF_IRQ @@ -307,6 +309,7 @@ config EZNPS_GIC config STM32_EXTI bool select IRQ_DOMAIN + select GENERIC_IRQ_CHIP config QCOM_IRQ_COMBINER bool "QCOM IRQ combiner support" @@ -324,3 +327,13 @@ config IRQ_UNIPHIER_AIDET select IRQ_DOMAIN_HIERARCHY help Support for the UniPhier AIDET (ARM Interrupt Detector). + +config MESON_IRQ_GPIO + bool "Meson GPIO Interrupt Multiplexer" + depends on ARCH_MESON + select IRQ_DOMAIN + select IRQ_DOMAIN_HIERARCHY + help + Support Meson SoC Family GPIO Interrupt Multiplexer + +endmenu diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 046df81c402a..dae7282bfdef 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -81,3 +81,5 @@ obj-$(CONFIG_ARCH_ASPEED) += irq-aspeed-vic.o irq-aspeed-i2c-ic.o obj-$(CONFIG_STM32_EXTI) += irq-stm32-exti.o obj-$(CONFIG_QCOM_IRQ_COMBINER) += qcom-irq-combiner.o obj-$(CONFIG_IRQ_UNIPHIER_AIDET) += irq-uniphier-aidet.o +obj-$(CONFIG_ARCH_SYNQUACER) += irq-sni-exiu.o +obj-$(CONFIG_MESON_IRQ_GPIO) += irq-meson-gpio.o diff --git a/drivers/irqchip/irq-aspeed-i2c-ic.c b/drivers/irqchip/irq-aspeed-i2c-ic.c index 815b88dd18f2..f20200af0992 100644 --- a/drivers/irqchip/irq-aspeed-i2c-ic.c +++ b/drivers/irqchip/irq-aspeed-i2c-ic.c @@ -76,8 +76,8 @@ static int __init aspeed_i2c_ic_of_init(struct device_node *node, return -ENOMEM; i2c_ic->base = of_iomap(node, 0); - if (IS_ERR(i2c_ic->base)) { - ret = PTR_ERR(i2c_ic->base); + if (!i2c_ic->base) { + ret = -ENOMEM; goto err_free_ic; } diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c index b009b916a292..691d20eb0bec 100644 --- a/drivers/irqchip/irq-brcmstb-l2.c +++ b/drivers/irqchip/irq-brcmstb-l2.c @@ -1,7 +1,7 @@ /* * Generic Broadcom Set Top Box Level 2 Interrupt controller driver * - * Copyright (C) 2014 Broadcom Corporation + * Copyright (C) 2014-2017 Broadcom * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -31,35 +31,82 @@ #include <linux/irqchip.h> #include <linux/irqchip/chained_irq.h> -/* Register offsets in the L2 interrupt controller */ -#define CPU_STATUS 0x00 -#define CPU_SET 0x04 -#define CPU_CLEAR 0x08 -#define CPU_MASK_STATUS 0x0c -#define CPU_MASK_SET 0x10 -#define CPU_MASK_CLEAR 0x14 +struct brcmstb_intc_init_params { + irq_flow_handler_t handler; + int cpu_status; + int cpu_clear; + int cpu_mask_status; + int cpu_mask_set; + int cpu_mask_clear; +}; + +/* Register offsets in the L2 latched interrupt controller */ +static const struct brcmstb_intc_init_params l2_edge_intc_init = { + .handler = handle_edge_irq, + .cpu_status = 0x00, + .cpu_clear = 0x08, + .cpu_mask_status = 0x0c, + .cpu_mask_set = 0x10, + .cpu_mask_clear = 0x14 +}; + +/* Register offsets in the L2 level interrupt controller */ +static const struct brcmstb_intc_init_params l2_lvl_intc_init = { + .handler = handle_level_irq, + .cpu_status = 0x00, + .cpu_clear = -1, /* Register not present */ + .cpu_mask_status = 0x04, + .cpu_mask_set = 0x08, + .cpu_mask_clear = 0x0C +}; /* L2 intc private data structure */ struct brcmstb_l2_intc_data { - int parent_irq; - void __iomem *base; struct irq_domain *domain; + struct irq_chip_generic *gc; + int status_offset; + int mask_offset; bool can_wake; u32 saved_mask; /* for suspend/resume */ }; +/** + * brcmstb_l2_mask_and_ack - Mask and ack pending interrupt + * @d: irq_data + * + * Chip has separate enable/disable registers instead of a single mask + * register and pending interrupt is acknowledged by setting a bit. + * + * Note: This function is generic and could easily be added to the + * generic irqchip implementation if there ever becomes a will to do so. + * Perhaps with a name like irq_gc_mask_disable_and_ack_set(). + * + * e.g.: https://patchwork.kernel.org/patch/9831047/ + */ +static void brcmstb_l2_mask_and_ack(struct irq_data *d) +{ + struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + struct irq_chip_type *ct = irq_data_get_chip_type(d); + u32 mask = d->mask; + + irq_gc_lock(gc); + irq_reg_writel(gc, mask, ct->regs.disable); + *ct->mask_cache &= ~mask; + irq_reg_writel(gc, mask, ct->regs.ack); + irq_gc_unlock(gc); +} + static void brcmstb_l2_intc_irq_handle(struct irq_desc *desc) { struct brcmstb_l2_intc_data *b = irq_desc_get_handler_data(desc); - struct irq_chip_generic *gc = irq_get_domain_generic_chip(b->domain, 0); struct irq_chip *chip = irq_desc_get_chip(desc); unsigned int irq; u32 status; chained_irq_enter(chip, desc); - status = irq_reg_readl(gc, CPU_STATUS) & - ~(irq_reg_readl(gc, CPU_MASK_STATUS)); + status = irq_reg_readl(b->gc, b->status_offset) & + ~(irq_reg_readl(b->gc, b->mask_offset)); if (status == 0) { raw_spin_lock(&desc->lock); @@ -70,10 +117,8 @@ static void brcmstb_l2_intc_irq_handle(struct irq_desc *desc) do { irq = ffs(status) - 1; - /* ack at our level */ - irq_reg_writel(gc, 1 << irq, CPU_CLEAR); status &= ~(1 << irq); - generic_handle_irq(irq_find_mapping(b->domain, irq)); + generic_handle_irq(irq_linear_revmap(b->domain, irq)); } while (status); out: chained_irq_exit(chip, desc); @@ -82,16 +127,17 @@ out: static void brcmstb_l2_intc_suspend(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + struct irq_chip_type *ct = irq_data_get_chip_type(d); struct brcmstb_l2_intc_data *b = gc->private; irq_gc_lock(gc); /* Save the current mask */ - b->saved_mask = irq_reg_readl(gc, CPU_MASK_STATUS); + b->saved_mask = irq_reg_readl(gc, ct->regs.mask); if (b->can_wake) { /* Program the wakeup mask */ - irq_reg_writel(gc, ~gc->wake_active, CPU_MASK_SET); - irq_reg_writel(gc, gc->wake_active, CPU_MASK_CLEAR); + irq_reg_writel(gc, ~gc->wake_active, ct->regs.disable); + irq_reg_writel(gc, gc->wake_active, ct->regs.enable); } irq_gc_unlock(gc); } @@ -99,49 +145,56 @@ static void brcmstb_l2_intc_suspend(struct irq_data *d) static void brcmstb_l2_intc_resume(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + struct irq_chip_type *ct = irq_data_get_chip_type(d); struct brcmstb_l2_intc_data *b = gc->private; irq_gc_lock(gc); - /* Clear unmasked non-wakeup interrupts */ - irq_reg_writel(gc, ~b->saved_mask & ~gc->wake_active, CPU_CLEAR); + if (ct->chip.irq_ack) { + /* Clear unmasked non-wakeup interrupts */ + irq_reg_writel(gc, ~b->saved_mask & ~gc->wake_active, + ct->regs.ack); + } /* Restore the saved mask */ - irq_reg_writel(gc, b->saved_mask, CPU_MASK_SET); - irq_reg_writel(gc, ~b->saved_mask, CPU_MASK_CLEAR); + irq_reg_writel(gc, b->saved_mask, ct->regs.disable); + irq_reg_writel(gc, ~b->saved_mask, ct->regs.enable); irq_gc_unlock(gc); } static int __init brcmstb_l2_intc_of_init(struct device_node *np, - struct device_node *parent) + struct device_node *parent, + const struct brcmstb_intc_init_params + *init_params) { unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; struct brcmstb_l2_intc_data *data; - struct irq_chip_generic *gc; struct irq_chip_type *ct; int ret; unsigned int flags; + int parent_irq; + void __iomem *base; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; - data->base = of_iomap(np, 0); - if (!data->base) { + base = of_iomap(np, 0); + if (!base) { pr_err("failed to remap intc L2 registers\n"); ret = -ENOMEM; goto out_free; } /* Disable all interrupts by default */ - writel(0xffffffff, data->base + CPU_MASK_SET); + writel(0xffffffff, base + init_params->cpu_mask_set); /* Wakeup interrupts may be retained from S5 (cold boot) */ data->can_wake = of_property_read_bool(np, "brcm,irq-can-wake"); - if (!data->can_wake) - writel(0xffffffff, data->base + CPU_CLEAR); + if (!data->can_wake && (init_params->cpu_clear >= 0)) + writel(0xffffffff, base + init_params->cpu_clear); - data->parent_irq = irq_of_parse_and_map(np, 0); - if (!data->parent_irq) { + parent_irq = irq_of_parse_and_map(np, 0); + if (!parent_irq) { pr_err("failed to find parent interrupt\n"); ret = -EINVAL; goto out_unmap; @@ -163,29 +216,39 @@ static int __init brcmstb_l2_intc_of_init(struct device_node *np, /* Allocate a single Generic IRQ chip for this node */ ret = irq_alloc_domain_generic_chips(data->domain, 32, 1, - np->full_name, handle_edge_irq, clr, 0, flags); + np->full_name, init_params->handler, clr, 0, flags); if (ret) { pr_err("failed to allocate generic irq chip\n"); goto out_free_domain; } /* Set the IRQ chaining logic */ - irq_set_chained_handler_and_data(data->parent_irq, + irq_set_chained_handler_and_data(parent_irq, brcmstb_l2_intc_irq_handle, data); - gc = irq_get_domain_generic_chip(data->domain, 0); - gc->reg_base = data->base; - gc->private = data; - ct = gc->chip_types; - - ct->chip.irq_ack = irq_gc_ack_set_bit; - ct->regs.ack = CPU_CLEAR; + data->gc = irq_get_domain_generic_chip(data->domain, 0); + data->gc->reg_base = base; + data->gc->private = data; + data->status_offset = init_params->cpu_status; + data->mask_offset = init_params->cpu_mask_status; + + ct = data->gc->chip_types; + + if (init_params->cpu_clear >= 0) { + ct->regs.ack = init_params->cpu_clear; + ct->chip.irq_ack = irq_gc_ack_set_bit; + ct->chip.irq_mask_ack = brcmstb_l2_mask_and_ack; + } else { + /* No Ack - but still slightly more efficient to define this */ + ct->chip.irq_mask_ack = irq_gc_mask_disable_reg; + } ct->chip.irq_mask = irq_gc_mask_disable_reg; - ct->regs.disable = CPU_MASK_SET; + ct->regs.disable = init_params->cpu_mask_set; + ct->regs.mask = init_params->cpu_mask_status; ct->chip.irq_unmask = irq_gc_unmask_enable_reg; - ct->regs.enable = CPU_MASK_CLEAR; + ct->regs.enable = init_params->cpu_mask_clear; ct->chip.irq_suspend = brcmstb_l2_intc_suspend; ct->chip.irq_resume = brcmstb_l2_intc_resume; @@ -195,21 +258,35 @@ static int __init brcmstb_l2_intc_of_init(struct device_node *np, /* This IRQ chip can wake the system, set all child interrupts * in wake_enabled mask */ - gc->wake_enabled = 0xffffffff; + data->gc->wake_enabled = 0xffffffff; ct->chip.irq_set_wake = irq_gc_set_wake; } pr_info("registered L2 intc (mem: 0x%p, parent irq: %d)\n", - data->base, data->parent_irq); + base, parent_irq); return 0; out_free_domain: irq_domain_remove(data->domain); out_unmap: - iounmap(data->base); + iounmap(base); out_free: kfree(data); return ret; } -IRQCHIP_DECLARE(brcmstb_l2_intc, "brcm,l2-intc", brcmstb_l2_intc_of_init); + +int __init brcmstb_l2_edge_intc_of_init(struct device_node *np, + struct device_node *parent) +{ + return brcmstb_l2_intc_of_init(np, parent, &l2_edge_intc_init); +} +IRQCHIP_DECLARE(brcmstb_l2_intc, "brcm,l2-intc", brcmstb_l2_edge_intc_of_init); + +int __init brcmstb_l2_lvl_intc_of_init(struct device_node *np, + struct device_node *parent) +{ + return brcmstb_l2_intc_of_init(np, parent, &l2_lvl_intc_init); +} +IRQCHIP_DECLARE(bcm7271_l2_intc, "brcm,bcm7271-l2-intc", + brcmstb_l2_lvl_intc_of_init); diff --git a/drivers/irqchip/irq-gic-common.c b/drivers/irqchip/irq-gic-common.c index 9ae71804b5dd..30017df5b54c 100644 --- a/drivers/irqchip/irq-gic-common.c +++ b/drivers/irqchip/irq-gic-common.c @@ -40,8 +40,9 @@ void gic_enable_quirks(u32 iidr, const struct gic_quirk *quirks, for (; quirks->desc; quirks++) { if (quirks->iidr != (quirks->mask & iidr)) continue; - quirks->init(data); - pr_info("GIC: enabling workaround for %s\n", quirks->desc); + if (quirks->init(data)) + pr_info("GIC: enabling workaround for %s\n", + quirks->desc); } } diff --git a/drivers/irqchip/irq-gic-common.h b/drivers/irqchip/irq-gic-common.h index 205e5fddf6da..3919cd7c5285 100644 --- a/drivers/irqchip/irq-gic-common.h +++ b/drivers/irqchip/irq-gic-common.h @@ -23,7 +23,7 @@ struct gic_quirk { const char *desc; - void (*init)(void *data); + bool (*init)(void *data); u32 iidr; u32 mask; }; diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index e88395605e32..4039e64cd342 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -83,6 +83,8 @@ struct its_baser { u32 psz; }; +struct its_device; + /* * The ITS structure - contains most of the infrastructure, with the * top-level MSI domain, the command queue, the collections, and the @@ -97,12 +99,18 @@ struct its_node { struct its_cmd_block *cmd_write; struct its_baser tables[GITS_BASER_NR_REGS]; struct its_collection *collections; + struct fwnode_handle *fwnode_handle; + u64 (*get_msi_base)(struct its_device *its_dev); struct list_head its_device_list; u64 flags; + unsigned long list_nr; u32 ite_size; u32 device_ids; int numa_node; + unsigned int msi_domain_flags; + u32 pre_its_base; /* for Socionext Synquacer */ bool is_v4; + int vlpi_redist_offset; }; #define ITS_ITT_ALIGN SZ_256 @@ -152,12 +160,6 @@ static DEFINE_SPINLOCK(its_lock); static struct rdists *gic_rdists; static struct irq_domain *its_parent; -/* - * We have a maximum number of 16 ITSs in the whole system if we're - * using the ITSList mechanism - */ -#define ITS_LIST_MAX 16 - static unsigned long its_list_map; static u16 vmovp_seq_num; static DEFINE_RAW_SPINLOCK(vmovp_lock); @@ -272,10 +274,12 @@ struct its_cmd_block { #define ITS_CMD_QUEUE_SZ SZ_64K #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) -typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *, +typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *, + struct its_cmd_block *, struct its_cmd_desc *); -typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_cmd_block *, +typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *, + struct its_cmd_block *, struct its_cmd_desc *); static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l) @@ -379,7 +383,8 @@ static inline void its_fixup_cmd(struct its_cmd_block *cmd) cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]); } -static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd, +static struct its_collection *its_build_mapd_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { unsigned long itt_addr; @@ -399,7 +404,8 @@ static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd, return NULL; } -static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd, +static struct its_collection *its_build_mapc_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { its_encode_cmd(cmd, GITS_CMD_MAPC); @@ -412,7 +418,8 @@ static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd, return desc->its_mapc_cmd.col; } -static struct its_collection *its_build_mapti_cmd(struct its_cmd_block *cmd, +static struct its_collection *its_build_mapti_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { struct its_collection *col; @@ -431,7 +438,8 @@ static struct its_collection *its_build_mapti_cmd(struct its_cmd_block *cmd, return col; } -static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd, +static struct its_collection *its_build_movi_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { struct its_collection *col; @@ -449,7 +457,8 @@ static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd, return col; } -static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd, +static struct its_collection *its_build_discard_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { struct its_collection *col; @@ -466,7 +475,8 @@ static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd, return col; } -static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd, +static struct its_collection *its_build_inv_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { struct its_collection *col; @@ -483,7 +493,8 @@ static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd, return col; } -static struct its_collection *its_build_int_cmd(struct its_cmd_block *cmd, +static struct its_collection *its_build_int_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { struct its_collection *col; @@ -500,7 +511,8 @@ static struct its_collection *its_build_int_cmd(struct its_cmd_block *cmd, return col; } -static struct its_collection *its_build_clear_cmd(struct its_cmd_block *cmd, +static struct its_collection *its_build_clear_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { struct its_collection *col; @@ -517,7 +529,8 @@ static struct its_collection *its_build_clear_cmd(struct its_cmd_block *cmd, return col; } -static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd, +static struct its_collection *its_build_invall_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { its_encode_cmd(cmd, GITS_CMD_INVALL); @@ -528,7 +541,8 @@ static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd, return NULL; } -static struct its_vpe *its_build_vinvall_cmd(struct its_cmd_block *cmd, +static struct its_vpe *its_build_vinvall_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { its_encode_cmd(cmd, GITS_CMD_VINVALL); @@ -539,17 +553,20 @@ static struct its_vpe *its_build_vinvall_cmd(struct its_cmd_block *cmd, return desc->its_vinvall_cmd.vpe; } -static struct its_vpe *its_build_vmapp_cmd(struct its_cmd_block *cmd, +static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { unsigned long vpt_addr; + u64 target; vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); + target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; its_encode_cmd(cmd, GITS_CMD_VMAPP); its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id); its_encode_valid(cmd, desc->its_vmapp_cmd.valid); - its_encode_target(cmd, desc->its_vmapp_cmd.col->target_address); + its_encode_target(cmd, target); its_encode_vpt_addr(cmd, vpt_addr); its_encode_vpt_size(cmd, LPI_NRBITS - 1); @@ -558,7 +575,8 @@ static struct its_vpe *its_build_vmapp_cmd(struct its_cmd_block *cmd, return desc->its_vmapp_cmd.vpe; } -static struct its_vpe *its_build_vmapti_cmd(struct its_cmd_block *cmd, +static struct its_vpe *its_build_vmapti_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { u32 db; @@ -580,7 +598,8 @@ static struct its_vpe *its_build_vmapti_cmd(struct its_cmd_block *cmd, return desc->its_vmapti_cmd.vpe; } -static struct its_vpe *its_build_vmovi_cmd(struct its_cmd_block *cmd, +static struct its_vpe *its_build_vmovi_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { u32 db; @@ -602,14 +621,18 @@ static struct its_vpe *its_build_vmovi_cmd(struct its_cmd_block *cmd, return desc->its_vmovi_cmd.vpe; } -static struct its_vpe *its_build_vmovp_cmd(struct its_cmd_block *cmd, +static struct its_vpe *its_build_vmovp_cmd(struct its_node *its, + struct its_cmd_block *cmd, struct its_cmd_desc *desc) { + u64 target; + + target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset; its_encode_cmd(cmd, GITS_CMD_VMOVP); its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num); its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list); its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id); - its_encode_target(cmd, desc->its_vmovp_cmd.col->target_address); + its_encode_target(cmd, target); its_fixup_cmd(cmd); @@ -688,9 +711,9 @@ static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) dsb(ishst); } -static void its_wait_for_range_completion(struct its_node *its, - struct its_cmd_block *from, - struct its_cmd_block *to) +static int its_wait_for_range_completion(struct its_node *its, + struct its_cmd_block *from, + struct its_cmd_block *to) { u64 rd_idx, from_idx, to_idx; u32 count = 1000000; /* 1s! */ @@ -711,12 +734,15 @@ static void its_wait_for_range_completion(struct its_node *its, count--; if (!count) { - pr_err_ratelimited("ITS queue timeout\n"); - return; + pr_err_ratelimited("ITS queue timeout (%llu %llu %llu)\n", + from_idx, to_idx, rd_idx); + return -1; } cpu_relax(); udelay(1); } + + return 0; } /* Warning, macro hell follows */ @@ -736,7 +762,7 @@ void name(struct its_node *its, \ raw_spin_unlock_irqrestore(&its->lock, flags); \ return; \ } \ - sync_obj = builder(cmd, desc); \ + sync_obj = builder(its, cmd, desc); \ its_flush_cmd(its, cmd); \ \ if (sync_obj) { \ @@ -744,7 +770,7 @@ void name(struct its_node *its, \ if (!sync_cmd) \ goto post; \ \ - buildfn(sync_cmd, sync_obj); \ + buildfn(its, sync_cmd, sync_obj); \ its_flush_cmd(its, sync_cmd); \ } \ \ @@ -752,10 +778,12 @@ post: \ next_cmd = its_post_commands(its); \ raw_spin_unlock_irqrestore(&its->lock, flags); \ \ - its_wait_for_range_completion(its, cmd, next_cmd); \ + if (its_wait_for_range_completion(its, cmd, next_cmd)) \ + pr_err_ratelimited("ITS cmd %ps failed\n", builder); \ } -static void its_build_sync_cmd(struct its_cmd_block *sync_cmd, +static void its_build_sync_cmd(struct its_node *its, + struct its_cmd_block *sync_cmd, struct its_collection *sync_col) { its_encode_cmd(sync_cmd, GITS_CMD_SYNC); @@ -767,7 +795,8 @@ static void its_build_sync_cmd(struct its_cmd_block *sync_cmd, static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t, struct its_collection, its_build_sync_cmd) -static void its_build_vsync_cmd(struct its_cmd_block *sync_cmd, +static void its_build_vsync_cmd(struct its_node *its, + struct its_cmd_block *sync_cmd, struct its_vpe *sync_vpe) { its_encode_cmd(sync_cmd, GITS_CMD_VSYNC); @@ -899,21 +928,16 @@ static void its_send_vmovi(struct its_device *dev, u32 id) its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc); } -static void its_send_vmapp(struct its_vpe *vpe, bool valid) +static void its_send_vmapp(struct its_node *its, + struct its_vpe *vpe, bool valid) { struct its_cmd_desc desc; - struct its_node *its; desc.its_vmapp_cmd.vpe = vpe; desc.its_vmapp_cmd.valid = valid; + desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx]; - list_for_each_entry(its, &its_nodes, entry) { - if (!its->is_v4) - continue; - - desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx]; - its_send_single_vcommand(its, its_build_vmapp_cmd, &desc); - } + its_send_single_vcommand(its, its_build_vmapp_cmd, &desc); } static void its_send_vmovp(struct its_vpe *vpe) @@ -951,6 +975,9 @@ static void its_send_vmovp(struct its_vpe *vpe) if (!its->is_v4) continue; + if (!vpe->its_vm->vlpi_count[its->list_nr]) + continue; + desc.its_vmovp_cmd.col = &its->collections[col_id]; its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); } @@ -958,18 +985,12 @@ static void its_send_vmovp(struct its_vpe *vpe) raw_spin_unlock_irqrestore(&vmovp_lock, flags); } -static void its_send_vinvall(struct its_vpe *vpe) +static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) { struct its_cmd_desc desc; - struct its_node *its; desc.its_vinvall_cmd.vpe = vpe; - - list_for_each_entry(its, &its_nodes, entry) { - if (!its->is_v4) - continue; - its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); - } + its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); } /* @@ -991,9 +1012,15 @@ static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) if (irqd_is_forwarded_to_vcpu(d)) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); u32 event = its_get_event_id(d); + struct its_vlpi_map *map; prop_page = its_dev->event_map.vm->vprop_page; - hwirq = its_dev->event_map.vlpi_maps[event].vintid; + map = &its_dev->event_map.vlpi_maps[event]; + hwirq = map->vintid; + + /* Remember the updated property */ + map->properties &= ~clr; + map->properties |= set | LPI_PROP_GROUP1; } else { prop_page = gic_rdists->prop_page; hwirq = d->hwirq; @@ -1099,6 +1126,13 @@ static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, return IRQ_SET_MASK_OK_DONE; } +static u64 its_irq_get_msi_base(struct its_device *its_dev) +{ + struct its_node *its = its_dev->its; + + return its->phys_base + GITS_TRANSLATER; +} + static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); @@ -1106,7 +1140,7 @@ static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) u64 addr; its = its_dev->its; - addr = its->phys_base + GITS_TRANSLATER; + addr = its->get_msi_base(its_dev); msg->address_lo = lower_32_bits(addr); msg->address_hi = upper_32_bits(addr); @@ -1133,6 +1167,60 @@ static int its_irq_set_irqchip_state(struct irq_data *d, return 0; } +static void its_map_vm(struct its_node *its, struct its_vm *vm) +{ + unsigned long flags; + + /* Not using the ITS list? Everything is always mapped. */ + if (!its_list_map) + return; + + raw_spin_lock_irqsave(&vmovp_lock, flags); + + /* + * If the VM wasn't mapped yet, iterate over the vpes and get + * them mapped now. + */ + vm->vlpi_count[its->list_nr]++; + + if (vm->vlpi_count[its->list_nr] == 1) { + int i; + + for (i = 0; i < vm->nr_vpes; i++) { + struct its_vpe *vpe = vm->vpes[i]; + struct irq_data *d = irq_get_irq_data(vpe->irq); + + /* Map the VPE to the first possible CPU */ + vpe->col_idx = cpumask_first(cpu_online_mask); + its_send_vmapp(its, vpe, true); + its_send_vinvall(its, vpe); + irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); + } + } + + raw_spin_unlock_irqrestore(&vmovp_lock, flags); +} + +static void its_unmap_vm(struct its_node *its, struct its_vm *vm) +{ + unsigned long flags; + + /* Not using the ITS list? Everything is always mapped. */ + if (!its_list_map) + return; + + raw_spin_lock_irqsave(&vmovp_lock, flags); + + if (!--vm->vlpi_count[its->list_nr]) { + int i; + + for (i = 0; i < vm->nr_vpes; i++) + its_send_vmapp(its, vm->vpes[i], false); + } + + raw_spin_unlock_irqrestore(&vmovp_lock, flags); +} + static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); @@ -1168,12 +1256,23 @@ static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) /* Already mapped, move it around */ its_send_vmovi(its_dev, event); } else { + /* Ensure all the VPEs are mapped on this ITS */ + its_map_vm(its_dev->its, info->map->vm); + + /* + * Flag the interrupt as forwarded so that we can + * start poking the virtual property table. + */ + irqd_set_forwarded_to_vcpu(d); + + /* Write out the property to the prop table */ + lpi_write_config(d, 0xff, info->map->properties); + /* Drop the physical mapping */ its_send_discard(its_dev, event); /* and install the virtual one */ its_send_vmapti(its_dev, event); - irqd_set_forwarded_to_vcpu(d); /* Increment the number of VLPIs */ its_dev->event_map.nr_vlpis++; @@ -1229,6 +1328,9 @@ static int its_vlpi_unmap(struct irq_data *d) LPI_PROP_ENABLED | LPI_PROP_GROUP1)); |