diff options
author | Thomas Gleixner <tglx@linutronix.de> | 2017-08-31 20:12:51 +0200 |
---|---|---|
committer | Thomas Gleixner <tglx@linutronix.de> | 2017-08-31 20:12:51 +0200 |
commit | 9fbd7fd28d1a1053325967670915c12b4b246a61 (patch) | |
tree | 9a462e5af1e90ef6c68b96fff75cad141e1a784f /drivers/irqchip/irq-gic-v3-its.c | |
parent | b33394ba5c0974a578c24b2fecbb91a984da5e09 (diff) | |
parent | ae3efabfadea92a7300f57792ebeb24b5d18469f (diff) |
Merge tag 'irqchip-4.14' of git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms into irq/core
Pull irqchip updates for 4.14 from Marc Zyngier:
- irqchip-specific part of the monster GICv4 series
- new UniPhier AIDET irqchip driver
- new variants of some Freescale MSI widget
- blanket removal of of_node->full_name in printk
- random collection of fixes
Diffstat (limited to 'drivers/irqchip/irq-gic-v3-its.c')
-rw-r--r-- | drivers/irqchip/irq-gic-v3-its.c | 1489 |
1 files changed, 1374 insertions, 115 deletions
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 22e228500357..578837cdddef 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved. + * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. * Author: Marc Zyngier <marc.zyngier@arm.com> * * This program is free software; you can redistribute it and/or modify @@ -36,6 +36,7 @@ #include <linux/irqchip.h> #include <linux/irqchip/arm-gic-v3.h> +#include <linux/irqchip/arm-gic-v4.h> #include <asm/cputype.h> #include <asm/exception.h> @@ -48,6 +49,19 @@ #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) +static u32 lpi_id_bits; + +/* + * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to + * deal with (one configuration byte per interrupt). PENDBASE has to + * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI). + */ +#define LPI_NRBITS lpi_id_bits +#define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K) +#define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K) + +#define LPI_PROP_DEFAULT_PRIO 0xa0 + /* * Collection structure - just an ID, and a redistributor address to * ping. We use one per CPU as a bag of interrupts assigned to this @@ -88,6 +102,7 @@ struct its_node { u32 ite_size; u32 device_ids; int numa_node; + bool is_v4; }; #define ITS_ITT_ALIGN SZ_256 @@ -100,11 +115,17 @@ struct event_lpi_map { u16 *col_map; irq_hw_number_t lpi_base; int nr_lpis; + struct mutex vlpi_lock; + struct its_vm *vm; + struct its_vlpi_map *vlpi_maps; + int nr_vlpis; }; /* - * The ITS view of a device - belongs to an ITS, a collection, owns an - * interrupt translation table, and a list of interrupts. + * The ITS view of a device - belongs to an ITS, owns an interrupt + * translation table, and a list of interrupts. If it some of its + * LPIs are injected into a guest (GICv4), the event_map.vm field + * indicates which one. */ struct its_device { struct list_head entry; @@ -115,13 +136,33 @@ struct its_device { u32 device_id; }; +static struct { + raw_spinlock_t lock; + struct its_device *dev; + struct its_vpe **vpes; + int next_victim; +} vpe_proxy; + static LIST_HEAD(its_nodes); static DEFINE_SPINLOCK(its_lock); static struct rdists *gic_rdists; static struct irq_domain *its_parent; +/* + * We have a maximum number of 16 ITSs in the whole system if we're + * using the ITSList mechanism + */ +#define ITS_LIST_MAX 16 + +static unsigned long its_list_map; +static u16 vmovp_seq_num; +static DEFINE_RAW_SPINLOCK(vmovp_lock); + +static DEFINE_IDA(its_vpeid_ida); + #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist)) #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) +#define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K) static struct its_collection *dev_event_to_col(struct its_device *its_dev, u32 event) @@ -145,6 +186,11 @@ struct its_cmd_desc { struct { struct its_device *dev; u32 event_id; + } its_clear_cmd; + + struct { + struct its_device *dev; + u32 event_id; } its_int_cmd; struct { @@ -177,6 +223,38 @@ struct its_cmd_desc { struct { struct its_collection *col; } its_invall_cmd; + + struct { + struct its_vpe *vpe; + } its_vinvall_cmd; + + struct { + struct its_vpe *vpe; + struct its_collection *col; + bool valid; + } its_vmapp_cmd; + + struct { + struct its_vpe *vpe; + struct its_device *dev; + u32 virt_id; + u32 event_id; + bool db_enabled; + } its_vmapti_cmd; + + struct { + struct its_vpe *vpe; + struct its_device *dev; + u32 event_id; + bool db_enabled; + } its_vmovi_cmd; + + struct { + struct its_vpe *vpe; + struct its_collection *col; + u16 seq_num; + u16 its_list; + } its_vmovp_cmd; }; }; @@ -193,6 +271,9 @@ struct its_cmd_block { typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *, struct its_cmd_desc *); +typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_cmd_block *, + struct its_cmd_desc *); + static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l) { u64 mask = GENMASK_ULL(h, l); @@ -245,6 +326,46 @@ static void its_encode_collection(struct its_cmd_block *cmd, u16 col) its_mask_encode(&cmd->raw_cmd[2], col, 15, 0); } +static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid) +{ + its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32); +} + +static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id) +{ + its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0); +} + +static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id) +{ + its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32); +} + +static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid) +{ + its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0); +} + +static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num) +{ + its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32); +} + +static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list) +{ + its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0); +} + +static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa) +{ + its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 50, 16); +} + +static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size) +{ + its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0); +} + static inline void its_fixup_cmd(struct its_cmd_block *cmd) { /* Let's fixup BE commands */ @@ -358,6 +479,40 @@ static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd, return col; } +static struct its_collection *its_build_int_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + struct its_collection *col; + + col = dev_event_to_col(desc->its_int_cmd.dev, + desc->its_int_cmd.event_id); + + its_encode_cmd(cmd, GITS_CMD_INT); + its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_int_cmd.event_id); + + its_fixup_cmd(cmd); + + return col; +} + +static struct its_collection *its_build_clear_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + struct its_collection *col; + + col = dev_event_to_col(desc->its_clear_cmd.dev, + desc->its_clear_cmd.event_id); + + its_encode_cmd(cmd, GITS_CMD_CLEAR); + its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); + its_encode_event_id(cmd, desc->its_clear_cmd.event_id); + + its_fixup_cmd(cmd); + + return col; +} + static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd, struct its_cmd_desc *desc) { @@ -369,6 +524,94 @@ static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd, return NULL; } +static struct its_vpe *its_build_vinvall_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_VINVALL); + its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id); + + its_fixup_cmd(cmd); + + return desc->its_vinvall_cmd.vpe; +} + +static struct its_vpe *its_build_vmapp_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + unsigned long vpt_addr; + + vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); + + its_encode_cmd(cmd, GITS_CMD_VMAPP); + its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id); + its_encode_valid(cmd, desc->its_vmapp_cmd.valid); + its_encode_target(cmd, desc->its_vmapp_cmd.col->target_address); + its_encode_vpt_addr(cmd, vpt_addr); + its_encode_vpt_size(cmd, LPI_NRBITS - 1); + + its_fixup_cmd(cmd); + + return desc->its_vmapp_cmd.vpe; +} + +static struct its_vpe *its_build_vmapti_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + u32 db; + + if (desc->its_vmapti_cmd.db_enabled) + db = desc->its_vmapti_cmd.vpe->vpe_db_lpi; + else + db = 1023; + + its_encode_cmd(cmd, GITS_CMD_VMAPTI); + its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id); + its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id); + its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id); + its_encode_db_phys_id(cmd, db); + its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id); + + its_fixup_cmd(cmd); + + return desc->its_vmapti_cmd.vpe; +} + +static struct its_vpe *its_build_vmovi_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + u32 db; + + if (desc->its_vmovi_cmd.db_enabled) + db = desc->its_vmovi_cmd.vpe->vpe_db_lpi; + else + db = 1023; + + its_encode_cmd(cmd, GITS_CMD_VMOVI); + its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id); + its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id); + its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id); + its_encode_db_phys_id(cmd, db); + its_encode_db_valid(cmd, true); + + its_fixup_cmd(cmd); + + return desc->its_vmovi_cmd.vpe; +} + +static struct its_vpe *its_build_vmovp_cmd(struct its_cmd_block *cmd, + struct its_cmd_desc *desc) +{ + its_encode_cmd(cmd, GITS_CMD_VMOVP); + its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num); + its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list); + its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id); + its_encode_target(cmd, desc->its_vmovp_cmd.col->target_address); + + its_fixup_cmd(cmd); + + return desc->its_vmovp_cmd.vpe; +} + static u64 its_cmd_ptr_to_offset(struct its_node *its, struct its_cmd_block *ptr) { @@ -453,7 +696,13 @@ static void its_wait_for_range_completion(struct its_node *its, while (1) { rd_idx = readl_relaxed(its->base + GITS_CREADR); - if (rd_idx >= to_idx || rd_idx < from_idx) + + /* Direct case */ + if (from_idx < to_idx && rd_idx >= to_idx) + break; + + /* Wrapped case */ + if (from_idx >= to_idx && rd_idx >= to_idx && rd_idx < from_idx) break; count--; @@ -466,42 +715,84 @@ static void its_wait_for_range_completion(struct its_node *its, } } -static void its_send_single_command(struct its_node *its, - its_cmd_builder_t builder, - struct its_cmd_desc *desc) +/* Warning, macro hell follows */ +#define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \ +void name(struct its_node *its, \ + buildtype builder, \ + struct its_cmd_desc *desc) \ +{ \ + struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \ + synctype *sync_obj; \ + unsigned long flags; \ + \ + raw_spin_lock_irqsave(&its->lock, flags); \ + \ + cmd = its_allocate_entry(its); \ + if (!cmd) { /* We're soooooo screewed... */ \ + raw_spin_unlock_irqrestore(&its->lock, flags); \ + return; \ + } \ + sync_obj = builder(cmd, desc); \ + its_flush_cmd(its, cmd); \ + \ + if (sync_obj) { \ + sync_cmd = its_allocate_entry(its); \ + if (!sync_cmd) \ + goto post; \ + \ + buildfn(sync_cmd, sync_obj); \ + its_flush_cmd(its, sync_cmd); \ + } \ + \ +post: \ + next_cmd = its_post_commands(its); \ + raw_spin_unlock_irqrestore(&its->lock, flags); \ + \ + its_wait_for_range_completion(its, cmd, next_cmd); \ +} + +static void its_build_sync_cmd(struct its_cmd_block *sync_cmd, + struct its_collection *sync_col) +{ + its_encode_cmd(sync_cmd, GITS_CMD_SYNC); + its_encode_target(sync_cmd, sync_col->target_address); + + its_fixup_cmd(sync_cmd); +} + +static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t, + struct its_collection, its_build_sync_cmd) + +static void its_build_vsync_cmd(struct its_cmd_block *sync_cmd, + struct its_vpe *sync_vpe) +{ + its_encode_cmd(sync_cmd, GITS_CMD_VSYNC); + its_encode_vpeid(sync_cmd, sync_vpe->vpe_id); + + its_fixup_cmd(sync_cmd); +} + +static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t, + struct its_vpe, its_build_vsync_cmd) + +static void its_send_int(struct its_device *dev, u32 event_id) { - struct its_cmd_block *cmd, *sync_cmd, *next_cmd; - struct its_collection *sync_col; - unsigned long flags; + struct its_cmd_desc desc; - raw_spin_lock_irqsave(&its->lock, flags); + desc.its_int_cmd.dev = dev; + desc.its_int_cmd.event_id = event_id; - cmd = its_allocate_entry(its); - if (!cmd) { /* We're soooooo screewed... */ - pr_err_ratelimited("ITS can't allocate, dropping command\n"); - raw_spin_unlock_irqrestore(&its->lock, flags); - return; - } - sync_col = builder(cmd, desc); - its_flush_cmd(its, cmd); + its_send_single_command(dev->its, its_build_int_cmd, &desc); +} - if (sync_col) { - sync_cmd = its_allocate_entry(its); - if (!sync_cmd) { - pr_err_ratelimited("ITS can't SYNC, skipping\n"); - goto post; - } - its_encode_cmd(sync_cmd, GITS_CMD_SYNC); - its_encode_target(sync_cmd, sync_col->target_address); - its_fixup_cmd(sync_cmd); - its_flush_cmd(its, sync_cmd); - } +static void its_send_clear(struct its_device *dev, u32 event_id) +{ + struct its_cmd_desc desc; -post: - next_cmd = its_post_commands(its); - raw_spin_unlock_irqrestore(&its->lock, flags); + desc.its_clear_cmd.dev = dev; + desc.its_clear_cmd.event_id = event_id; - its_wait_for_range_completion(its, cmd, next_cmd); + its_send_single_command(dev->its, its_build_clear_cmd, &desc); } static void its_send_inv(struct its_device *dev, u32 event_id) @@ -577,6 +868,106 @@ static void its_send_invall(struct its_node *its, struct its_collection *col) its_send_single_command(its, its_build_invall_cmd, &desc); } +static void its_send_vmapti(struct its_device *dev, u32 id) +{ + struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id]; + struct its_cmd_desc desc; + + desc.its_vmapti_cmd.vpe = map->vpe; + desc.its_vmapti_cmd.dev = dev; + desc.its_vmapti_cmd.virt_id = map->vintid; + desc.its_vmapti_cmd.event_id = id; + desc.its_vmapti_cmd.db_enabled = map->db_enabled; + + its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc); +} + +static void its_send_vmovi(struct its_device *dev, u32 id) +{ + struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id]; + struct its_cmd_desc desc; + + desc.its_vmovi_cmd.vpe = map->vpe; + desc.its_vmovi_cmd.dev = dev; + desc.its_vmovi_cmd.event_id = id; + desc.its_vmovi_cmd.db_enabled = map->db_enabled; + + its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc); +} + +static void its_send_vmapp(struct its_vpe *vpe, bool valid) +{ + struct its_cmd_desc desc; + struct its_node *its; + + desc.its_vmapp_cmd.vpe = vpe; + desc.its_vmapp_cmd.valid = valid; + + list_for_each_entry(its, &its_nodes, entry) { + if (!its->is_v4) + continue; + + desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx]; + its_send_single_vcommand(its, its_build_vmapp_cmd, &desc); + } +} + +static void its_send_vmovp(struct its_vpe *vpe) +{ + struct its_cmd_desc desc; + struct its_node *its; + unsigned long flags; + int col_id = vpe->col_idx; + + desc.its_vmovp_cmd.vpe = vpe; + desc.its_vmovp_cmd.its_list = (u16)its_list_map; + + if (!its_list_map) { + its = list_first_entry(&its_nodes, struct its_node, entry); + desc.its_vmovp_cmd.seq_num = 0; + desc.its_vmovp_cmd.col = &its->collections[col_id]; + its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); + return; + } + + /* + * Yet another marvel of the architecture. If using the + * its_list "feature", we need to make sure that all ITSs + * receive all VMOVP commands in the same order. The only way + * to guarantee this is to make vmovp a serialization point. + * + * Wall <-- Head. + */ + raw_spin_lock_irqsave(&vmovp_lock, flags); + + desc.its_vmovp_cmd.seq_num = vmovp_seq_num++; + + /* Emit VMOVPs */ + list_for_each_entry(its, &its_nodes, entry) { + if (!its->is_v4) + continue; + + desc.its_vmovp_cmd.col = &its->collections[col_id]; + its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); + } + + raw_spin_unlock_irqrestore(&vmovp_lock, flags); +} + +static void its_send_vinvall(struct its_vpe *vpe) +{ + struct its_cmd_desc desc; + struct its_node *its; + + desc.its_vinvall_cmd.vpe = vpe; + + list_for_each_entry(its, &its_nodes, entry) { + if (!its->is_v4) + continue; + its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); + } +} + /* * irqchip functions - assumes MSI, mostly. */ @@ -587,17 +978,26 @@ static inline u32 its_get_event_id(struct irq_data *d) return d->hwirq - its_dev->event_map.lpi_base; } -static void lpi_set_config(struct irq_data *d, bool enable) +static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) { - struct its_device *its_dev = irq_data_get_irq_chip_data(d); - irq_hw_number_t hwirq = d->hwirq; - u32 id = its_get_event_id(d); - u8 *cfg = page_address(gic_rdists->prop_page) + hwirq - 8192; + irq_hw_number_t hwirq; + struct page *prop_page; + u8 *cfg; - if (enable) - *cfg |= LPI_PROP_ENABLED; - else - *cfg &= ~LPI_PROP_ENABLED; + if (irqd_is_forwarded_to_vcpu(d)) { + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); + + prop_page = its_dev->event_map.vm->vprop_page; + hwirq = its_dev->event_map.vlpi_maps[event].vintid; + } else { + prop_page = gic_rdists->prop_page; + hwirq = d->hwirq; + } + + cfg = page_address(prop_page) + hwirq - 8192; + *cfg &= ~clr; + *cfg |= set | LPI_PROP_GROUP1; /* * Make the above write visible to the redistributors. @@ -608,17 +1008,53 @@ static void lpi_set_config(struct irq_data *d, bool enable) gic_flush_dcache_to_poc(cfg, sizeof(*cfg)); else dsb(ishst); - its_send_inv(its_dev, id); +} + +static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + + lpi_write_config(d, clr, set); + its_send_inv(its_dev, its_get_event_id(d)); +} + +static void its_vlpi_set_doorbell(struct irq_data *d, bool enable) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); + + if (its_dev->event_map.vlpi_maps[event].db_enabled == enable) + return; + + its_dev->event_map.vlpi_maps[event].db_enabled = enable; + + /* + * More fun with the architecture: + * + * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI + * value or to 1023, depending on the enable bit. But that + * would be issueing a mapping for an /existing/ DevID+EventID + * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI + * to the /same/ vPE, using this opportunity to adjust the + * doorbell. Mouahahahaha. We loves it, Precious. + */ + its_send_vmovi(its_dev, event); } static void its_mask_irq(struct irq_data *d) { - lpi_set_config(d, false); + if (irqd_is_forwarded_to_vcpu(d)) + its_vlpi_set_doorbell(d, false); + + lpi_update_config(d, LPI_PROP_ENABLED, 0); } static void its_unmask_irq(struct irq_data *d) { - lpi_set_config(d, true); + if (irqd_is_forwarded_to_vcpu(d)) + its_vlpi_set_doorbell(d, true); + + lpi_update_config(d, 0, LPI_PROP_ENABLED); } static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, @@ -630,6 +1066,10 @@ static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, struct its_collection *target_col; u32 id = its_get_event_id(d); + /* A forwarded interrupt should use irq_set_vcpu_affinity */ + if (irqd_is_forwarded_to_vcpu(d)) + return -EINVAL; + /* lpi cannot be routed to a redistributor that is on a foreign node */ if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { if (its_dev->its->numa_node >= 0) { @@ -671,6 +1111,179 @@ static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) iommu_dma_map_msi_msg(d->irq, msg); } +static int its_irq_set_irqchip_state(struct irq_data *d, + enum irqchip_irq_state which, + bool state) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); + + if (which != IRQCHIP_STATE_PENDING) + return -EINVAL; + + if (state) + its_send_int(its_dev, event); + else + its_send_clear(its_dev, event); + + return 0; +} + +static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); + int ret = 0; + + if (!info->map) + return -EINVAL; + + mutex_lock(&its_dev->event_map.vlpi_lock); + + if (!its_dev->event_map.vm) { + struct its_vlpi_map *maps; + + maps = kzalloc(sizeof(*maps) * its_dev->event_map.nr_lpis, + GFP_KERNEL); + if (!maps) { + ret = -ENOMEM; + goto out; + } + + its_dev->event_map.vm = info->map->vm; + its_dev->event_map.vlpi_maps = maps; + } else if (its_dev->event_map.vm != info->map->vm) { + ret = -EINVAL; + goto out; + } + + /* Get our private copy of the mapping information */ + its_dev->event_map.vlpi_maps[event] = *info->map; + + if (irqd_is_forwarded_to_vcpu(d)) { + /* Already mapped, move it around */ + its_send_vmovi(its_dev, event); + } else { + /* Drop the physical mapping */ + its_send_discard(its_dev, event); + + /* and install the virtual one */ + its_send_vmapti(its_dev, event); + irqd_set_forwarded_to_vcpu(d); + + /* Increment the number of VLPIs */ + its_dev->event_map.nr_vlpis++; + } + +out: + mutex_unlock(&its_dev->event_map.vlpi_lock); + return ret; +} + +static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); + int ret = 0; + + mutex_lock(&its_dev->event_map.vlpi_lock); + + if (!its_dev->event_map.vm || + !its_dev->event_map.vlpi_maps[event].vm) { + ret = -EINVAL; + goto out; + } + + /* Copy our mapping information to the incoming request */ + *info->map = its_dev->event_map.vlpi_maps[event]; + +out: + mutex_unlock(&its_dev->event_map.vlpi_lock); + return ret; +} + +static int its_vlpi_unmap(struct irq_data *d) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + u32 event = its_get_event_id(d); + int ret = 0; + + mutex_lock(&its_dev->event_map.vlpi_lock); + + if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) { + ret = -EINVAL; + goto out; + } + + /* Drop the virtual mapping */ + its_send_discard(its_dev, event); + + /* and restore the physical one */ + irqd_clr_forwarded_to_vcpu(d); + its_send_mapti(its_dev, d->hwirq, event); + lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO | + LPI_PROP_ENABLED | + LPI_PROP_GROUP1)); + + /* + * Drop the refcount and make the device available again if + * this was the last VLPI. + */ + if (!--its_dev->event_map.nr_vlpis) { + its_dev->event_map.vm = NULL; + kfree(its_dev->event_map.vlpi_maps); + } + +out: + mutex_unlock(&its_dev->event_map.vlpi_lock); + return ret; +} + +static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + + if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) + return -EINVAL; + + if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI) + lpi_update_config(d, 0xff, info->config); + else + lpi_write_config(d, 0xff, info->config); + its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED)); + + return 0; +} + +static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + struct its_cmd_info *info = vcpu_info; + + /* Need a v4 ITS */ + if (!its_dev->its->is_v4) + return -EINVAL; + + /* Unmap request? */ + if (!info) + return its_vlpi_unmap(d); + + switch (info->cmd_type) { + case MAP_VLPI: + return its_vlpi_map(d, info); + + case GET_VLPI: + return its_vlpi_get(d, info); + + case PROP_UPDATE_VLPI: + case PROP_UPDATE_AND_INV_VLPI: + return its_vlpi_prop_update(d, info); + + default: + return -EINVAL; + } +} + static struct irq_chip its_irq_chip = { .name = "ITS", .irq_mask = its_mask_irq, @@ -678,6 +1291,8 @@ static struct irq_chip its_irq_chip = { .irq_eoi = irq_chip_eoi_parent, .irq_set_affinity = its_set_affinity, .irq_compose_msi_msg = its_irq_compose_msi_msg, + .irq_set_irqchip_state = its_irq_set_irqchip_state, + .irq_set_vcpu_affinity = its_irq_set_vcpu_affinity, }; /* @@ -696,7 +1311,6 @@ static struct irq_chip its_irq_chip = { static unsigned long *lpi_bitmap; static u32 lpi_chunks; -static u32 lpi_id_bits; static DEFINE_SPINLOCK(lpi_lock); static int its_lpi_to_chunk(int lpi) @@ -767,16 +1381,15 @@ out: return bitmap; } -static void its_lpi_free(struct event_lpi_map *map) +static void its_lpi_free_chunks(unsigned long *bitmap, int base, int nr_ids) { - int base = map->lpi_base; - int nr_ids = map->nr_lpis; int lpi; spin_lock(&lpi_lock); for (lpi = base; lpi < (base + nr_ids); lpi += IRQS_PER_CHUNK) { int chunk = its_lpi_to_chunk(lpi); + BUG_ON(chunk > lpi_chunks); if (test_bit(chunk, lpi_bitmap)) { clear_bit(chunk, lpi_bitmap); @@ -787,28 +1400,40 @@ static void its_lpi_free(struct event_lpi_map *map) spin_unlock(&lpi_lock); - kfree(map->lpi_map); - kfree(map->col_map); + kfree(bitmap); } -/* - * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to - * deal with (one configuration byte per interrupt). PENDBASE has to - * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI). - */ -#define LPI_NRBITS lpi_id_bits -#define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K) -#define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K) +static struct page *its_allocate_prop_table(gfp_t gfp_flags) +{ + struct page *prop_page; -#define LPI_PROP_DEFAULT_PRIO 0xa0 + prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ)); + if (!prop_page) + return NULL; + + /* Priority 0xa0, Group-1, disabled */ + memset(page_address(prop_page), + LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, + LPI_PROPBASE_SZ); + + /* Make sure the GIC will observe the written configuration */ + gic_flush_dcache_to_poc(page_address(prop_page), LPI_PROPBASE_SZ); + + return prop_page; +} + +static void its_free_prop_table(struct page *prop_page) +{ + free_pages((unsigned long)page_address(prop_page), + get_order(LPI_PROPBASE_SZ)); +} static int __init its_alloc_lpi_tables(void) { phys_addr_t paddr; lpi_id_bits = min_t(u32, gic_rdists->id_bits, ITS_MAX_LPI_NRBITS); - gic_rdists->prop_page = alloc_pages(GFP_NOWAIT, - get_order(LPI_PROPBASE_SZ)); + gic_rdists->prop_page = its_allocate_prop_table(GFP_NOWAIT); if (!gic_rdists->prop_page) { pr_err("Failed to allocate PROPBASE\n"); return -ENOMEM; @@ -817,14 +1442,6 @@ static int __init its_alloc_lpi_tables(void) paddr = page_to_phys(gic_rdists->prop_page); pr_info("GIC: using LPI property table @%pa\n", &paddr); - /* Priority 0xa0, Group-1, disabled */ - memset(page_address(gic_rdists->prop_page), - LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, - LPI_PROPBASE_SZ); - - /* Make sure the GIC will observe the written configuration */ - gic_flush_dcache_to_poc(page_address(gic_rdists->prop_page), LPI_PROPBASE_SZ); - return its_lpi_init(lpi_id_bits); } @@ -963,10 +1580,13 @@ retry_baser: return 0; } -static bool its_parse_baser_device(struct its_node *its, struct its_baser *baser, - u32 psz, u32 *order) +static bool its_parse_indirect_baser(struct its_node *its, + struct its_baser *baser, + u32 psz, u32 *order) { - u64 esz = GITS_BASER_ENTRY_SIZE(its_read_baser(its, baser)); + u64 tmp = its_read_baser(its, baser); + u64 type = GITS_BASER_TYPE(tmp); + u64 esz = GITS_BASER_ENTRY_SIZE(tmp); u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb; u32 ids = its->device_ids; u32 new_order = *order; @@ -1005,8 +1625,9 @@ static bool its_parse_baser_device(struct its_node *its, struct its_baser *baser if (new_order >= MAX_ORDER) { new_order = MAX_ORDER - 1; ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz); - pr_warn("ITS@%pa: Device Table too large, reduce ids %u->%u\n", - &its->phys_base, its->device_ids, ids); + pr_warn("ITS@%pa: %s Table too large, reduce ids %u->%u\n", + &its->phys_base, its_base_type_string[type], + its->device_ids, ids); } *order = new_order; @@ -1054,11 +1675,16 @@ static int its_alloc_tables(struct its_node *its) u32 order = get_order(psz); bool indirect = false; - if (type == GITS_BASER_TYPE_NONE) + switch (type) { + case GITS_BASER_TYPE_NONE: continue; - if (type == GITS_BASER_TYPE_DEVICE) - indirect = its_parse_baser_device(its, baser, psz, &order); + case GITS_BASER_TYPE_DEVICE: + case GITS_BASER_TYPE_VCPU: + indirect = its_parse_indirect_baser(its, baser, + psz, &order); + break; + } err = its_setup_baser(its, baser, cache, shr, psz, order, indirect); if (err < 0) { @@ -1085,6 +1711,30 @@ static int its_alloc_collections(struct its_node *its) return 0; } +static struct page *its_allocate_pending_table(gfp_t gfp_flags) +{ + struct page *pend_page; + /* + * The pending pages have to be at least 64kB aligned, + * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below. + */ + pend_page = alloc_pages(gfp_flags | __GFP_ZERO, + get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K))); + if (!pend_page) + return NULL; + + /* Make sure the GIC will observe the zero-ed page */ + gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ); + + return pend_page; +} + +static void its_free_pending_table(struct page *pt) +{ + free_pages((unsigned long)page_address(pt), + get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K))); +} + static void its_cpu_init_lpis(void) { void __iomem *rbase = gic_data_rdist_rd_base(); @@ -1095,21 +1745,14 @@ static void its_cpu_init_lpis(void) pend_page = gic_data_rdist()->pend_page; if (!pend_page) { phys_addr_t paddr; - /* - * The pending pages have to be at least 64kB aligned, - * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below. - */ - pend_page = alloc_pages(GFP_NOWAIT | __GFP_ZERO, - get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K))); + + pend_page = its_allocate_pending_table(GFP_NOWAIT); if (!pend_page) { pr_err("Failed to allocate PENDBASE for CPU%d\n", smp_processor_id()); return; } - /* Make sure the GIC will observe the zero-ed page */ - gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ); - paddr = page_to_phys(pend_page); pr_info("CPU%d: using LPI pending table @%pa\n", smp_processor_id(), &paddr); @@ -1260,26 +1903,19 @@ static struct its_baser *its_get_baser(struct its_node *its, u32 type) return NULL; } -static bool its_alloc_device_table(struct its_node *its, u32 dev_id) +static bool its_alloc_table_entry(struct its_baser *baser, u32 id) { - struct its_baser *baser; struct page *page; u32 esz, idx; __le64 *table; - baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE); - - /* Don't allow device id that exceeds ITS hardware limit */ - if (!baser) - return (ilog2(dev_id) < its->device_ids); - /* Don't allow device id that exceeds single, flat table limit */ esz = GITS_BASER_ENTRY_SIZE(baser->val); if (!(baser->val & GITS_BASER_INDIRECT)) - return (dev_id < (PAGE_ORDER_TO_SIZE(baser->order) / esz)); + return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz)); /* Compute 1st level table index & check if that exceeds table limit */ - idx = dev_id >> ilog2(baser->psz / esz); + idx = id >> ilog2(baser->psz / esz); if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE)) return false; @@ -1308,11 +1944,52 @@ static bool its_alloc_device_table(struct its_node *its, u32 dev_id) return true; } +static bool its_alloc_device_table(struct its_node *its, u32 dev_id) +{ + struct its_baser *baser; + + baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE); + + /* Don't allow device id that exceeds ITS hardware limit */ + if (!baser) + return (ilog2(dev_id) < its->device_ids); + + return its_alloc_table_entry(baser, dev_id); +} + +static bool its_alloc_vpe_table(u32 vpe_id) +{ + struct its_node *its; + + /* + * Make sure the L2 tables are allocated on *all* v4 ITSs. We + * could try and only do it on ITSs corresponding to devices + * that have interrupts targeted at this VPE, but the + * complexity becomes crazy (and you have tons of memory + * anyway, right?). + */ + list_for_each_entry(its, &its_nodes, entry) { |