diff options
Diffstat (limited to 'virt/kvm/arm/vgic/vgic-its.c')
| -rw-r--r-- | virt/kvm/arm/vgic/vgic-its.c | 2783 |
1 files changed, 0 insertions, 2783 deletions
diff --git a/virt/kvm/arm/vgic/vgic-its.c b/virt/kvm/arm/vgic/vgic-its.c deleted file mode 100644 index c012a52b19f5..000000000000 --- a/virt/kvm/arm/vgic/vgic-its.c +++ /dev/null @@ -1,2783 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * GICv3 ITS emulation - * - * Copyright (C) 2015,2016 ARM Ltd. - * Author: Andre Przywara <andre.przywara@arm.com> - */ - -#include <linux/cpu.h> -#include <linux/kvm.h> -#include <linux/kvm_host.h> -#include <linux/interrupt.h> -#include <linux/list.h> -#include <linux/uaccess.h> -#include <linux/list_sort.h> - -#include <linux/irqchip/arm-gic-v3.h> - -#include <asm/kvm_emulate.h> -#include <asm/kvm_arm.h> -#include <asm/kvm_mmu.h> - -#include "vgic.h" -#include "vgic-mmio.h" - -static int vgic_its_save_tables_v0(struct vgic_its *its); -static int vgic_its_restore_tables_v0(struct vgic_its *its); -static int vgic_its_commit_v0(struct vgic_its *its); -static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq, - struct kvm_vcpu *filter_vcpu, bool needs_inv); - -/* - * Creates a new (reference to a) struct vgic_irq for a given LPI. - * If this LPI is already mapped on another ITS, we increase its refcount - * and return a pointer to the existing structure. - * If this is a "new" LPI, we allocate and initialize a new struct vgic_irq. - * This function returns a pointer to the _unlocked_ structure. - */ -static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid, - struct kvm_vcpu *vcpu) -{ - struct vgic_dist *dist = &kvm->arch.vgic; - struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid), *oldirq; - unsigned long flags; - int ret; - - /* In this case there is no put, since we keep the reference. */ - if (irq) - return irq; - - irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL); - if (!irq) - return ERR_PTR(-ENOMEM); - - INIT_LIST_HEAD(&irq->lpi_list); - INIT_LIST_HEAD(&irq->ap_list); - raw_spin_lock_init(&irq->irq_lock); - - irq->config = VGIC_CONFIG_EDGE; - kref_init(&irq->refcount); - irq->intid = intid; - irq->target_vcpu = vcpu; - irq->group = 1; - - raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); - - /* - * There could be a race with another vgic_add_lpi(), so we need to - * check that we don't add a second list entry with the same LPI. - */ - list_for_each_entry(oldirq, &dist->lpi_list_head, lpi_list) { - if (oldirq->intid != intid) - continue; - - /* Someone was faster with adding this LPI, lets use that. */ - kfree(irq); - irq = oldirq; - - /* - * This increases the refcount, the caller is expected to - * call vgic_put_irq() on the returned pointer once it's - * finished with the IRQ. - */ - vgic_get_irq_kref(irq); - - goto out_unlock; - } - - list_add_tail(&irq->lpi_list, &dist->lpi_list_head); - dist->lpi_list_count++; - -out_unlock: - raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); - - /* - * We "cache" the configuration table entries in our struct vgic_irq's. - * However we only have those structs for mapped IRQs, so we read in - * the respective config data from memory here upon mapping the LPI. - * - * Should any of these fail, behave as if we couldn't create the LPI - * by dropping the refcount and returning the error. - */ - ret = update_lpi_config(kvm, irq, NULL, false); - if (ret) { - vgic_put_irq(kvm, irq); - return ERR_PTR(ret); - } - - ret = vgic_v3_lpi_sync_pending_status(kvm, irq); - if (ret) { - vgic_put_irq(kvm, irq); - return ERR_PTR(ret); - } - - return irq; -} - -struct its_device { - struct list_head dev_list; - - /* the head for the list of ITTEs */ - struct list_head itt_head; - u32 num_eventid_bits; - gpa_t itt_addr; - u32 device_id; -}; - -#define COLLECTION_NOT_MAPPED ((u32)~0) - -struct its_collection { - struct list_head coll_list; - - u32 collection_id; - u32 target_addr; -}; - -#define its_is_collection_mapped(coll) ((coll) && \ - ((coll)->target_addr != COLLECTION_NOT_MAPPED)) - -struct its_ite { - struct list_head ite_list; - - struct vgic_irq *irq; - struct its_collection *collection; - u32 event_id; -}; - -struct vgic_translation_cache_entry { - struct list_head entry; - phys_addr_t db; - u32 devid; - u32 eventid; - struct vgic_irq *irq; -}; - -/** - * struct vgic_its_abi - ITS abi ops and settings - * @cte_esz: collection table entry size - * @dte_esz: device table entry size - * @ite_esz: interrupt translation table entry size - * @save tables: save the ITS tables into guest RAM - * @restore_tables: restore the ITS internal structs from tables - * stored in guest RAM - * @commit: initialize the registers which expose the ABI settings, - * especially the entry sizes - */ -struct vgic_its_abi { - int cte_esz; - int dte_esz; - int ite_esz; - int (*save_tables)(struct vgic_its *its); - int (*restore_tables)(struct vgic_its *its); - int (*commit)(struct vgic_its *its); -}; - -#define ABI_0_ESZ 8 -#define ESZ_MAX ABI_0_ESZ - -static const struct vgic_its_abi its_table_abi_versions[] = { - [0] = { - .cte_esz = ABI_0_ESZ, - .dte_esz = ABI_0_ESZ, - .ite_esz = ABI_0_ESZ, - .save_tables = vgic_its_save_tables_v0, - .restore_tables = vgic_its_restore_tables_v0, - .commit = vgic_its_commit_v0, - }, -}; - -#define NR_ITS_ABIS ARRAY_SIZE(its_table_abi_versions) - -inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its) -{ - return &its_table_abi_versions[its->abi_rev]; -} - -static int vgic_its_set_abi(struct vgic_its *its, u32 rev) -{ - const struct vgic_its_abi *abi; - - its->abi_rev = rev; - abi = vgic_its_get_abi(its); - return abi->commit(its); -} - -/* - * Find and returns a device in the device table for an ITS. - * Must be called with the its_lock mutex held. - */ -static struct its_device *find_its_device(struct vgic_its *its, u32 device_id) -{ - struct its_device *device; - - list_for_each_entry(device, &its->device_list, dev_list) - if (device_id == device->device_id) - return device; - - return NULL; -} - -/* - * Find and returns an interrupt translation table entry (ITTE) for a given - * Device ID/Event ID pair on an ITS. - * Must be called with the its_lock mutex held. - */ -static struct its_ite *find_ite(struct vgic_its *its, u32 device_id, - u32 event_id) -{ - struct its_device *device; - struct its_ite *ite; - - device = find_its_device(its, device_id); - if (device == NULL) - return NULL; - - list_for_each_entry(ite, &device->itt_head, ite_list) - if (ite->event_id == event_id) - return ite; - - return NULL; -} - -/* To be used as an iterator this macro misses the enclosing parentheses */ -#define for_each_lpi_its(dev, ite, its) \ - list_for_each_entry(dev, &(its)->device_list, dev_list) \ - list_for_each_entry(ite, &(dev)->itt_head, ite_list) - -#define GIC_LPI_OFFSET 8192 - -#define VITS_TYPER_IDBITS 16 -#define VITS_TYPER_DEVBITS 16 -#define VITS_DTE_MAX_DEVID_OFFSET (BIT(14) - 1) -#define VITS_ITE_MAX_EVENTID_OFFSET (BIT(16) - 1) - -/* - * Finds and returns a collection in the ITS collection table. - * Must be called with the its_lock mutex held. - */ -static struct its_collection *find_collection(struct vgic_its *its, int coll_id) -{ - struct its_collection *collection; - - list_for_each_entry(collection, &its->collection_list, coll_list) { - if (coll_id == collection->collection_id) - return collection; - } - - return NULL; -} - -#define LPI_PROP_ENABLE_BIT(p) ((p) & LPI_PROP_ENABLED) -#define LPI_PROP_PRIORITY(p) ((p) & 0xfc) - -/* - * Reads the configuration data for a given LPI from guest memory and - * updates the fields in struct vgic_irq. - * If filter_vcpu is not NULL, applies only if the IRQ is targeting this - * VCPU. Unconditionally applies if filter_vcpu is NULL. - */ -static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq, - struct kvm_vcpu *filter_vcpu, bool needs_inv) -{ - u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser); - u8 prop; - int ret; - unsigned long flags; - - ret = kvm_read_guest_lock(kvm, propbase + irq->intid - GIC_LPI_OFFSET, - &prop, 1); - - if (ret) - return ret; - - raw_spin_lock_irqsave(&irq->irq_lock, flags); - - if (!filter_vcpu || filter_vcpu == irq->target_vcpu) { - irq->priority = LPI_PROP_PRIORITY(prop); - irq->enabled = LPI_PROP_ENABLE_BIT(prop); - - if (!irq->hw) { - vgic_queue_irq_unlock(kvm, irq, flags); - return 0; - } - } - - raw_spin_unlock_irqrestore(&irq->irq_lock, flags); - - if (irq->hw) - return its_prop_update_vlpi(irq->host_irq, prop, needs_inv); - - return 0; -} - -/* - * Create a snapshot of the current LPIs targeting @vcpu, so that we can - * enumerate those LPIs without holding any lock. - * Returns their number and puts the kmalloc'ed array into intid_ptr. - */ -int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr) -{ - struct vgic_dist *dist = &kvm->arch.vgic; - struct vgic_irq *irq; - unsigned long flags; - u32 *intids; - int irq_count, i = 0; - - /* - * There is an obvious race between allocating the array and LPIs - * being mapped/unmapped. If we ended up here as a result of a - * command, we're safe (locks are held, preventing another - * command). If coming from another path (such as enabling LPIs), - * we must be careful not to overrun the array. - */ - irq_count = READ_ONCE(dist->lpi_list_count); - intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL); - if (!intids) - return -ENOMEM; - - raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); - list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) { - if (i == irq_count) - break; - /* We don't need to "get" the IRQ, as we hold the list lock. */ - if (vcpu && irq->target_vcpu != vcpu) - continue; - intids[i++] = irq->intid; - } - raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); - - *intid_ptr = intids; - return i; -} - -static int update_affinity(struct vgic_irq *irq, struct kvm_vcpu *vcpu) -{ - int ret = 0; - unsigned long flags; - - raw_spin_lock_irqsave(&irq->irq_lock, flags); - irq->target_vcpu = vcpu; - raw_spin_unlock_irqrestore(&irq->irq_lock, flags); - - if (irq->hw) { - struct its_vlpi_map map; - - ret = its_get_vlpi(irq->host_irq, &map); - if (ret) - return ret; - - if (map.vpe) - atomic_dec(&map.vpe->vlpi_count); - map.vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe; - atomic_inc(&map.vpe->vlpi_count); - - ret = its_map_vlpi(irq->host_irq, &map); - } - - return ret; -} - -/* - * Promotes the ITS view of affinity of an ITTE (which redistributor this LPI - * is targeting) to the VGIC's view, which deals with target VCPUs. - * Needs to be called whenever either the collection for a LPIs has - * changed or the collection itself got retargeted. - */ -static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite) -{ - struct kvm_vcpu *vcpu; - - if (!its_is_collection_mapped(ite->collection)) - return; - - vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr); - update_affinity(ite->irq, vcpu); -} - -/* - * Updates the target VCPU for every LPI targeting this collection. - * Must be called with the its_lock mutex held. - */ -static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its, - struct its_collection *coll) -{ - struct its_device *device; - struct its_ite *ite; - - for_each_lpi_its(device, ite, its) { - if (!ite->collection || coll != ite->collection) - continue; - - update_affinity_ite(kvm, ite); - } -} - -static u32 max_lpis_propbaser(u64 propbaser) -{ - int nr_idbits = (propbaser & 0x1f) + 1; - - return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS); -} - -/* - * Sync the pending table pending bit of LPIs targeting @vcpu - * with our own data structures. This relies on the LPI being - * mapped before. - */ -static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu) -{ - gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser); - struct vgic_irq *irq; - int last_byte_offset = -1; - int ret = 0; - u32 *intids; - int nr_irqs, i; - unsigned long flags; - u8 pendmask; - - nr_irqs = vgic_copy_lpi_list(vcpu->kvm, vcpu, &intids); - if (nr_irqs < 0) - return nr_irqs; - - for (i = 0; i < nr_irqs; i++) { - int byte_offset, bit_nr; - - byte_offset = intids[i] / BITS_PER_BYTE; - bit_nr = intids[i] % BITS_PER_BYTE; - - /* - * For contiguously allocated LPIs chances are we just read - * this very same byte in the last iteration. Reuse that. - */ - if (byte_offset != last_byte_offset) { - ret = kvm_read_guest_lock(vcpu->kvm, - pendbase + byte_offset, - &pendmask, 1); - if (ret) { - kfree(intids); - return ret; - } - last_byte_offset = byte_offset; - } - - irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]); - raw_spin_lock_irqsave(&irq->irq_lock, flags); - irq->pending_latch = pendmask & (1U << bit_nr); - vgic_queue_irq_unlock(vcpu->kvm, irq, flags); - vgic_put_irq(vcpu->kvm, irq); - } - - kfree(intids); - - return ret; -} - -static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm, - struct vgic_its *its, - gpa_t addr, unsigned int len) -{ - const struct vgic_its_abi *abi = vgic_its_get_abi(its); - u64 reg = GITS_TYPER_PLPIS; - - /* - * We use linear CPU numbers for redistributor addressing, - * so GITS_TYPER.PTA is 0. - * Also we force all PROPBASER registers to be the same, so - * CommonLPIAff is 0 as well. - * To avoid memory waste in the guest, we keep the number of IDBits and - * DevBits low - as least for the time being. - */ - reg |= GIC_ENCODE_SZ(VITS_TYPER_DEVBITS, 5) << GITS_TYPER_DEVBITS_SHIFT; - reg |= GIC_ENCODE_SZ(VITS_TYPER_IDBITS, 5) << GITS_TYPER_IDBITS_SHIFT; - reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT; - - return extract_bytes(reg, addr & 7, len); -} - -static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm, - struct vgic_its *its, - gpa_t addr, unsigned int len) -{ - u32 val; - - val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK; - val |= (PRODUCT_ID_KVM << GITS_IIDR_PRODUCTID_SHIFT) | IMPLEMENTER_ARM; - return val; -} - -static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm, - struct vgic_its *its, - gpa_t addr, unsigned int len, - unsigned long val) -{ - u32 rev = GITS_IIDR_REV(val); - - if (rev >= NR_ITS_ABIS) - return -EINVAL; - return vgic_its_set_abi(its, rev); -} - -static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm, - struct vgic_its *its, - gpa_t addr, unsigned int len) -{ - switch (addr & 0xffff) { - case GITS_PIDR0: - return 0x92; /* part number, bits[7:0] */ - case GITS_PIDR1: - return 0xb4; /* part number, bits[11:8] */ - case GITS_PIDR2: - return GIC_PIDR2_ARCH_GICv3 | 0x0b; - case GITS_PIDR4: - return 0x40; /* This is a 64K software visible page */ - /* The following are the ID registers for (any) GIC. */ - case GITS_CIDR0: - return 0x0d; - case GITS_CIDR1: - return 0xf0; - case GITS_CIDR2: - return 0x05; - case GITS_CIDR3: - return 0xb1; - } - - return 0; -} - -static struct vgic_irq *__vgic_its_check_cache(struct vgic_dist *dist, - phys_addr_t db, - u32 devid, u32 eventid) -{ - struct vgic_translation_cache_entry *cte; - - list_for_each_entry(cte, &dist->lpi_translation_cache, entry) { - /* - * If we hit a NULL entry, there is nothing after this - * point. - */ - if (!cte->irq) - break; - - if (cte->db != db || cte->devid != devid || - cte->eventid != eventid) - continue; - - /* - * Move this entry to the head, as it is the most - * recently used. - */ - if (!list_is_first(&cte->entry, &dist->lpi_translation_cache)) - list_move(&cte->entry, &dist->lpi_translation_cache); - - return cte->irq; - } - - return NULL; -} - -static struct vgic_irq *vgic_its_check_cache(struct kvm *kvm, phys_addr_t db, - u32 devid, u32 eventid) -{ - struct vgic_dist *dist = &kvm->arch.vgic; - struct vgic_irq *irq; - unsigned long flags; - - raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); - irq = __vgic_its_check_cache(dist, db, devid, eventid); - raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); - - return irq; -} - -static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its, - u32 devid, u32 eventid, - struct vgic_irq *irq) -{ - struct vgic_dist *dist = &kvm->arch.vgic; - struct vgic_translation_cache_entry *cte; - unsigned long flags; - phys_addr_t db; - - /* Do not cache a directly injected interrupt */ - if (irq->hw) - return; - - raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); - - if (unlikely(list_empty(&dist->lpi_translation_cache))) - goto out; - - /* - * We could have raced with another CPU caching the same - * translation behind our back, so let's check it is not in - * already - */ - db = its->vgic_its_base + GITS_TRANSLATER; - if (__vgic_its_check_cache(dist, db, devid, eventid)) - goto out; - - /* Always reuse the last entry (LRU policy) */ - cte = list_last_entry(&dist->lpi_translation_cache, - typeof(*cte), entry); - - /* - * Caching the translation implies having an extra reference - * to the interrupt, so drop the potential reference on what - * was in the cache, and increment it on the new interrupt. - */ - if (cte->irq) - __vgic_put_lpi_locked(kvm, cte->irq); - - vgic_get_irq_kref(irq); - - cte->db = db; - cte->devid = devid; - cte->eventid = eventid; - cte->irq = irq; - - /* Move the new translation to the head of the list */ - list_move(&cte->entry, &dist->lpi_translation_cache); - -out: - raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); -} - -void vgic_its_invalidate_cache(struct kvm *kvm) -{ - struct vgic_dist *dist = &kvm->arch.vgic; - struct vgic_translation_cache_entry *cte; - unsigned long flags; - - raw_spin_lock_irqsave(&dist->lpi_list_lock, flags); - - list_for_each_entry(cte, &dist->lpi_translation_cache, entry) { - /* - * If we hit a NULL entry, there is nothing after this - * point. - */ - if (!cte->irq) - break; - - __vgic_put_lpi_locked(kvm, cte->irq); - cte->irq = NULL; - } - - raw_spin_unlock_irqrestore(&dist->lpi_list_lock, flags); -} - -int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its, - u32 devid, u32 eventid, struct vgic_irq **irq) -{ - struct kvm_vcpu *vcpu; - struct its_ite *ite; - - if (!its->enabled) - return -EBUSY; - - ite = find_ite(its, devid, eventid); - if (!ite || !its_is_collection_mapped(ite->collection)) - return E_ITS_INT_UNMAPPED_INTERRUPT; - - vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr); - if (!vcpu) - return E_ITS_INT_UNMAPPED_INTERRUPT; - - if (!vcpu->arch.vgic_cpu.lpis_enabled) - return -EBUSY; - - vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq); - - *irq = ite->irq; - return 0; -} - -struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi) -{ - u64 address; - struct kvm_io_device *kvm_io_dev; - struct vgic_io_device *iodev; - - if (!vgic_has_its(kvm)) - return ERR_PTR(-ENODEV); - - if (!(msi->flags & KVM_MSI_VALID_DEVID)) - return ERR_PTR(-EINVAL); - - address = (u64)msi->address_hi << 32 | msi->address_lo; - - kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, address); - if (!kvm_io_dev) - return ERR_PTR(-EINVAL); - - if (kvm_io_dev->ops != &kvm_io_gic_ops) - return ERR_PTR(-EINVAL); - - iodev = container_of(kvm_io_dev, struct vgic_io_device, dev); - if (iodev->iodev_type != IODEV_ITS) - return ERR_PTR(-EINVAL); - - return iodev->its; -} - -/* - * Find the target VCPU and the LPI number for a given devid/eventid pair - * and make this IRQ pending, possibly injecting it. - * Must be called with the its_lock mutex held. - * Returns 0 on success, a positive error value for any ITS mapping - * related errors and negative error values for generic errors. - */ -static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its, - u32 devid, u32 eventid) -{ - struct vgic_irq *irq = NULL; - unsigned long flags; - int err; - - err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq); - if (err) - return err; - - if (irq->hw) - return irq_set_irqchip_state(irq->host_irq, - IRQCHIP_STATE_PENDING, true); - - raw_spin_lock_irqsave(&irq->irq_lock, flags); - irq->pending_latch = true; - vgic_queue_irq_unlock(kvm, irq, flags); - - return 0; -} - -int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi) -{ - struct vgic_irq *irq; - unsigned long flags; - phys_addr_t db; - - db = (u64)msi->address_hi << 32 | msi->address_lo; - irq = vgic_its_check_cache(kvm, db, msi->devid, msi->data); - - if (!irq) - return -1; - - raw_spin_lock_irqsave(&irq->irq_lock, flags); - irq->pending_latch = true; - vgic_queue_irq_unlock(kvm, irq, flags); - - return 0; -} - -/* - * Queries the KVM IO bus framework to get the ITS pointer from the given - * doorbell address. - * We then call vgic_its_trigger_msi() with the decoded data. - * According to the KVM_SIGNAL_MSI API description returns 1 on success. - */ -int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi) -{ - struct vgic_its *its; - int ret; - - if (!vgic_its_inject_cached_translation(kvm, msi)) - return 1; - - its = vgic_msi_to_its(kvm, msi); - if (IS_ERR(its)) - return PTR_ERR(its); - - mutex_lock(&its->its_lock); - ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data); - mutex_unlock(&its->its_lock); - - if (ret < 0) - return ret; - - /* - * KVM_SIGNAL_MSI demands a return value > 0 for success and 0 - * if the guest has blocked the MSI. So we map any LPI mapping - * related error to that. - */ - if (ret) - return 0; - else - return 1; -} - -/* Requires the its_lock to be held. */ -static void its_free_ite(struct kvm *kvm, struct its_ite *ite) -{ - list_del(&ite->ite_list); - - /* This put matches the get in vgic_add_lpi. */ - if (ite->irq) { - if (ite->irq->hw) - WARN_ON(its_unmap_vlpi(ite->irq->host_irq)); - - vgic_put_irq(kvm, ite->irq); - } - - kfree(ite); -} - -static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size) -{ - return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1); -} - -#define its_cmd_get_command(cmd) its_cmd_mask_field(cmd, 0, 0, 8) -#define its_cmd_get_deviceid(cmd) its_cmd_mask_field(cmd, 0, 32, 32) -#define its_cmd_get_size(cmd) (its_cmd_mask_field(cmd, 1, 0, 5) + 1) -#define its_cmd_get_id(cmd) its_cmd_mask_field(cmd, 1, 0, 32) -#define its_cmd_get_physical_id(cmd) its_cmd_mask_field(cmd, 1, 32, 32) -#define its_cmd_get_collection(cmd) its_cmd_mask_field(cmd, 2, 0, 16) -#define its_cmd_get_ittaddr(cmd) (its_cmd_mask_field(cmd, 2, 8, 44) << 8) -#define its_cmd_get_target_addr(cmd) its_cmd_mask_field(cmd, 2, 16, 32) -#define its_cmd_get_validbit(cmd) its_cmd_mask_field(cmd, 2, 63, 1) - -/* - * The DISCARD command frees an Interrupt Translation Table Entry (ITTE). - * Must be called with the its_lock mutex held. - */ -static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its, - u64 *its_cmd) -{ - u32 device_id = its_cmd_get_deviceid(its_cmd); - u32 event_id = its_cmd_get_id(its_cmd); - struct its_ite *ite; - - ite = find_ite(its, device_id, event_id); - if (ite && its_is_collection_mapped(ite->collection)) { - /* - * Though the spec talks about removing the pending state, we - * don't bother here since we clear the ITTE anyway and the - * pending state is a property of the ITTE struct. - */ - vgic_its_invalidate_cache(kvm); - - its_free_ite(kvm, ite); - return 0; - } - - return E_ITS_DISCARD_UNMAPPED_INTERRUPT; -} - -/* - * The MOVI command moves an ITTE to a different collection. - * Must be called with the its_lock mutex held. - */ -static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its, - u64 *its_cmd) -{ - u32 device_id = its_cmd_get_deviceid(its_cmd); - u32 event_id = its_cmd_get_id(its_cmd); - u32 coll_id = its_cmd_get_collection(its_cmd); - struct kvm_vcpu *vcpu; - struct its_ite *ite; - struct its_collection *collection; - - ite = find_ite(its, device_id, event_id); - if (!ite) - return E_ITS_MOVI_UNMAPPED_INTERRUPT; - - if (!its_is_collection_mapped(ite->collection)) - return E_ITS_MOVI_UNMAPPED_COLLECTION; - - collection = find_collection(its, coll_id); - if (!its_is_collection_mapped(collection)) - return E_ITS_MOVI_UNMAPPED_COLLECTION; - - ite->collection = collection; - vcpu = kvm_get_vcpu(kvm, collection->target_addr); - - vgic_its_invalidate_cache(kvm); - - return update_affinity(ite->irq, vcpu); -} - -/* - * Check whether an ID can be stored into the corresponding guest table. - * For a direct table this is pretty easy, but gets a bit nasty for - * indirect tables. We check whether the resulting guest physical address - * is actually valid (covered by a memslot and guest accessible). - * For this we have to read the respective first level entry. - */ -static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id, - gpa_t *eaddr) -{ - int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K; - u64 indirect_ptr, type = GITS_BASER_TYPE(baser); - phys_addr_t base = GITS_BASER_ADDR_48_to_52(baser); - int esz = GITS_BASER_ENTRY_SIZE(baser); - int index, idx; - gfn_t gfn; - bool ret; - - switch (type) { - case GITS_BASER_TYPE_DEVICE: - if (id >= BIT_ULL(VITS_TYPER_DEVBITS)) - return false; - break; - case GITS_BASER_TYPE_COLLECTION: - /* as GITS_TYPER.CIL == 0, ITS supports 16-bit collection ID */ - if (id >= BIT_ULL(16)) - return false; - break; - default: - return false; - } - - if (!(baser & GITS_BASER_INDIRECT)) { - phys_addr_t addr; - - if (id >= (l1_tbl_size / esz)) - return false; - - addr = base + id * esz; - gfn = addr >> PAGE_SHIFT; - - if (eaddr) - *eaddr = addr; - - goto out; - } - - /* calculate and check the index into the 1st level */ - index = id / (SZ_64K / esz); - if (index >= (l1_tbl_size / sizeof(u64))) - return false; - - /* Each 1st level entry is represented by a 64-bit value. */ - if (kvm_read_guest_lock(its->dev->kvm, - base + index * sizeof(indirect_ptr), - &indirect_ptr, sizeof(indirect_ptr))) - return false; - - indirect_ptr = le64_to_cpu(indirect_ptr); - - /* check the valid bit of the first level entry */ - if (!(indirect_ptr & BIT_ULL(63))) - return false; - - /* Mask the guest physical address and calculate the frame number. */ - indirect_ptr &= GENMASK_ULL(51, 16); - - /* Find the address of the actual entry */ - index = id % (SZ_64K / esz); - indirect_ptr += index * esz; - gfn = indirect_ptr >> PAGE_SHIFT; - - if (eaddr) - *eaddr = indirect_ptr; - -out: - idx = srcu_read_lock(&its->dev->kvm->srcu); - ret = kvm_is_visible_gfn(its->dev->kvm, gfn); - srcu_read_unlock(&its->dev->kvm->srcu, idx); - return ret; -} - -static int vgic_its_alloc_collection(struct vgic_its *its, - struct its_collection **colp, - u32 coll_id) -{ - struct its_collection *collection; - - if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL)) - return E_ITS_MAPC_COLLECTION_OOR; - - collection = kzalloc(sizeof(*collection), GFP_KERNEL); - if (!collection) - return -ENOMEM; - - collection->collection_id = coll_id; - collection->target_addr = COLLECTION_NOT_MAPPED; - - list_add_tail(&collection->coll_list, &its->collection_list); - *colp = collection; - - return 0; -} - -static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id) -{ - struct its_collection *collection; - struct its_device *device; - struct its_ite *ite; - - /* - * Clearing the mapping for that collection ID removes the - * entry from the list. If there wasn't any before, we can - * go home early. - */ - collection = find_collection(its, coll_id); - if (!collection) - return; - - for_each_lpi_its(device, ite, its) - if (ite->collection && - ite->collection->collection_id == coll_id) - ite->collection = NULL; - - list_del(&collection->coll_list); - kfree(collection); -} - -/* Must be called with its_lock mutex held */ -static struct its_ite *vgic_its_alloc_ite(struct its_device *device, - struct its_collection *collection, - u32 event_id) -{ - struct its_ite *ite; - - ite = kzalloc(sizeof(*ite), GFP_KERNEL); - if (!ite) - return ERR_PTR(-ENOMEM); - - ite->event_id = event_id; - ite->collection = collection; - - list_add_tail(&ite->ite_list, &device->itt_head); - return ite; -} - -/* - * The MAPTI and MAPI commands map LPIs to ITTEs. - * Must be called with its_lock mutex held. - */ -static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its, - u64 *its_cmd) -{ - u32 device_id = its_cmd_get_deviceid(its_cmd); - u32 event_id = its_cmd_get_id(its_cmd); - u32 coll_id = its_cmd_get_collection(its_cmd); - struct its_ite *ite; - struct kvm_vcpu *vcpu = NULL; - struct its_device *device; - struct its_collection *collection, *new_coll = NULL; - struct vgic_irq *irq; - int lpi_nr; - - device = find_its_device(its, device_id); - if (!device) - return E_ITS_MAPTI_UNMAPPED_DEVICE; - - if (event_id >= BIT_ULL(device->num_eventid_bits)) - return E_ITS_MAPTI_ID_OOR; - - if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI) - lpi_nr = its_cmd_get_physical_id(its_cmd); - else - lpi_nr = event_id; - if (lpi_nr < GIC_LPI_OFFSET || - lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser)) - return E_ITS_MAPTI_PHYSICALID_OOR; - - /* If there is an existing mapping, behavior is UNPREDICTABLE. */ - if (find_ite(its, device_id, event_id)) - return 0; - - collection = find_collection(its, coll_id); - if (!collection) { - int ret = vgic_its_alloc_collection(its, &collection, coll_id); - if (ret) - return ret; - new_coll = collection; - } - - ite = vgic_its_alloc_ite(device, collection, event_id); - if (IS_ERR(ite)) { - if (new_coll) - vgic_its_free_collection(its, coll_id); - return PTR_ERR(ite); - } - - if (its_is_collection_mapped(collection)) - vcpu = kvm_get_vcpu(kvm, collection->target_addr); - - irq = vgic_add_lpi(kvm, lpi_nr, vcpu); - if (IS_ERR(irq)) { - if (new_coll) - vgic_its_free_collection(its, coll_id); - its_free_ite(kvm, ite); - return PTR_ERR(irq); - } - ite->irq = irq; - - return 0; -} - -/* Requires the its_lock to be held. */ -static void vgic_its_free_device(struct kvm *kvm, struct its_device *device) -{ - struct its_ite *ite, *temp; - - /* - * The spec says that unmapping a device with still valid - * ITTEs associated is UNPREDICTABLE. We remove all ITTEs, - * since we cannot leave the memory unreferenced. - */ - list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list) - its_free_ite(kvm, ite); - - vgic_its_invalidate_cache(kvm); - - list_del(&device->dev_list); - kfree(device); -} - -/* its lock must be held */ -static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its) -{ - struct its_device *cur, *temp; - - list_for_each_entry_safe(cur, temp, &its->device_list, dev_list) - vgic_its_free_device(kvm, cur); -} - -/* its lock must be held */ -static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its) -{ - struct its_collection *cur, *temp; - - list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list) - vgic_its_free_collection(its, cur->collection_id); -} - -/* Must be called with its_lock mutex held */ -static struct its_device *vgic_its_alloc_device(struct vgic_its *its, - u32 device_id, gpa_t itt_addr, - u8 num_eventid_bits) -{ - struct its_device *device; - - device = kzalloc(sizeof(*device), GFP_KERNEL); - if (!device) - return ERR_PTR(-ENOMEM); - - device->device_id = device_id; - device->itt_addr = itt_addr; - device->num_eventid_bits = num_eventid_bits; - INIT_LIST_HEAD(&device->itt_head); - - list_add_tail(&device->dev_list, &its->device_list); - return device; -} - -/* - * MAPD maps or unmaps a device ID to Interrupt Translation Tables (ITTs). - * Must be called with the its_lock mutex held. - */ -static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its, - u64 *its_cmd) -{ - u32 device_id = its_cmd_get_deviceid(its_cmd); - bool valid = its_cmd_get_validbit(its_cmd); - u8 num_eventid_bits = its_cmd_get_size(its_cmd); - gpa_t itt_addr = its_cmd_get_ittaddr(its_cmd); - struct its_device *device; - - if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL)) - return E_ITS_MAPD_DEVICE_OOR; - - if (valid && num_eventid_bits > VITS_TYPER_IDBITS) - return E_ITS_MAPD_ITTSIZE_OOR; - - device = find_its_device(its, device_id); - - /* - * The spec says that calling MAPD on an already mapped device - * invalidates all cached data for this device. We implement this - * by removing the mapping and re-establishing it. - */ - if (device) - vgic_its_free_device(kvm, device); - - /* - * The spec does |