diff options
Diffstat (limited to 'drivers')
| -rw-r--r-- | drivers/misc/cxl/fault.c | 2 | ||||
| -rw-r--r-- | drivers/misc/ocxl/Makefile | 3 | ||||
| -rw-r--r-- | drivers/misc/ocxl/afu_irq.c | 102 | ||||
| -rw-r--r-- | drivers/misc/ocxl/config.c | 13 | ||||
| -rw-r--r-- | drivers/misc/ocxl/context.c | 31 | ||||
| -rw-r--r-- | drivers/misc/ocxl/core.c | 574 | ||||
| -rw-r--r-- | drivers/misc/ocxl/file.c | 182 | ||||
| -rw-r--r-- | drivers/misc/ocxl/link.c | 42 | ||||
| -rw-r--r-- | drivers/misc/ocxl/mmio.c | 234 | ||||
| -rw-r--r-- | drivers/misc/ocxl/ocxl_internal.h | 94 | ||||
| -rw-r--r-- | drivers/misc/ocxl/pci.c | 565 | ||||
| -rw-r--r-- | drivers/misc/ocxl/sysfs.c | 54 | ||||
| -rw-r--r-- | drivers/misc/ocxl/trace.h | 12 |
13 files changed, 1150 insertions, 758 deletions
diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c index dc7b34174f85..a4d17a5a9763 100644 --- a/drivers/misc/cxl/fault.c +++ b/drivers/misc/cxl/fault.c @@ -168,7 +168,7 @@ int cxl_handle_mm_fault(struct mm_struct *mm, u64 dsisr, u64 dar) if (dsisr & CXL_PSL_DSISR_An_S) access |= _PAGE_WRITE; - if (!mm && (REGION_ID(dar) != USER_REGION_ID)) + if (!mm && (get_region_id(dar) != USER_REGION_ID)) access |= _PAGE_PRIVILEGED; if (dsisr & DSISR_NOHPTE) diff --git a/drivers/misc/ocxl/Makefile b/drivers/misc/ocxl/Makefile index 5229dcda8297..d07d1bb8e8d4 100644 --- a/drivers/misc/ocxl/Makefile +++ b/drivers/misc/ocxl/Makefile @@ -1,8 +1,9 @@ # SPDX-License-Identifier: GPL-2.0+ ccflags-$(CONFIG_PPC_WERROR) += -Werror -ocxl-y += main.o pci.o config.o file.o pasid.o +ocxl-y += main.o pci.o config.o file.o pasid.o mmio.o ocxl-y += link.o context.o afu_irq.o sysfs.o trace.o +ocxl-y += core.o obj-$(CONFIG_OCXL) += ocxl.o # For tracepoints to include our trace.h from tracepoint infrastructure: diff --git a/drivers/misc/ocxl/afu_irq.c b/drivers/misc/ocxl/afu_irq.c index 11ab996657a2..70f8f1c3929d 100644 --- a/drivers/misc/ocxl/afu_irq.c +++ b/drivers/misc/ocxl/afu_irq.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0+ // Copyright 2017 IBM Corp. #include <linux/interrupt.h> -#include <linux/eventfd.h> +#include <asm/pnv-ocxl.h> #include "ocxl_internal.h" #include "trace.h" @@ -11,27 +11,59 @@ struct afu_irq { unsigned int virq; char *name; u64 trigger_page; - struct eventfd_ctx *ev_ctx; + irqreturn_t (*handler)(void *private); + void (*free_private)(void *private); + void *private; }; -static int irq_offset_to_id(struct ocxl_context *ctx, u64 offset) +int ocxl_irq_offset_to_id(struct ocxl_context *ctx, u64 offset) { return (offset - ctx->afu->irq_base_offset) >> PAGE_SHIFT; } -static u64 irq_id_to_offset(struct ocxl_context *ctx, int id) +u64 ocxl_irq_id_to_offset(struct ocxl_context *ctx, int irq_id) { - return ctx->afu->irq_base_offset + (id << PAGE_SHIFT); + return ctx->afu->irq_base_offset + (irq_id << PAGE_SHIFT); } +int ocxl_irq_set_handler(struct ocxl_context *ctx, int irq_id, + irqreturn_t (*handler)(void *private), + void (*free_private)(void *private), + void *private) +{ + struct afu_irq *irq; + int rc; + + mutex_lock(&ctx->irq_lock); + irq = idr_find(&ctx->irq_idr, irq_id); + if (!irq) { + rc = -EINVAL; + goto unlock; + } + + irq->handler = handler; + irq->private = private; + irq->free_private = free_private; + + rc = 0; + // Fall through to unlock + +unlock: + mutex_unlock(&ctx->irq_lock); + return rc; +} +EXPORT_SYMBOL_GPL(ocxl_irq_set_handler); + static irqreturn_t afu_irq_handler(int virq, void *data) { struct afu_irq *irq = (struct afu_irq *) data; trace_ocxl_afu_irq_receive(virq); - if (irq->ev_ctx) - eventfd_signal(irq->ev_ctx, 1); - return IRQ_HANDLED; + + if (irq->handler) + return irq->handler(irq->private); + + return IRQ_HANDLED; // Just drop it on the ground } static int setup_afu_irq(struct ocxl_context *ctx, struct afu_irq *irq) @@ -69,7 +101,7 @@ static void release_afu_irq(struct afu_irq *irq) kfree(irq->name); } -int ocxl_afu_irq_alloc(struct ocxl_context *ctx, u64 *irq_offset) +int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id) { struct afu_irq *irq; int rc; @@ -101,11 +133,11 @@ int ocxl_afu_irq_alloc(struct ocxl_context *ctx, u64 *irq_offset) if (rc) goto err_alloc; - *irq_offset = irq_id_to_offset(ctx, irq->id); - - trace_ocxl_afu_irq_alloc(ctx->pasid, irq->id, irq->virq, irq->hw_irq, - *irq_offset); + trace_ocxl_afu_irq_alloc(ctx->pasid, irq->id, irq->virq, irq->hw_irq); mutex_unlock(&ctx->irq_lock); + + *irq_id = irq->id; + return 0; err_alloc: @@ -117,29 +149,29 @@ err_unlock: kfree(irq); return rc; } +EXPORT_SYMBOL_GPL(ocxl_afu_irq_alloc); static void afu_irq_free(struct afu_irq *irq, struct ocxl_context *ctx) { trace_ocxl_afu_irq_free(ctx->pasid, irq->id); if (ctx->mapping) unmap_mapping_range(ctx->mapping, - irq_id_to_offset(ctx, irq->id), + ocxl_irq_id_to_offset(ctx, irq->id), 1 << PAGE_SHIFT, 1); release_afu_irq(irq); - if (irq->ev_ctx) - eventfd_ctx_put(irq->ev_ctx); + if (irq->free_private) + irq->free_private(irq->private); ocxl_link_free_irq(ctx->afu->fn->link, irq->hw_irq); kfree(irq); } -int ocxl_afu_irq_free(struct ocxl_context *ctx, u64 irq_offset) +int ocxl_afu_irq_free(struct ocxl_context *ctx, int irq_id) { struct afu_irq *irq; - int id = irq_offset_to_id(ctx, irq_offset); mutex_lock(&ctx->irq_lock); - irq = idr_find(&ctx->irq_idr, id); + irq = idr_find(&ctx->irq_idr, irq_id); if (!irq) { mutex_unlock(&ctx->irq_lock); return -EINVAL; @@ -149,6 +181,7 @@ int ocxl_afu_irq_free(struct ocxl_context *ctx, u64 irq_offset) mutex_unlock(&ctx->irq_lock); return 0; } +EXPORT_SYMBOL_GPL(ocxl_afu_irq_free); void ocxl_afu_irq_free_all(struct ocxl_context *ctx) { @@ -161,41 +194,16 @@ void ocxl_afu_irq_free_all(struct ocxl_context *ctx) mutex_unlock(&ctx->irq_lock); } -int ocxl_afu_irq_set_fd(struct ocxl_context *ctx, u64 irq_offset, int eventfd) -{ - struct afu_irq *irq; - struct eventfd_ctx *ev_ctx; - int rc = 0, id = irq_offset_to_id(ctx, irq_offset); - - mutex_lock(&ctx->irq_lock); - irq = idr_find(&ctx->irq_idr, id); - if (!irq) { - rc = -EINVAL; - goto unlock; - } - - ev_ctx = eventfd_ctx_fdget(eventfd); - if (IS_ERR(ev_ctx)) { - rc = -EINVAL; - goto unlock; - } - - irq->ev_ctx = ev_ctx; -unlock: - mutex_unlock(&ctx->irq_lock); - return rc; -} - -u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, u64 irq_offset) +u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, int irq_id) { struct afu_irq *irq; - int id = irq_offset_to_id(ctx, irq_offset); u64 addr = 0; mutex_lock(&ctx->irq_lock); - irq = idr_find(&ctx->irq_idr, id); + irq = idr_find(&ctx->irq_idr, irq_id); if (irq) addr = irq->trigger_page; mutex_unlock(&ctx->irq_lock); return addr; } +EXPORT_SYMBOL_GPL(ocxl_afu_irq_get_addr); diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c index 8f2c5d8bd2ee..5e65acb8e134 100644 --- a/drivers/misc/ocxl/config.c +++ b/drivers/misc/ocxl/config.c @@ -2,8 +2,8 @@ // Copyright 2017 IBM Corp. #include <linux/pci.h> #include <asm/pnv-ocxl.h> -#include <misc/ocxl.h> #include <misc/ocxl-config.h> +#include "ocxl_internal.h" #define EXTRACT_BIT(val, bit) (!!(val & BIT(bit))) #define EXTRACT_BITS(val, s, e) ((val & GENMASK(e, s)) >> s) @@ -68,7 +68,7 @@ static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx) return 0; } -static int read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn) +static void read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn) { u16 val; int pos; @@ -89,7 +89,6 @@ static int read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn) out: dev_dbg(&dev->dev, "PASID capability:\n"); dev_dbg(&dev->dev, " Max PASID log = %d\n", fn->max_pasid_log); - return 0; } static int read_dvsec_tl(struct pci_dev *dev, struct ocxl_fn_config *fn) @@ -205,11 +204,7 @@ int ocxl_config_read_function(struct pci_dev *dev, struct ocxl_fn_config *fn) { int rc; - rc = read_pasid(dev, fn); - if (rc) { - dev_err(&dev->dev, "Invalid PASID configuration: %d\n", rc); - return -ENODEV; - } + read_pasid(dev, fn); rc = read_dvsec_tl(dev, fn); if (rc) { @@ -304,7 +299,6 @@ int ocxl_config_check_afu_index(struct pci_dev *dev, } return 1; } -EXPORT_SYMBOL_GPL(ocxl_config_check_afu_index); static int read_afu_name(struct pci_dev *dev, struct ocxl_fn_config *fn, struct ocxl_afu_config *afu) @@ -540,7 +534,6 @@ int ocxl_config_get_pasid_info(struct pci_dev *dev, int *count) { return pnv_ocxl_get_pasid_count(dev, count); } -EXPORT_SYMBOL_GPL(ocxl_config_get_pasid_info); void ocxl_config_set_afu_pasid(struct pci_dev *dev, int pos, int pasid_base, u32 pasid_count_log) diff --git a/drivers/misc/ocxl/context.c b/drivers/misc/ocxl/context.c index c10a940e3b38..bab9c9364184 100644 --- a/drivers/misc/ocxl/context.c +++ b/drivers/misc/ocxl/context.c @@ -4,15 +4,17 @@ #include "trace.h" #include "ocxl_internal.h" -struct ocxl_context *ocxl_context_alloc(void) -{ - return kzalloc(sizeof(struct ocxl_context), GFP_KERNEL); -} - -int ocxl_context_init(struct ocxl_context *ctx, struct ocxl_afu *afu, +int ocxl_context_alloc(struct ocxl_context **context, struct ocxl_afu *afu, struct address_space *mapping) { int pasid; + struct ocxl_context *ctx; + + *context = kzalloc(sizeof(struct ocxl_context), GFP_KERNEL); + if (!*context) + return -ENOMEM; + + ctx = *context; ctx->afu = afu; mutex_lock(&afu->contexts_lock); @@ -43,6 +45,7 @@ int ocxl_context_init(struct ocxl_context *ctx, struct ocxl_afu *afu, ocxl_afu_get(afu); return 0; } +EXPORT_SYMBOL_GPL(ocxl_context_alloc); /* * Callback for when a translation fault triggers an error @@ -63,7 +66,7 @@ static void xsl_fault_error(void *data, u64 addr, u64 dsisr) wake_up_all(&ctx->events_wq); } -int ocxl_context_attach(struct ocxl_context *ctx, u64 amr) +int ocxl_context_attach(struct ocxl_context *ctx, u64 amr, struct mm_struct *mm) { int rc; @@ -75,7 +78,7 @@ int ocxl_context_attach(struct ocxl_context *ctx, u64 amr) } rc = ocxl_link_add_pe(ctx->afu->fn->link, ctx->pasid, - current->mm->context.id, ctx->tidr, amr, current->mm, + mm->context.id, ctx->tidr, amr, mm, xsl_fault_error, ctx); if (rc) goto out; @@ -85,13 +88,15 @@ out: mutex_unlock(&ctx->status_mutex); return rc; } +EXPORT_SYMBOL_GPL(ocxl_context_attach); static vm_fault_t map_afu_irq(struct vm_area_struct *vma, unsigned long address, u64 offset, struct ocxl_context *ctx) { u64 trigger_addr; + int irq_id = ocxl_irq_offset_to_id(ctx, offset); - trigger_addr = ocxl_afu_irq_get_addr(ctx, offset); + trigger_addr = ocxl_afu_irq_get_addr(ctx, irq_id); if (!trigger_addr) return VM_FAULT_SIGBUS; @@ -151,12 +156,14 @@ static const struct vm_operations_struct ocxl_vmops = { static int check_mmap_afu_irq(struct ocxl_context *ctx, struct vm_area_struct *vma) { + int irq_id = ocxl_irq_offset_to_id(ctx, vma->vm_pgoff << PAGE_SHIFT); + /* only one page */ if (vma_pages(vma) != 1) return -EINVAL; /* check offset validty */ - if (!ocxl_afu_irq_get_addr(ctx, vma->vm_pgoff << PAGE_SHIFT)) + if (!ocxl_afu_irq_get_addr(ctx, irq_id)) return -EINVAL; /* @@ -238,11 +245,12 @@ int ocxl_context_detach(struct ocxl_context *ctx) } rc = ocxl_link_remove_pe(ctx->afu->fn->link, ctx->pasid); if (rc) { - dev_warn(&ctx->afu->dev, + dev_warn(&dev->dev, "Couldn't remove PE entry cleanly: %d\n", rc); } return 0; } +EXPORT_SYMBOL_GPL(ocxl_context_detach); void ocxl_context_detach_all(struct ocxl_afu *afu) { @@ -280,3 +288,4 @@ void ocxl_context_free(struct ocxl_context *ctx) ocxl_afu_put(ctx->afu); kfree(ctx); } +EXPORT_SYMBOL_GPL(ocxl_context_free); diff --git a/drivers/misc/ocxl/core.c b/drivers/misc/ocxl/core.c new file mode 100644 index 000000000000..b7a09b21ab36 --- /dev/null +++ b/drivers/misc/ocxl/core.c @@ -0,0 +1,574 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2019 IBM Corp. +#include <linux/idr.h> +#include "ocxl_internal.h" + +static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn) +{ + return (get_device(&fn->dev) == NULL) ? NULL : fn; +} + +static void ocxl_fn_put(struct ocxl_fn *fn) +{ + put_device(&fn->dev); +} + +static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn) +{ + struct ocxl_afu *afu; + + afu = kzalloc(sizeof(struct ocxl_afu), GFP_KERNEL); + if (!afu) + return NULL; + + kref_init(&afu->kref); + mutex_init(&afu->contexts_lock); + mutex_init(&afu->afu_control_lock); + idr_init(&afu->contexts_idr); + afu->fn = fn; + ocxl_fn_get(fn); + return afu; +} + +static void free_afu(struct kref *kref) +{ + struct ocxl_afu *afu = container_of(kref, struct ocxl_afu, kref); + + idr_destroy(&afu->contexts_idr); + ocxl_fn_put(afu->fn); + kfree(afu); +} + +void ocxl_afu_get(struct ocxl_afu *afu) +{ + kref_get(&afu->kref); +} +EXPORT_SYMBOL_GPL(ocxl_afu_get); + +void ocxl_afu_put(struct ocxl_afu *afu) +{ + kref_put(&afu->kref, free_afu); +} +EXPORT_SYMBOL_GPL(ocxl_afu_put); + +static int assign_afu_actag(struct ocxl_afu *afu) +{ + struct ocxl_fn *fn = afu->fn; + int actag_count, actag_offset; + struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent); + + /* + * if there were not enough actags for the function, each afu + * reduces its count as well + */ + actag_count = afu->config.actag_supported * + fn->actag_enabled / fn->actag_supported; + actag_offset = ocxl_actag_afu_alloc(fn, actag_count); + if (actag_offset < 0) { + dev_err(&pci_dev->dev, "Can't allocate %d actags for AFU: %d\n", + actag_count, actag_offset); + return actag_offset; + } + afu->actag_base = fn->actag_base + actag_offset; + afu->actag_enabled = actag_count; + + ocxl_config_set_afu_actag(pci_dev, afu->config.dvsec_afu_control_pos, + afu->actag_base, afu->actag_enabled); + dev_dbg(&pci_dev->dev, "actag base=%d enabled=%d\n", + afu->actag_base, afu->actag_enabled); + return 0; +} + +static void reclaim_afu_actag(struct ocxl_afu *afu) +{ + struct ocxl_fn *fn = afu->fn; + int start_offset, size; + + start_offset = afu->actag_base - fn->actag_base; + size = afu->actag_enabled; + ocxl_actag_afu_free(afu->fn, start_offset, size); +} + +static int assign_afu_pasid(struct ocxl_afu *afu) +{ + struct ocxl_fn *fn = afu->fn; + int pasid_count, pasid_offset; + struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent); + + /* + * We only support the case where the function configuration + * requested enough PASIDs to cover all AFUs. + */ + pasid_count = 1 << afu->config.pasid_supported_log; + pasid_offset = ocxl_pasid_afu_alloc(fn, pasid_count); + if (pasid_offset < 0) { + dev_err(&pci_dev->dev, "Can't allocate %d PASIDs for AFU: %d\n", + pasid_count, pasid_offset); + return pasid_offset; + } + afu->pasid_base = fn->pasid_base + pasid_offset; + afu->pasid_count = 0; + afu->pasid_max = pasid_count; + + ocxl_config_set_afu_pasid(pci_dev, afu->config.dvsec_afu_control_pos, + afu->pasid_base, + afu->config.pasid_supported_log); + dev_dbg(&pci_dev->dev, "PASID base=%d, enabled=%d\n", + afu->pasid_base, pasid_count); + return 0; +} + +static void reclaim_afu_pasid(struct ocxl_afu *afu) +{ + struct ocxl_fn *fn = afu->fn; + int start_offset, size; + + start_offset = afu->pasid_base - fn->pasid_base; + size = 1 << afu->config.pasid_supported_log; + ocxl_pasid_afu_free(afu->fn, start_offset, size); +} + +static int reserve_fn_bar(struct ocxl_fn *fn, int bar) +{ + struct pci_dev *dev = to_pci_dev(fn->dev.parent); + int rc, idx; + + if (bar != 0 && bar != 2 && bar != 4) + return -EINVAL; + + idx = bar >> 1; + if (fn->bar_used[idx]++ == 0) { + rc = pci_request_region(dev, bar, "ocxl"); + if (rc) + return rc; + } + return 0; +} + +static void release_fn_bar(struct ocxl_fn *fn, int bar) +{ + struct pci_dev *dev = to_pci_dev(fn->dev.parent); + int idx; + + if (bar != 0 && bar != 2 && bar != 4) + return; + + idx = bar >> 1; + if (--fn->bar_used[idx] == 0) + pci_release_region(dev, bar); + WARN_ON(fn->bar_used[idx] < 0); +} + +static int map_mmio_areas(struct ocxl_afu *afu) +{ + int rc; + struct pci_dev *pci_dev = to_pci_dev(afu->fn->dev.parent); + + rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar); + if (rc) + return rc; + + rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar); + if (rc) { + release_fn_bar(afu->fn, afu->config.global_mmio_bar); + return rc; + } + + afu->global_mmio_start = + pci_resource_start(pci_dev, afu->config.global_mmio_bar) + + afu->config.global_mmio_offset; + afu->pp_mmio_start = + pci_resource_start(pci_dev, afu->config.pp_mmio_bar) + + afu->config.pp_mmio_offset; + + afu->global_mmio_ptr = ioremap(afu->global_mmio_start, + afu->config.global_mmio_size); + if (!afu->global_mmio_ptr) { + release_fn_bar(afu->fn, afu->config.pp_mmio_bar); + release_fn_bar(afu->fn, afu->config.global_mmio_bar); + dev_err(&pci_dev->dev, "Error mapping global mmio area\n"); + return -ENOMEM; + } + + /* + * Leave an empty page between the per-process mmio area and + * the AFU interrupt mappings + */ + afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE; + return 0; +} + +static void unmap_mmio_areas(struct ocxl_afu *afu) +{ + if (afu->global_mmio_ptr) { + iounmap(afu->global_mmio_ptr); + afu->global_mmio_ptr = NULL; + } + afu->global_mmio_start = 0; + afu->pp_mmio_start = 0; + release_fn_bar(afu->fn, afu->config.pp_mmio_bar); + release_fn_bar(afu->fn, afu->config.global_mmio_bar); +} + +static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev) +{ + int rc; + + rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx); + if (rc) + return rc; + + rc = assign_afu_actag(afu); + if (rc) + return rc; + + rc = assign_afu_pasid(afu); + if (rc) + goto err_free_actag; + + rc = map_mmio_areas(afu); + if (rc) + goto err_free_pasid; + + return 0; + +err_free_pasid: + reclaim_afu_pasid(afu); +err_free_actag: + reclaim_afu_actag(afu); + return rc; +} + +static void deconfigure_afu(struct ocxl_afu *afu) +{ + unmap_mmio_areas(afu); + reclaim_afu_pasid(afu); + reclaim_afu_actag(afu); +} + +static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu) +{ + ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1); + + return 0; +} + +static void deactivate_afu(struct ocxl_afu *afu) +{ + struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent); + + ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0); +} + +static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx) +{ + int rc; + struct ocxl_afu *afu; + + afu = alloc_afu(fn); + if (!afu) + return -ENOMEM; + + rc = configure_afu(afu, afu_idx, dev); + if (rc) { + ocxl_afu_put(afu); + return rc; + } + + rc = activate_afu(dev, afu); + if (rc) { + deconfigure_afu(afu); + ocxl_afu_put(afu); + return rc; + } + + list_add_tail(&afu->list, &fn->afu_list); + + return 0; +} + +static void remove_afu(struct ocxl_afu *afu) +{ + list_del(&afu->list); + ocxl_context_detach_all(afu); + deactivate_afu(afu); + deconfigure_afu(afu); + ocxl_afu_put(afu); // matches the implicit get in alloc_afu +} + +static struct ocxl_fn *alloc_function(void) +{ + struct ocxl_fn *fn; + + fn = kzalloc(sizeof(struct ocxl_fn), GFP_KERNEL); + if (!fn) + return NULL; + + INIT_LIST_HEAD(&fn->afu_list); + INIT_LIST_HEAD(&fn->pasid_list); + INIT_LIST_HEAD(&fn->actag_list); + + return fn; +} + +static void free_function(struct ocxl_fn *fn) +{ + WARN_ON(!list_empty(&fn->afu_list)); + WARN_ON(!list_empty(&fn->pasid_list)); + kfree(fn); +} + +static void free_function_dev(struct device *dev) +{ + struct ocxl_fn *fn = container_of(dev, struct ocxl_fn, dev); + + free_function(fn); +} + +static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev) +{ + int rc; + + fn->dev.parent = &dev->dev; + fn->dev.release = free_function_dev; + rc = dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev)); + if (rc) + return rc; + return 0; +} + +static int assign_function_actag(struct ocxl_fn *fn) +{ + struct pci_dev *dev = to_pci_dev(fn->dev.parent); + u16 base, enabled, supported; + int rc; + + rc = ocxl_config_get_actag_info(dev, &base, &enabled, &supported); + if (rc) + return rc; + + fn->actag_base = base; + fn->actag_enabled = enabled; + fn->actag_supported = supported; + + ocxl_config_set_actag(dev, fn->config.dvsec_function_pos, + fn->actag_base, fn->actag_enabled); + dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n", + fn->actag_base, fn->actag_enabled); + return 0; +} + +static int set_function_pasid(struct ocxl_fn *fn) +{ + struct pci_dev *dev = to_pci_dev(fn->dev.parent); + int rc, desired_count, max_count; + + /* A function may not require any PASID */ + if (fn->config.max_pasid_log < 0) + return 0; + + rc = ocxl_config_get_pasid_info(dev, &max_count); + if (rc) + return rc; + + desired_count = 1 << fn->config.max_pasid_log; + + if (desired_count > max_count) { + dev_err(&fn->dev, + "Function requires more PASIDs than is available (%d vs. %d)\n", + desired_count, max_count); + return -ENOSPC; + } + + fn->pasid_base = 0; + return 0; +} + +static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev) +{ + int rc; + + rc = pci_enable_device(dev); + if (rc) { + dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc); + return rc; + } + + /* + * Once it has been confirmed to work on our hardware, we + * should reset the function, to force the adapter to restart + * from scratch. + * A function reset would also reset all its AFUs. + * + * Some hints for implementation: + * + * - there's not status bit to know when the reset is done. We + * should try reading the config space to know when it's + * done. + * - probably something like: + * Reset + * wait 100ms + * issue config read + * allow device up to 1 sec to return success on config + * read before declaring it broken + * + * Some shared logic on the card (CFG, TLX) won't be reset, so + * there's no guarantee that it will be enough. + */ + rc = ocxl_config_read_function(dev, &fn->config); + if (rc) + return rc; + + rc = set_function_device(fn, dev); + if (rc) + return rc; + + rc = assign_function_actag(fn); + if (rc) + return rc; + + rc = set_function_pasid(fn); + if (rc) + return rc; + + rc = ocxl_link_setup(dev, 0, &fn->link); + if (rc) + return rc; + + rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos); + if (rc) { + ocxl_link_release(dev, fn->link); + return rc; + } + return 0; +} + +static void deconfigure_function(struct ocxl_fn *fn) +{ + struct pci_dev *dev = to_pci_dev(fn->dev.parent); + + ocxl_link_release(dev, fn->link); + pci_disable_device(dev); +} + +static struct ocxl_fn *init_function(struct pci_dev *dev) +{ + struct ocxl_fn *fn; + int rc; + + fn = alloc_function(); + if (!fn) + return ERR_PTR(-ENOMEM); + + rc = configure_function(fn, dev); + if (rc) { + free_function(fn); + return ERR_PTR(rc); + } + + rc = device_register(&fn->dev); + if (rc) { + deconfigure_function(fn); + put_device(&fn->dev); + return ERR_PTR(rc); + } + return fn; +} + +// Device detection & initialisation + +struct ocxl_fn *ocxl_function_open(struct pci_dev *dev) +{ + int rc, afu_count = 0; + u8 afu; + struct ocxl_fn *fn; + + if (!radix_enabled()) { + dev_err(&dev->dev, "Unsupported memory model (hash)\n"); + return ERR_PTR(-ENODEV); + } + + fn = init_function(dev); + if (IS_ERR(fn)) { + dev_err(&dev->dev, "function init failed: %li\n", + PTR_ERR(fn)); + return fn; + } + + for (afu = 0; afu <= fn->config.max_afu_index; afu++) { + rc = ocxl_config_check_afu_index(dev, &fn->config, afu); + if (rc > 0) { + rc = init_afu(dev, fn, afu); + if (rc) { + dev_err(&dev->dev, + "Can't initialize AFU index %d\n", afu); + continue; + } + afu_count++; + } + } + dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count); + return fn; +} +EXPORT_SYMBOL_GPL(ocxl_function_open); + +struct list_head *ocxl_function_afu_list(struct ocxl_fn *fn) +{ + return &fn->afu_list; +} +EXPORT_SYMBOL_GPL(ocxl_function_afu_list); + +struct ocxl_afu *ocxl_function_fetch_afu(struct ocxl_fn *fn, u8 afu_idx) +{ + struct ocxl_afu *afu; + + list_for_each_entry(afu, &fn->afu_list, list) { + if (afu->config.idx == afu_idx) + return afu; + } + + return NULL; +} +EXPORT_SYMBOL_GPL(ocxl_function_fetch_afu); + +const struct ocxl_fn_config *ocxl_function_config(struct ocxl_fn *fn) +{ + return &fn->config; +} +EXPORT_SYMBOL_GPL(ocxl_function_config); + +void ocxl_function_close(struct ocxl_fn *fn) +{ + struct ocxl_afu *afu, *tmp; + + list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) { + remove_afu(afu); + } + + deconfigure_function(fn); + device_unregister(&fn->dev); +} +EXPORT_SYMBOL_GPL(ocxl_function_close); + +// AFU Metadata + +struct ocxl_afu_config *ocxl_afu_config(struct ocxl_afu *afu) +{ + return &afu->config; +} +EXPORT_SYMBOL_GPL(ocxl_afu_config); + +void ocxl_afu_set_private(struct ocxl_afu *afu, void *private) +{ + afu->private = private; +} +EXPORT_SYMBOL_GPL(ocxl_afu_set_private); + +void *ocxl_afu_get_private(struct ocxl_afu *afu) +{ + if (afu) + return afu->private; + + return NULL; +} +EXPORT_SYMBOL_GPL(ocxl_afu_get_private); diff --git a/drivers/misc/ocxl/file.c b/drivers/misc/ocxl/file.c index e6a607488f8a..2870c25da166 100644 --- a/drivers/misc/ocxl/file.c +++ b/drivers/misc/ocxl/file.c @@ -3,6 +3,7 @@ #include <linux/fs.h> #include <linux/poll.h> #include <linux/sched/signal.h> +#include <linux/eventfd.h> #include <linux/uaccess.h> #include <uapi/misc/ocxl.h> #include <asm/reg.h> @@ -17,70 +18,56 @@ static struct class *ocxl_class; static struct mutex minors_idr_lock; static struct idr minors_idr; -static struct ocxl_afu *find_and_get_afu(dev_t devno) +static struct ocxl_file_info *find_file_info(dev_t devno) { - struct ocxl_afu *afu; - int afu_minor; + struct ocxl_file_info *info; - afu_minor = MINOR(devno); /* * We don't declare an RCU critical section here, as our AFU * is protected by a reference counter on the device. By the time the - * minor number of a device is removed from the idr, the ref count of + * info reference is removed from the idr, the ref count of * the device is already at 0, so no user API will access that AFU and * this function can't return it. */ - afu = idr_find(&minors_idr, afu_minor); - if (afu) - ocxl_afu_get(afu); - return afu; + info = idr_find(&minors_idr, MINOR(devno)); + return info; } -static int allocate_afu_minor(struct ocxl_afu *afu) +static int allocate_minor(struct ocxl_file_info *info) { int minor; mutex_lock(&minors_idr_lock); - minor = idr_alloc(&minors_idr, afu, 0, OCXL_NUM_MINORS, GFP_KERNEL); + minor = idr_alloc(&minors_idr, info, 0, OCXL_NUM_MINORS, GFP_KERNEL); mutex_unlock(&minors_idr_lock); return minor; } -static void free_afu_minor(struct ocxl_afu *afu) +static void free_minor(struct ocxl_file_info *info) { mutex_lock(&minors_idr_lock); - idr_remove(&minors_idr, MINOR(afu->dev.devt)); + idr_remove(&minors_idr, MINOR(info->dev.devt)); mutex_unlock(&minors_idr_lock); } static int afu_open(struct inode *inode, struct file *file) { - struct ocxl_afu *afu; + struct ocxl_file_info *info; struct ocxl_context *ctx; int rc; pr_debug("%s for device %x\n", __func__, inode->i_rdev); - afu = find_and_get_afu(inode->i_rdev); - if (!afu) + info = find_file_info(inode->i_rdev); + if (!info) return -ENODEV; - ctx = ocxl_context_alloc(); - if (!ctx) { - rc = -ENOMEM; - goto put_afu; - } - - rc = ocxl_context_init(ctx, afu, inode->i_mapping); + rc = ocxl_context_alloc(&ctx, info->afu, inode->i_mapping); if (rc) - goto put_afu; + return rc; + file->private_data = ctx; - ocxl_afu_put(afu); return 0; - -put_afu: - ocxl_afu_put(afu); - return rc; } static long afu_ioctl_attach(struct ocxl_context *ctx, @@ -100,7 +87,7 @@ static long afu_ioctl_attach(struct ocxl_context *ctx, return -EINVAL; amr = arg.amr & mfspr(SPRN_UAMOR); - rc = ocxl_context_attach(ctx, amr); + rc = ocxl_context_attach(ctx, amr, current->mm); return rc; } @@ -151,10 +138,9 @@ static long afu_ioctl_enable_p9_wait(struct ocxl_context *ctx, mutex_unlock(&ctx->status_mutex); if (status == ATTACHED) { - int rc; - struct link *link = ctx->afu->fn->link; + int rc = ocxl_link_update_pe(ctx->afu->fn->link, + ctx->pasid, ctx->tidr); - rc = ocxl_link_update_pe(link, ctx->pasid, ctx->tidr); if (rc) return rc; } @@ -198,18 +184,40 @@ static long afu_ioctl_get_features(struct ocxl_context *ctx, x == OCXL_IOCTL_GET_FEATURES ? "GET_FEATURES" : \ "UNKNOWN") +static irqreturn_t irq_handler(void *private) +{ + struct eventfd_ctx *ev_ctx = private; + + eventfd_signal(ev_ctx, 1); + return IRQ_HANDLED; +} + +static void irq_free(void *private) +{ + struct eventfd_ctx *ev_ctx = private; + + eventfd_ctx_put(ev_ctx); +} + static long afu_ioctl(struct file *file, unsigned int cmd, unsigned long args) { struct ocxl_context *ctx = file->private_data; struct ocxl_ioctl_irq_fd irq_fd; + struct eventfd_ctx *ev_ctx; + int irq_id; u64 irq_offset; long rc; + bool closed; pr_debug("%s for context %d, command %s\n", __func__, ctx->pasid, CMD_STR(cmd)); - if (ctx->status == CLOSED) + mutex_lock(&ctx->status_mutex); + closed = (ctx->status == CLOSED); + mutex_unlock(&ctx->status_mutex); + + if (closed) return -EIO; switch (cmd) { @@ -219,12 +227,13 @@ static long afu_io |