diff options
author | Rob Gardner <rob.gardner@oracle.com> | 2017-12-05 19:40:43 -0700 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2018-01-22 08:17:16 -0800 |
commit | dd0273284c7474100bcd331887443f0e4b1dcce8 (patch) | |
tree | 2fe7bb4e2925efebad2ac1550f38f5dd88bdb7eb /drivers | |
parent | c2b5934ff505dc71247b2c7f5927c1e9b6b13c68 (diff) |
sparc64: Oracle DAX driver
DAX is a coprocessor which resides on the SPARC M7 (DAX1) and M8
(DAX2) processor chips, and has direct access to the CPU's L3 caches
as well as physical memory. It can perform several operations on data
streams with various input and output formats. This driver provides a
transport mechanism and has limited knowledge of the various opcodes
and data formats. A user space library provides high level services
and translates these into low level commands which are then passed
into the driver and subsequently the hypervisor and the coprocessor.
The library is the recommended way for applications to use the
coprocessor, and the driver interface is not intended for general use.
Signed-off-by: Rob Gardner <rob.gardner@oracle.com>
Signed-off-by: Jonathan Helman <jonathan.helman@oracle.com>
Signed-off-by: Sanath Kumar <sanath099@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'drivers')
-rw-r--r-- | drivers/sbus/char/Kconfig | 8 | ||||
-rw-r--r-- | drivers/sbus/char/Makefile | 1 | ||||
-rw-r--r-- | drivers/sbus/char/oradax.c | 1005 |
3 files changed, 1014 insertions, 0 deletions
diff --git a/drivers/sbus/char/Kconfig b/drivers/sbus/char/Kconfig index 5ba684f73ab8..a785aa7660c3 100644 --- a/drivers/sbus/char/Kconfig +++ b/drivers/sbus/char/Kconfig @@ -70,5 +70,13 @@ config DISPLAY7SEG another UltraSPARC-IIi-cEngine boardset with a 7-segment display, you should say N to this option. +config ORACLE_DAX + tristate "Oracle Data Analytics Accelerator" + default m if SPARC64 + help + Driver for Oracle Data Analytics Accelerator, which is + a coprocessor that performs database operations in hardware. + It is available on M7 and M8 based systems only. + endmenu diff --git a/drivers/sbus/char/Makefile b/drivers/sbus/char/Makefile index ae478144c551..8c48ed96683f 100644 --- a/drivers/sbus/char/Makefile +++ b/drivers/sbus/char/Makefile @@ -17,3 +17,4 @@ obj-$(CONFIG_SUN_OPENPROMIO) += openprom.o obj-$(CONFIG_TADPOLE_TS102_UCTRL) += uctrl.o obj-$(CONFIG_SUN_JSFLASH) += jsflash.o obj-$(CONFIG_BBC_I2C) += bbc.o +obj-$(CONFIG_ORACLE_DAX) += oradax.o diff --git a/drivers/sbus/char/oradax.c b/drivers/sbus/char/oradax.c new file mode 100644 index 000000000000..10452ae18ef1 --- /dev/null +++ b/drivers/sbus/char/oradax.c @@ -0,0 +1,1005 @@ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/* + * Oracle Data Analytics Accelerator (DAX) + * + * DAX is a coprocessor which resides on the SPARC M7 (DAX1) and M8 + * (DAX2) processor chips, and has direct access to the CPU's L3 + * caches as well as physical memory. It can perform several + * operations on data streams with various input and output formats. + * The driver provides a transport mechanism only and has limited + * knowledge of the various opcodes and data formats. A user space + * library provides high level services and translates these into low + * level commands which are then passed into the driver and + * subsequently the hypervisor and the coprocessor. The library is + * the recommended way for applications to use the coprocessor, and + * the driver interface is not intended for general use. + * + * See Documentation/sparc/oradax/oracle_dax.txt for more details. + */ + +#include <linux/uaccess.h> +#include <linux/module.h> +#include <linux/delay.h> +#include <linux/cdev.h> +#include <linux/slab.h> +#include <linux/mm.h> + +#include <asm/hypervisor.h> +#include <asm/mdesc.h> +#include <asm/oradax.h> + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Driver for Oracle Data Analytics Accelerator"); + +#define DAX_DBG_FLG_BASIC 0x01 +#define DAX_DBG_FLG_STAT 0x02 +#define DAX_DBG_FLG_INFO 0x04 +#define DAX_DBG_FLG_ALL 0xff + +#define dax_err(fmt, ...) pr_err("%s: " fmt "\n", __func__, ##__VA_ARGS__) +#define dax_info(fmt, ...) pr_info("%s: " fmt "\n", __func__, ##__VA_ARGS__) + +#define dax_dbg(fmt, ...) do { \ + if (dax_debug & DAX_DBG_FLG_BASIC)\ + dax_info(fmt, ##__VA_ARGS__); \ + } while (0) +#define dax_stat_dbg(fmt, ...) do { \ + if (dax_debug & DAX_DBG_FLG_STAT) \ + dax_info(fmt, ##__VA_ARGS__); \ + } while (0) +#define dax_info_dbg(fmt, ...) do { \ + if (dax_debug & DAX_DBG_FLG_INFO) \ + dax_info(fmt, ##__VA_ARGS__); \ + } while (0) + +#define DAX1_MINOR 1 +#define DAX1_MAJOR 1 +#define DAX2_MINOR 0 +#define DAX2_MAJOR 2 + +#define DAX1_STR "ORCL,sun4v-dax" +#define DAX2_STR "ORCL,sun4v-dax2" + +#define DAX_CA_ELEMS (DAX_MMAP_LEN / sizeof(struct dax_cca)) + +#define DAX_CCB_USEC 100 +#define DAX_CCB_RETRIES 10000 + +/* stream types */ +enum { + OUT, + PRI, + SEC, + TBL, + NUM_STREAM_TYPES +}; + +/* completion status */ +#define CCA_STAT_NOT_COMPLETED 0 +#define CCA_STAT_COMPLETED 1 +#define CCA_STAT_FAILED 2 +#define CCA_STAT_KILLED 3 +#define CCA_STAT_NOT_RUN 4 +#define CCA_STAT_PIPE_OUT 5 +#define CCA_STAT_PIPE_SRC 6 +#define CCA_STAT_PIPE_DST 7 + +/* completion err */ +#define CCA_ERR_SUCCESS 0x0 /* no error */ +#define CCA_ERR_OVERFLOW 0x1 /* buffer overflow */ +#define CCA_ERR_DECODE 0x2 /* CCB decode error */ +#define CCA_ERR_PAGE_OVERFLOW 0x3 /* page overflow */ +#define CCA_ERR_KILLED 0x7 /* command was killed */ +#define CCA_ERR_TIMEOUT 0x8 /* Timeout */ +#define CCA_ERR_ADI 0x9 /* ADI error */ +#define CCA_ERR_DATA_FMT 0xA /* data format error */ +#define CCA_ERR_OTHER_NO_RETRY 0xE /* Other error, do not retry */ +#define CCA_ERR_OTHER_RETRY 0xF /* Other error, retry */ +#define CCA_ERR_PARTIAL_SYMBOL 0x80 /* QP partial symbol warning */ + +/* CCB address types */ +#define DAX_ADDR_TYPE_NONE 0 +#define DAX_ADDR_TYPE_VA_ALT 1 /* secondary context */ +#define DAX_ADDR_TYPE_RA 2 /* real address */ +#define DAX_ADDR_TYPE_VA 3 /* virtual address */ + +/* dax_header_t opcode */ +#define DAX_OP_SYNC_NOP 0x0 +#define DAX_OP_EXTRACT 0x1 +#define DAX_OP_SCAN_VALUE 0x2 +#define DAX_OP_SCAN_RANGE 0x3 +#define DAX_OP_TRANSLATE 0x4 +#define DAX_OP_SELECT 0x5 +#define DAX_OP_INVERT 0x10 /* OR with translate, scan opcodes */ + +struct dax_header { + u32 ccb_version:4; /* 31:28 CCB Version */ + /* 27:24 Sync Flags */ + u32 pipe:1; /* Pipeline */ + u32 longccb:1; /* Longccb. Set for scan with lu2, lu3, lu4. */ + u32 cond:1; /* Conditional */ + u32 serial:1; /* Serial */ + u32 opcode:8; /* 23:16 Opcode */ + /* 15:0 Address Type. */ + u32 reserved:3; /* 15:13 reserved */ + u32 table_addr_type:2; /* 12:11 Huffman Table Address Type */ + u32 out_addr_type:3; /* 10:8 Destination Address Type */ + u32 sec_addr_type:3; /* 7:5 Secondary Source Address Type */ + u32 pri_addr_type:3; /* 4:2 Primary Source Address Type */ + u32 cca_addr_type:2; /* 1:0 Completion Address Type */ +}; + +struct dax_control { + u32 pri_fmt:4; /* 31:28 Primary Input Format */ + u32 pri_elem_size:5; /* 27:23 Primary Input Element Size(less1) */ + u32 pri_offset:3; /* 22:20 Primary Input Starting Offset */ + u32 sec_encoding:1; /* 19 Secondary Input Encoding */ + /* (must be 0 for Select) */ + u32 sec_offset:3; /* 18:16 Secondary Input Starting Offset */ + u32 sec_elem_size:2; /* 15:14 Secondary Input Element Size */ + /* (must be 0 for Select) */ + u32 out_fmt:2; /* 13:12 Output Format */ + u32 out_elem_size:2; /* 11:10 Output Element Size */ + u32 misc:10; /* 9:0 Opcode specific info */ +}; + +struct dax_data_access { + u64 flow_ctrl:2; /* 63:62 Flow Control Type */ + u64 pipe_target:2; /* 61:60 Pipeline Target */ + u64 out_buf_size:20; /* 59:40 Output Buffer Size */ + /* (cachelines less 1) */ + u64 unused1:8; /* 39:32 Reserved, Set to 0 */ + u64 out_alloc:5; /* 31:27 Output Allocation */ + u64 unused2:1; /* 26 Reserved */ + u64 pri_len_fmt:2; /* 25:24 Input Length Format */ + u64 pri_len:24; /* 23:0 Input Element/Byte/Bit Count */ + /* (less 1) */ +}; + +struct dax_ccb { + struct dax_header hdr; /* CCB Header */ + struct dax_control ctrl;/* Control Word */ + void *ca; /* Completion Address */ + void *pri; /* Primary Input Address */ + struct dax_data_access dac; /* Data Access Control */ + void *sec; /* Secondary Input Address */ + u64 dword5; /* depends on opcode */ + void *out; /* Output Address */ + void *tbl; /* Table Address or bitmap */ +}; + +struct dax_cca { + u8 status; /* user may mwait on this address */ + u8 err; /* user visible error notification */ + u8 rsvd[2]; /* reserved */ + u32 n_remaining; /* for QP partial symbol warning */ + u32 output_sz; /* output in bytes */ + u32 rsvd2; /* reserved */ + u64 run_cycles; /* run time in OCND2 cycles */ + u64 run_stats; /* nothing reported in version 1.0 */ + u32 n_processed; /* number input elements */ + u32 rsvd3[5]; /* reserved */ + u64 retval; /* command return value */ + u64 rsvd4[8]; /* reserved */ +}; + +/* per thread CCB context */ +struct dax_ctx { + struct dax_ccb *ccb_buf; + u64 ccb_buf_ra; /* cached RA of ccb_buf */ + struct dax_cca *ca_buf; + u64 ca_buf_ra; /* cached RA of ca_buf */ + struct page *pages[DAX_CA_ELEMS][NUM_STREAM_TYPES]; + /* array of locked pages */ + struct task_struct *owner; /* thread that owns ctx */ + struct task_struct *client; /* requesting thread */ + union ccb_result result; + u32 ccb_count; + u32 fail_count; +}; + +/* driver public entry points */ +static int dax_open(struct inode *inode, struct file *file); +static ssize_t dax_read(struct file *filp, char __user *buf, + size_t count, loff_t *ppos); +static ssize_t dax_write(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos); +static int dax_devmap(struct file *f, struct vm_area_struct *vma); +static int dax_close(struct inode *i, struct file *f); + +static const struct file_operations dax_fops = { + .owner = THIS_MODULE, + .open = dax_open, + .read = dax_read, + .write = dax_write, + .mmap = dax_devmap, + .release = dax_close, +}; + +static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf, + size_t count, loff_t *ppos); +static int dax_ccb_info(u64 ca, struct ccb_info_result *info); +static int dax_ccb_kill(u64 ca, u16 *kill_res); + +static struct cdev c_dev; +static struct class *cl; +static dev_t first; + +static int max_ccb_version; +static int dax_debug; +module_param(dax_debug, int, 0644); +MODULE_PARM_DESC(dax_debug, "Debug flags"); + +static int __init dax_attach(void) +{ + unsigned long dummy, hv_rv, major, minor, minor_requested, max_ccbs; + struct mdesc_handle *hp = mdesc_grab(); + char *prop, *dax_name; + bool found = false; + int len, ret = 0; + u64 pn; + + if (hp == NULL) { + dax_err("Unable to grab mdesc"); + return -ENODEV; + } + + mdesc_for_each_node_by_name(hp, pn, "virtual-device") { + prop = (char *)mdesc_get_property(hp, pn, "name", &len); + if (prop == NULL) + continue; + if (strncmp(prop, "dax", strlen("dax"))) + continue; + dax_dbg("Found node 0x%llx = %s", pn, prop); + + prop = (char *)mdesc_get_property(hp, pn, "compatible", &len); + if (prop == NULL) + continue; + dax_dbg("Found node 0x%llx = %s", pn, prop); + found = true; + break; + } + + if (!found) { + dax_err("No DAX device found"); + ret = -ENODEV; + goto done; + } + + if (strncmp(prop, DAX2_STR, strlen(DAX2_STR)) == 0) { + dax_name = DAX_NAME "2"; + major = DAX2_MAJOR; + minor_requested = DAX2_MINOR; + max_ccb_version = 1; + dax_dbg("MD indicates DAX2 coprocessor"); + } else if (strncmp(prop, DAX1_STR, strlen(DAX1_STR)) == 0) { + dax_name = DAX_NAME "1"; + major = DAX1_MAJOR; + minor_requested = DAX1_MINOR; + max_ccb_version = 0; + dax_dbg("MD indicates DAX1 coprocessor"); + } else { + dax_err("Unknown dax type: %s", prop); + ret = -ENODEV; + goto done; + } + + minor = minor_requested; + dax_dbg("Registering DAX HV api with major %ld minor %ld", major, + minor); + if (sun4v_hvapi_register(HV_GRP_DAX, major, &minor)) { + dax_err("hvapi_register failed"); + ret = -ENODEV; + goto done; + } else { + dax_dbg("Max minor supported by HV = %ld (major %ld)", minor, + major); + minor = min(minor, minor_requested); + dax_dbg("registered DAX major %ld minor %ld", major, minor); + } + + /* submit a zero length ccb array to query coprocessor queue size */ + hv_rv = sun4v_ccb_submit(0, 0, HV_CCB_QUERY_CMD, 0, &max_ccbs, &dummy); + if (hv_rv != 0) { + dax_err("get_hwqueue_size failed with status=%ld and max_ccbs=%ld", + hv_rv, max_ccbs); + ret = -ENODEV; + goto done; + } + + if (max_ccbs != DAX_MAX_CCBS) { + dax_err("HV reports unsupported max_ccbs=%ld", max_ccbs); + ret = -ENODEV; + goto done; + } + + if (alloc_chrdev_region(&first, 0, 1, DAX_NAME) < 0) { + dax_err("alloc_chrdev_region failed"); + ret = -ENXIO; + goto done; + } + + cl = class_create(THIS_MODULE, DAX_NAME); + if (cl == NULL) { + dax_err("class_create failed"); + ret = -ENXIO; + goto class_error; + } + + if (device_create(cl, NULL, first, NULL, dax_name) == NULL) { + dax_err("device_create failed"); + ret = -ENXIO; + goto device_error; + } + + cdev_init(&c_dev, &dax_fops); + if (cdev_add(&c_dev, first, 1) == -1) { + dax_err("cdev_add failed"); + ret = -ENXIO; + goto cdev_error; + } + + pr_info("Attached DAX module\n"); + goto done; + +cdev_error: + device_destroy(cl, first); +device_error: + class_destroy(cl); +class_error: + unregister_chrdev_region(first, 1); +done: + mdesc_release(hp); + return ret; +} +module_init(dax_attach); + +static void __exit dax_detach(void) +{ + pr_info("Cleaning up DAX module\n"); + cdev_del(&c_dev); + device_destroy(cl, first); + class_destroy(cl); + unregister_chrdev_region(first, 1); +} +module_exit(dax_detach); + +/* map completion area */ +static int dax_devmap(struct file *f, struct vm_area_struct *vma) +{ + struct dax_ctx *ctx = (struct dax_ctx *)f->private_data; + size_t len = vma->vm_end - vma->vm_start; + + dax_dbg("len=0x%lx, flags=0x%lx", len, vma->vm_flags); + + if (ctx->owner != current) { + dax_dbg("devmap called from wrong thread"); + return -EINVAL; + } + + if (len != DAX_MMAP_LEN) { + dax_dbg("len(%lu) != DAX_MMAP_LEN(%d)", len, DAX_MMAP_LEN); + return -EINVAL; + } + + /* completion area is mapped read-only for user */ + if (vma->vm_flags & VM_WRITE) + return -EPERM; + vma->vm_flags &= ~VM_MAYWRITE; + + if (remap_pfn_range(vma, vma->vm_start, ctx->ca_buf_ra >> PAGE_SHIFT, + len, vma->vm_page_prot)) + return -EAGAIN; + + dax_dbg("mmapped completion area at uva 0x%lx", vma->vm_start); + return 0; +} + +/* Unlock user pages. Called during dequeue or device close */ +static void dax_unlock_pages(struct dax_ctx *ctx, int ccb_index, int nelem) +{ + int i, j; + + for (i = ccb_index; i < ccb_index + nelem; i++) { + for (j = 0; j < NUM_STREAM_TYPES; j++) { + struct page *p = ctx->pages[i][j]; + + if (p) { + dax_dbg("freeing page %p", p); + if (j == OUT) + set_page_dirty(p); + put_page(p); + ctx->pages[i][j] = NULL; + } + } + } +} + +static int dax_lock_page(void *va, struct page **p) +{ + int ret; + + dax_dbg("uva %p", va); + + ret = get_user_pages_fast((unsigned long)va, 1, 1, p); + if (ret == 1) { + dax_dbg("locked page %p, for VA %p", *p, va); + return 0; + } + + dax_dbg("get_user_pages failed, va=%p, ret=%d", va, ret); + return -1; +} + +static int dax_lock_pages(struct dax_ctx *ctx, int idx, + int nelem, u64 *err_va) +{ + int i; + + for (i = 0; i < nelem; i++) { + struct dax_ccb *ccbp = &ctx->ccb_buf[i]; + + /* + * For each address in the CCB whose type is virtual, + * lock the page and change the type to virtual alternate + * context. On error, return the offending address in + * err_va. + */ + if (ccbp->hdr.out_addr_type == DAX_ADDR_TYPE_VA) { + dax_dbg("output"); + if (dax_lock_page(ccbp->out, + &ctx->pages[i + idx][OUT]) != 0) { + *err_va = (u64)ccbp->out; + goto error; + } + ccbp->hdr.out_addr_type = DAX_ADDR_TYPE_VA_ALT; + } + + if (ccbp->hdr.pri_addr_type == DAX_ADDR_TYPE_VA) { + dax_dbg("input"); + if (dax_lock_page(ccbp->pri, + &ctx->pages[i + idx][PRI]) != 0) { + *err_va = (u64)ccbp->pri; + goto error; + } + ccbp->hdr.pri_addr_type = DAX_ADDR_TYPE_VA_ALT; + } + + if (ccbp->hdr.sec_addr_type == DAX_ADDR_TYPE_VA) { + dax_dbg("sec input"); + if (dax_lock_page(ccbp->sec, + &ctx->pages[i + idx][SEC]) != 0) { + *err_va = (u64)ccbp->sec; + goto error; + } + ccbp->hdr.sec_addr_type = DAX_ADDR_TYPE_VA_ALT; + } + + if (ccbp->hdr.table_addr_type == DAX_ADDR_TYPE_VA) { + dax_dbg("tbl"); + if (dax_lock_page(ccbp->tbl, + &ctx->pages[i + idx][TBL]) != 0) { + *err_va = (u64)ccbp->tbl; + goto error; + } + ccbp->hdr.table_addr_type = DAX_ADDR_TYPE_VA_ALT; + } + + /* skip over 2nd 64 bytes of long CCB */ + if (ccbp->hdr.longccb) + i++; + } + return DAX_SUBMIT_OK; + +error: + dax_unlock_pages(ctx, idx, nelem); + return DAX_SUBMIT_ERR_NOACCESS; +} + +static void dax_ccb_wait(struct dax_ctx *ctx, int idx) +{ + int ret, nretries; + u16 kill_res; + + dax_dbg("idx=%d", idx); + + for (nretries = 0; nretries < DAX_CCB_RETRIES; nretries++) { + if (ctx->ca_buf[idx].status == CCA_STAT_NOT_COMPLETED) + udelay(DAX_CCB_USEC); + else + return; + } + dax_dbg("ctx (%p): CCB[%d] timed out, wait usec=%d, retries=%d. Killing ccb", + (void *)ctx, idx, DAX_CCB_USEC, DAX_CCB_RETRIES); + + ret = dax_ccb_kill(ctx->ca_buf_ra + idx * sizeof(struct dax_cca), + &kill_res); + dax_dbg("Kill CCB[%d] %s", idx, ret ? "failed" : "succeeded"); +} + +static int dax_close(struct inode *ino, struct file *f) +{ + struct dax_ctx *ctx = (struct dax_ctx *)f->private_data; + int i; + + f->private_data = NULL; + + for (i = 0; i < DAX_CA_ELEMS; i++) { + if (ctx->ca_buf[i].status == CCA_STAT_NOT_COMPLETED) { + dax_dbg("CCB[%d] not completed", i); + dax_ccb_wait(ctx, i); + } + dax_unlock_pages(ctx, i, 1); + } + + kfree(ctx->ccb_buf); + kfree(ctx->ca_buf); + dax_stat_dbg("CCBs: %d good, %d bad", ctx->ccb_count, ctx->fail_count); + kfree(ctx); + + return 0; +} + +static ssize_t dax_read(struct file *f, char __user *buf, + size_t count, loff_t *ppos) +{ + struct dax_ctx *ctx = f->private_data; + + if (ctx->client != current) + return -EUSERS; + + ctx->client = NULL; + + if (count != sizeof(union ccb_result)) + return -EINVAL; + if (copy_to_user(buf, &ctx->result, sizeof(union ccb_result))) + return -EFAULT; + return count; +} + +static ssize_t dax_write(struct file *f, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct dax_ctx *ctx = f->private_data; + struct dax_command hdr; + unsigned long ca; + int i, idx, ret; + + if (ctx->client != NULL) + return -EINVAL; + + if (count == 0 || count > DAX_MAX_CCBS * sizeof(struct dax_ccb)) + return -EINVAL; + + if (count % sizeof(struct dax_ccb) == 0) + return dax_ccb_exec(ctx, buf, count, ppos); /* CCB EXEC */ + + if (count != sizeof(struct dax_command)) + return -EINVAL; + + /* immediate command */ + if (ctx->owner != current) + return -EUSERS; + + if (copy_from_user(&hdr, buf, sizeof(hdr))) + return -EFAULT; + + ca = ctx->ca_buf_ra + hdr.ca_offset; + + switch (hdr.command) { + case CCB_KILL: + if (hdr.ca_offset >= DAX_MMAP_LEN) { + dax_dbg("invalid ca_offset (%d) >= ca_buflen (%d)", + hdr.ca_offset, DAX_MMAP_LEN); + return -EINVAL; + } + + ret = dax_ccb_kill(ca, &ctx->result.kill.action); + if (ret != 0) { + dax_dbg("dax_ccb_kill failed (ret=%d)", ret); + return ret; + } + + dax_info_dbg("killed (ca_offset %d)", hdr.ca_offset); + idx = hdr.ca_offset / sizeof(struct dax_cca); + ctx->ca_buf[idx].status = CCA_STAT_KILLED; + ctx->ca_buf[idx].err = CCA_ERR_KILLED; + ctx->client = current; + return count; + + case CCB_INFO: + if (hdr.ca_offset >= DAX_MMAP_LEN) { + dax_dbg("invalid ca_offset (%d) >= ca_buflen (%d)", + hdr.ca_offset, DAX_MMAP_LEN); + return -EINVAL; + } + + ret = dax_ccb_info(ca, &ctx->result.info); + if (ret != 0) { + dax_dbg("dax_ccb_info failed (ret=%d)", ret); + return ret; + } + + dax_info_dbg("info succeeded on ca_offset %d", hdr.ca_offset); + ctx->client = current; + return count; + + case CCB_DEQUEUE: + for (i = 0; i < DAX_CA_ELEMS; i++) { + if (ctx->ca_buf[i].status != + CCA_STAT_NOT_COMPLETED) + dax_unlock_pages(ctx, i, 1); + } + return count; + + default: + return -EINVAL; + } +} + +static int dax_open(struct inode *inode, struct file *f) +{ + struct dax_ctx *ctx = NULL; + int i; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (ctx == NULL) + goto done; + + ctx->ccb_buf = kcalloc(DAX_MAX_CCBS, sizeof(struct dax_ccb), + GFP_KERNEL); + if (ctx->ccb_buf == NULL) + goto done; + + ctx->ccb_buf_ra = virt_to_phys(ctx->ccb_buf); + dax_dbg("ctx->ccb_buf=0x%p, ccb_buf_ra=0x%llx", + (void *)ctx->ccb_buf, ctx->ccb_buf_ra); + + /* allocate CCB completion area buffer */ + ctx->ca_buf = kzalloc(DAX_MMAP_LEN, GFP_KERNEL); + if (ctx->ca_buf == NULL) + goto alloc_error; + for (i = 0; i < DAX_CA_ELEMS; i++) + ctx->ca_buf[i].status = CCA_STAT_COMPLETED; + + ctx->ca_buf_ra = virt_to_phys(ctx->ca_buf); + dax_dbg("ctx=0x%p, ctx->ca_buf=0x%p, ca_buf_ra=0x%llx", + (void *)ctx, (void *)ctx->ca_buf, ctx->ca_buf_ra); + + ctx->owner = current; + f->private_data = ctx; + return 0; + +alloc_error: + kfree(ctx->ccb_buf); +done: + if (ctx != NULL) + kfree(ctx); + return -ENOMEM; +} + +static char *dax_hv_errno(unsigned long hv_ret, int *ret) +{ + switch (hv_ret) { + case HV_EBADALIGN: + *ret = -EFAULT; + return "HV_EBADALIGN"; + case HV_ENORADDR: + *ret = -EFAULT; + return "HV_ENORADDR"; + case HV_EINVAL: + *ret = -EINVAL; + return "HV_EINVAL"; + case HV_EWOULDBLOCK: + *ret = -EAGAIN; + return "HV_EWOULDBLOCK"; + case HV_ENOACCESS: + *ret = -EPERM; + return "HV_ENOACCESS"; + default: + break; + } + + *ret = -EIO; + return "UNKNOWN"; +} + +static int dax_ccb_kill(u64 ca, u16 *kill_res) +{ + unsigned long hv_ret; + int count, ret = 0; + char *err_str; + + for (count = 0; count < DAX_CCB_RETRIES; count++) { + dax_dbg("attempting kill on ca_ra 0x%llx", ca); + hv_ret = sun4v_ccb_kill(ca, kill_res); + + if (hv_ret == HV_EOK) { + dax_info_dbg("HV_EOK (ca_ra 0x%llx): %d", ca, + *kill_res); + } else { + err_str = dax_hv_errno(hv_ret, &ret); + dax_dbg("%s (ca_ra 0x%llx)", err_str, ca); + } + + if (ret != -EAGAIN) + return ret; + dax_info_dbg("ccb_kill count = %d", count); + udelay(DAX_CCB_USEC); + } + + return -EAGAIN; +} + +static int dax_ccb_info(u64 ca, struct ccb_info_result *info) +{ + unsigned long hv_ret; + char *err_str; + int ret = 0; + + dax_dbg("attempting info on ca_ra 0x%llx", ca); + hv_ret = sun4v_ccb_info(ca, info); + + if (hv_ret == HV_EOK) { + dax_info_dbg("HV_EOK (ca_ra 0x%llx): %d", ca, info->state); + if (info->state == DAX_CCB_ENQUEUED) { + dax_info_dbg("dax_unit %d, queue_num %d, queue_pos %d", + info->inst_num, info->q_num, info->q_pos); + } + } else { + err_str = dax_hv_errno(hv_ret, &ret); + dax_dbg("%s (ca_ra 0x%llx)", err_str, ca); + } + + return ret; +} + +static void dax_prt_ccbs(struct dax_ccb *ccb, int nelem) +{ + int i, j; + u64 *ccbp; + + dax_dbg("ccb buffer:"); + for (i = 0; i < nelem; i++) { + ccbp = (u64 *)&ccb[i]; + dax_dbg(" %sccb[%d]", ccb[i].hdr.longccb ? "long " : "", i); + for (j = 0; j < 8; j++) + dax_dbg("\tccb[%d].dwords[%d]=0x%llx", + i, j, *(ccbp + j)); + } +} + +/* + * Validates user CCB content. Also sets completion address and address types + * for all addresses contained in CCB. + */ +static int dax_preprocess_usr_ccbs(struct dax_ctx *ctx, int idx, int nelem) +{ + int i; + + /* + * The user is not allowed to specify real address types in + * the CCB header. This must be enforced by the kernel before + * submitting the CCBs to HV. The only allowed values for all + * address fields are VA or IMM + */ + for (i = 0; i < nelem; i++) { + struct dax_ccb *ccbp = &ctx->ccb_buf[i]; + unsigned long ca_offset; + + if (ccbp->hdr.ccb_version > max_ccb_version) + return DAX_SUBMIT_ERR_CCB_INVAL; + + switch (ccbp->hdr.opcode) { + case DAX_OP_SYNC_NOP: + case DAX_OP_EXTRACT: + case DAX_OP_SCAN_VALUE: + case DAX_OP_SCAN_RANGE: + case DAX_OP_TRANSLATE: + case DAX_OP_SCAN_VALUE | DAX_OP_INVERT: + case DAX_OP_SCAN_RANGE | DAX_OP_INVERT: + case DAX_OP_TRANSLATE | DAX_OP_INVERT: + case DAX_OP_SELECT: + break; + default: + return DAX_SUBMIT_ERR_CCB_INVAL; + } + + if (ccbp->hdr.out_addr_type != DAX_ADDR_TYPE_VA && + ccbp->hdr.out_addr_type != DAX_ADDR_TYPE_NONE) { + dax_dbg("invalid out_addr_type in user CCB[%d]", i); + return DAX_SUBMIT_ERR_CCB_INVAL; + } + + if (ccbp->hdr.pri_addr_type != DAX_ADDR_TYPE_VA && + ccbp->hdr.pri_addr_type != DAX_ADDR_TYPE_NONE) { + dax_dbg("invalid pri_addr_type in user CCB[%d]", i); + return DAX_SUBMIT_ERR_CCB_INVAL; + } + + if (ccbp->hdr.sec_addr_type != DAX_ADDR_TYPE_VA && + ccbp->hdr.sec_addr_type != DAX_ADDR_TYPE_NONE) { + dax_dbg("invalid sec_addr_type in user CCB[%d]", i); + return DAX_SUBMIT_ERR_CCB_INVAL; + } + + if (ccbp->hdr.table_addr_type != DAX_ADDR_TYPE_VA && + ccbp->hdr.table_addr_type != DAX_ADDR_TYPE_NONE) { + dax_dbg("invalid table_addr_type in user CCB[%d]", i); + return DAX_SUBMIT_ERR_CCB_INVAL; + } + + /* set completion (real) address and address type */ + ccbp->hdr.cca_addr_type = DAX_ADDR_TYPE_RA; + ca_offset = (idx + i) * sizeof(struct dax_cca); + ccbp->ca = (void *)ctx->ca_buf_ra + ca_offset; + memset(&ctx->ca_buf[idx + i], 0, sizeof(struct dax_cca)); + + dax_dbg("ccb[%d]=%p, ca_offset=0x%lx, compl RA=0x%llx", + i, ccbp, ca_offset, ctx->ca_buf_ra + ca_offset); + + /* skip over 2nd 64 bytes of long CCB */ + if (ccbp->hdr.longccb) + i++; + } + + return DAX_SUBMIT_OK; +} + +static int dax_ccb_exec(struct dax_ctx *ctx, const char __user *buf, + size_t count, loff_t *ppos) +{ + unsigned long accepted_len, hv_rv; + int i, idx, nccbs, naccepted; + + ctx->client = current; + idx = *ppos; + nccbs = count / sizeof(struct dax_ccb); + + if (ctx->owner != current) { + dax_dbg("wrong thread"); + ctx->result.exec.status = DAX_SUBMIT_ERR_THR_INIT; + return 0; + } + dax_dbg("args: ccb_buf_len=%ld, idx=%d", count, idx); + + /* for given index and length, verify ca_buf range exists */ + if (idx + nccbs >= DAX_CA_ELEMS) { + ctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL; + return 0; + } + + /* + * Copy CCBs into kernel buffer to prevent modification by the + * user in between validation and submission. + */ + if (copy_from_user(ctx->ccb_buf, buf, count)) { + dax_dbg("copyin of user CCB buffer failed"); + ctx->result.exec.status = DAX_SUBMIT_ERR_CCB_ARR_MMU_MISS; + return 0; + } + + /* check to see if ca_buf[idx] .. ca_buf[idx + nccbs] are available */ + for (i = idx; i < idx + nccbs; i++) { + if (ctx->ca_buf[i].status == CCA_STAT_NOT_COMPLETED) { + dax_dbg("CA range not available, dequeue needed"); + ctx->result.exec.status = DAX_SUBMIT_ERR_NO_CA_AVAIL; + return 0; + } + } + dax_unlock_pages(ctx, idx, nccbs); + + ctx->result.exec.status = dax_preprocess_usr_ccbs(ctx, idx, nccbs); + if (ctx->result.exec.status != DAX_SUBMIT_OK) + return 0; + + ctx->result.exec.status = dax_lock_pages(ctx, idx, nccbs, + &ctx->result.exec.status_data); + if (ctx->result.exec.status != DAX_SUBMIT_OK) + return 0; + + if (dax_debug & DAX_DBG_FLG_BASIC) + dax_prt_ccbs(ctx->ccb_buf, nccbs); + + hv_rv = sun4v_ccb_submit(ctx->ccb_buf_ra, count, + HV_CCB_QUERY_CMD | HV_CCB_VA_SECONDARY, 0, + &accepted_len, &ctx->result.exec.status_data); + + switch (hv_rv) { + case HV_EOK: + /* + * Hcall succeeded with no errors but the accepted + * length may be less than the requested length. The + * only way the driver can resubmit the remainder is + * to wait for completion of the submitted CCBs since + * there is no way to guarantee the ordering semantics + * required by the client applications. Therefore we + * let the user library deal with resubmissions. + */ + ctx->result.exec.status = DAX_SUBMIT_OK; + break; + case HV_EWOULDBLOCK: + /* + * This is a transient HV API error. The user library + * can retry. + */ + dax_dbg("hcall returned HV_EWOULDBLOCK"); + ctx->result.exec.status = DAX_SUBMIT_ERR_WOULDBLOCK; + break; + case HV_ENOMAP: + /* + * HV was unable to translate a VA. The VA it could + * not translate is returned in the status_data param. + */ + dax_dbg("hcall returned HV_ENOMAP"); + ctx->result.exec.status = DAX_SUBMIT_ERR_NOMAP; + break; + case HV_EINVAL: + /* + * This is the result of an invalid user CCB as HV is + * validating some of the user CCB fields. Pass this + * error back to the user. There is no supporting info + * to isolate the invalid field. + */ + dax_dbg("hcall returned HV_EINVAL"); + ctx->result.exec.status = DAX_SUBMIT_ERR_CCB_INVAL; + break; + case HV_ENOACCESS: + /* + * HV found a VA that did not have the appropriate + * permissions (such as the w bit). The VA in question + * is returned in status_data param. + */ + dax_dbg("hcall returned HV_ENOACCESS"); + ctx->result.exec.status = DAX_SUBMIT_ERR_NOACCESS; + break; + case HV_EUNAVAILABLE: + /* + * The requested CCB operation could not be performed + * at this time. Return the specific unavailable code + * in the status_data field. + */ + dax_dbg("hcall returned HV_EUNAVAILABLE"); + ctx->result.exec.status = DAX_SUBMIT_ERR_UNAVAIL; + break; + default: + ctx->result.exec.status = DAX_SUBMIT_ERR_INTERNAL; + dax_dbg("unknown hcall return value (%ld)", hv_rv); + break; + } + + /* unlock pages associated with the unaccepted CCBs */ + naccepted = accepted_len / sizeof(struct dax_ccb); + dax_unlock_pages(ctx, idx + naccepted, nccbs - naccepted); + + /* mark unaccepted CCBs as not completed */ + for (i = idx + naccepted; i < idx + nccbs; i++) + ctx->ca_buf[i].status = CCA_STAT_COMPLETED; + + ctx->ccb_count += naccepted; + ctx->fail_count += nccbs - naccepted; + + dax_dbg("hcall rv=%ld, accepted_len=%ld, status_data=0x%llx, ret status=%d", + hv_rv, accepted_len, ctx->result.exec.status_data, + ctx->result.exec.status); + + if (count == accepted_len) + ctx->client = NULL; /* no read needed to complete protocol */ + return accepted_len; +} |