diff options
author | Israel Rukshin <israelr@mellanox.com> | 2019-06-11 18:52:47 +0300 |
---|---|---|
committer | Jason Gunthorpe <jgg@mellanox.com> | 2019-06-24 11:49:27 -0300 |
commit | b76a439982f8483beb2ffcfe93be50026940030a (patch) | |
tree | 8da11022e23b94fc9c9d60b420c85887790552d6 /drivers/infiniband/ulp/iser/iscsi_iser.h | |
parent | 38ca87c6f1e514686d4a385246d1afe1e1f2e482 (diff) |
IB/iser: Use IB_WR_REG_MR_INTEGRITY for PI handover
Using this new API reduces iSER code complexity.
It also reduces the maximum number of work requests per task and the need
of dealing with multiple MRs (and their registrations and invalidations)
per task. It is done by using a single WR and a special MR type
(IB_MR_TYPE_INTEGRITY) for PI operation.
The setup of the tested benchmark:
- 2 servers with 24 cores (1 initiator and 1 target)
- 24 target sessions with 1 LUN each
- ramdisk backstore
- PI active
Performance results running fio (24 jobs, 128 iodepth) using
write_generate=0 and read_verify=0 (w/w.o patch):
bs IOPS(read) IOPS(write)
---- ---------- ----------
512 1236.6K/1164.3K 1357.2K/1332.8K
1k 1196.5K/1163.8K 1348.4K/1262.7K
2k 1016.7K/921950 1003.7K/931230
4k 662728/600545 595423/501513
8k 385954/384345 333775/277090
16k 222864/222820 170317/170671
32k 116869/114896 82331/82244
64k 55205/54931 40264/40021
Using write_generate=1 and read_verify=1 (w/w.o patch):
bs IOPS(read) IOPS(write)
---- ---------- ----------
512 1090.1K/1030.9K 1303.9K/1101.4K
1k 1057.7K/904583 1318.4K/988085
2k 965226/638799 1008.6K/692514
4k 555479/410151 542414/414517
8k 298675/224964 264729/237508
16k 133485/122481 164625/138647
32k 74329/67615 80143/78743
64k 35716/35519 39294/37334
We get performance improvement at all block sizes.
The most significant improvement is when writing 4k bs (almost 30% more
iops).
Signed-off-by: Israel Rukshin <israelr@mellanox.com>
Reviewed-by: Max Gurtovoy <maxg@mellanox.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
Diffstat (limited to 'drivers/infiniband/ulp/iser/iscsi_iser.h')
-rw-r--r-- | drivers/infiniband/ulp/iser/iscsi_iser.h | 38 |
1 files changed, 8 insertions, 30 deletions
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h index 36d525110fd2..6bf9eaa8ec96 100644 --- a/drivers/infiniband/ulp/iser/iscsi_iser.h +++ b/drivers/infiniband/ulp/iser/iscsi_iser.h @@ -225,13 +225,11 @@ enum iser_desc_type { ISCSI_TX_DATAOUT }; -/* Maximum number of work requests per task: - * Data memory region local invalidate + fast registration - * Protection memory region local invalidate + fast registration - * Signature memory region local invalidate + fast registration - * PDU send +/* + * Maximum number of work requests per task + * (invalidate, registration, send) */ -#define ISER_MAX_WRS 7 +#define ISER_MAX_WRS 3 /** * struct iser_tx_desc - iSER TX descriptor @@ -247,9 +245,6 @@ enum iser_desc_type { * @mapped: Is the task header mapped * @wr_idx: Current WR index * @wrs: Array of WRs per task - * @data_reg: Data buffer registration details - * @prot_reg: Protection buffer registration details - * @sig_attrs: Signature attributes */ struct iser_tx_desc { struct iser_ctrl iser_header; @@ -264,11 +259,7 @@ struct iser_tx_desc { union iser_wr { struct ib_send_wr send; struct ib_reg_wr fast_reg; - struct ib_sig_handover_wr sig; } wrs[ISER_MAX_WRS]; - struct iser_mem_reg data_reg; - struct iser_mem_reg prot_reg; - struct ib_sig_attrs sig_attrs; }; #define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \ @@ -388,6 +379,7 @@ struct iser_device { * * @mr: memory region * @fmr_pool: pool of fmrs + * @sig_mr: signature memory region * @page_vec: fast reg page list used by fmr pool * @mr_valid: is mr valid indicator */ @@ -396,36 +388,22 @@ struct iser_reg_resources { struct ib_mr *mr; struct ib_fmr_pool *fmr_pool; }; + struct ib_mr *sig_mr; struct iser_page_vec *page_vec; u8 mr_valid:1; }; /** - * struct iser_pi_context - Protection information context - * - * @rsc: protection buffer registration resources - * @sig_mr: signature enable memory region - * @sig_mr_valid: is sig_mr valid indicator - * @sig_protected: is region protected indicator - */ -struct iser_pi_context { - struct iser_reg_resources rsc; - struct ib_mr *sig_mr; - u8 sig_mr_valid:1; - u8 sig_protected:1; -}; - -/** * struct iser_fr_desc - Fast registration descriptor * * @list: entry in connection fastreg pool * @rsc: data buffer registration resources - * @pi_ctx: protection information context + * @sig_protected: is region protected indicator */ struct iser_fr_desc { struct list_head list; struct iser_reg_resources rsc; - struct iser_pi_context *pi_ctx; + bool sig_protected; struct list_head all_list; }; |