From 434dda422cb2d52af4c75857c3f2084fe8f3a2ba Mon Sep 17 00:00:00 2001 From: Sergey Gorenko Date: Mon, 21 May 2018 18:55:53 +0300 Subject: [PATCH] IB/iser: Do not reduce max_sectors The iSER driver reduces max_sectors. For example, if you load the ib_iser module with max_sectors=1024, you will see that /sys/class/block//queue/max_hw_sectors_kb is 508. It is an incorrect value. The expected value is (max_sectors * sector_size) / 1024 = 512. Reducing of max_sectors can cause performance degradation due to unnecessary splitting of IO requests. The number of pages per MR has been fixed here, so there is no longer any need to reduce max_sectors. Fixes: 9c674815d346 ("IB/iser: Fix max_sectors calculation") Signed-off-by: Sergey Gorenko Reviewed-by: Israel Rukshin Reviewed-by: Max Gurtovoy Cc: Christoph Hellwig Cc: Sagi Grimberg Acked-by: Sagi Grimberg Signed-off-by: Doug Ledford --- drivers/infiniband/ulp/iser/iscsi_iser.c | 12 +++++------ drivers/infiniband/ulp/iser/iscsi_iser.h | 2 ++ drivers/infiniband/ulp/iser/iser_initiator.c | 2 +- drivers/infiniband/ulp/iser/iser_verbs.c | 21 +++++++++++++++++--- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c index 0336643c2ed6..9a6434c31db2 100644 --- a/drivers/infiniband/ulp/iser/iscsi_iser.c +++ b/drivers/infiniband/ulp/iser/iscsi_iser.c @@ -665,19 +665,17 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep, goto free_host; } - /* - * FRs or FMRs can only map up to a (device) page per entry, but if the - * first entry is misaligned we'll end up using using two entries - * (head and tail) for a single page worth data, so we have to drop - * one segment from the calculation. - */ - max_fr_sectors = ((shost->sg_tablesize - 1) * PAGE_SIZE) >> 9; + max_fr_sectors = (shost->sg_tablesize * PAGE_SIZE) >> 9; shost->max_sectors = min(iser_max_sectors, max_fr_sectors); iser_dbg("iser_conn %p, sg_tablesize %u, max_sectors %u\n", iser_conn, shost->sg_tablesize, shost->max_sectors); + if (shost->max_sectors < iser_max_sectors) + iser_warn("max_sectors was reduced from %u to %u\n", + iser_max_sectors, shost->max_sectors); + if (cmds_max > max_cmds) { iser_info("cmds_max changed from %u to %u\n", cmds_max, max_cmds); diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h index c1ae4aeae2f9..c30ceb08db21 100644 --- a/drivers/infiniband/ulp/iser/iscsi_iser.h +++ b/drivers/infiniband/ulp/iser/iscsi_iser.h @@ -498,6 +498,7 @@ struct ib_conn { * @rx_descs: rx buffers array (cyclic buffer) * @num_rx_descs: number of rx descriptors * @scsi_sg_tablesize: scsi host sg_tablesize + * @pages_per_mr: maximum pages available for registration */ struct iser_conn { struct ib_conn ib_conn; @@ -520,6 +521,7 @@ struct iser_conn { struct iser_rx_desc *rx_descs; u32 num_rx_descs; unsigned short scsi_sg_tablesize; + unsigned short pages_per_mr; bool snd_w_inv; }; diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c index df49c4eb67f7..ca858d6bd37a 100644 --- a/drivers/infiniband/ulp/iser/iser_initiator.c +++ b/drivers/infiniband/ulp/iser/iser_initiator.c @@ -251,7 +251,7 @@ int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2; if (device->reg_ops->alloc_reg_res(ib_conn, session->scsi_cmds_max, - iser_conn->scsi_sg_tablesize)) + iser_conn->pages_per_mr)) goto create_rdma_reg_res_failed; if (iser_alloc_login_buf(iser_conn)) diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c index 56b7240a3fc3..616d978cbf2b 100644 --- a/drivers/infiniband/ulp/iser/iser_verbs.c +++ b/drivers/infiniband/ulp/iser/iser_verbs.c @@ -703,19 +703,34 @@ iser_calc_scsi_params(struct iser_conn *iser_conn, unsigned int max_sectors) { struct iser_device *device = iser_conn->ib_conn.device; + struct ib_device_attr *attr = &device->ib_device->attrs; unsigned short sg_tablesize, sup_sg_tablesize; + unsigned short reserved_mr_pages; + + /* + * FRs without SG_GAPS or FMRs can only map up to a (device) page per + * entry, but if the first entry is misaligned we'll end up using two + * entries (head and tail) for a single page worth data, so one + * additional entry is required. + */ + if ((attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) && + (attr->device_cap_flags & IB_DEVICE_SG_GAPS_REG)) + reserved_mr_pages = 0; + else + reserved_mr_pages = 1; sg_tablesize = DIV_ROUND_UP(max_sectors * 512, SIZE_4K); - if (device->ib_device->attrs.device_cap_flags & - IB_DEVICE_MEM_MGT_EXTENSIONS) + if (attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) sup_sg_tablesize = min_t( uint, ISCSI_ISER_MAX_SG_TABLESIZE, - device->ib_device->attrs.max_fast_reg_page_list_len); + attr->max_fast_reg_page_list_len - reserved_mr_pages); else sup_sg_tablesize = ISCSI_ISER_MAX_SG_TABLESIZE; iser_conn->scsi_sg_tablesize = min(sg_tablesize, sup_sg_tablesize); + iser_conn->pages_per_mr = + iser_conn->scsi_sg_tablesize + reserved_mr_pages; } /** -- 2.30.2