RDS: Base init_depth and responder_resources on hw values

Instead of using a constant for initiator_depth and
responder_resources, read the per-QP values when the
device is enumerated, and then use these values when creating
the connection.

Signed-off-by: Andy Grover <andy.grover@oracle.com>

diff --git a/net/rds/ib.c b/net/rds/ib.c
index f0d29656baff725ede7f240f5087d61ad625ae4b..72a5116c11de5e6bf6db24d8cb2be2f70abb2bc3 100644 (file)
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -91,6 +91,9 @@ void rds_ib_add_one(struct ib_device *device)
                        min_t(unsigned int, dev_attr->max_fmr, fmr_pool_size) :
                        fmr_pool_size;
 
+       rds_ibdev->max_initiator_depth = dev_attr->max_qp_init_rd_atom;
+       rds_ibdev->max_responder_resources = dev_attr->max_qp_rd_atom;
+
        rds_ibdev->dev = device;
        rds_ibdev->pd = ib_alloc_pd(device);
        if (IS_ERR(rds_ibdev->pd))

diff --git a/net/rds/ib.h b/net/rds/ib.h
index d2fd0aa4fde7da2b8725d372f55d540592f07c79..a303f13111c2cfecea5fb49f4b88ef2205cedf02 100644 (file)
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -164,6 +164,8 @@ struct rds_ib_device {
        unsigned int            max_fmrs;
        int                     max_sge;
        unsigned int            max_wrs;
+       unsigned int            max_initiator_depth;
+       unsigned int            max_responder_resources;
        spinlock_t              spinlock;       /* protect the above */
 };
 

diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index b46bc2f22ab6778f1865a546b7233530c075be72..3134336ca17dea459d3ca7524041b70a084e912f 100644 (file)
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -153,18 +153,25 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_even
 static void rds_ib_cm_fill_conn_param(struct rds_connection *conn,
                        struct rdma_conn_param *conn_param,
                        struct rds_ib_connect_private *dp,
-                       u32 protocol_version)
+                       u32 protocol_version,
+                       u32 max_responder_resources,
+                       u32 max_initiator_depth)
 {
+       struct rds_ib_connection *ic = conn->c_transport_data;
+       struct rds_ib_device *rds_ibdev;
+
        memset(conn_param, 0, sizeof(struct rdma_conn_param));
-       /* XXX tune these? */
-       conn_param->responder_resources = 1;
-       conn_param->initiator_depth = 1;
+
+       rds_ibdev = ib_get_client_data(ic->i_cm_id->device, &rds_ib_client);
+
+       conn_param->responder_resources =
+               min_t(u32, rds_ibdev->max_responder_resources, max_responder_resources);
+       conn_param->initiator_depth =
+               min_t(u32, rds_ibdev->max_initiator_depth, max_initiator_depth);
        conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7);
        conn_param->rnr_retry_count = 7;
 
        if (dp) {
-               struct rds_ib_connection *ic = conn->c_transport_data;
-
                memset(dp, 0, sizeof(*dp));
                dp->dp_saddr = conn->c_laddr;
                dp->dp_daddr = conn->c_faddr;
@@ -479,7 +486,9 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
                goto out;
        }
 
-       rds_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version);
+       rds_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version,
+               event->param.conn.responder_resources,
+               event->param.conn.initiator_depth);
 
        /* rdma_accept() calls rdma_reject() internally if it fails */
        err = rdma_accept(cm_id, &conn_param);
@@ -516,8 +525,8 @@ int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id)
                goto out;
        }
 
-       rds_ib_cm_fill_conn_param(conn, &conn_param, &dp, RDS_PROTOCOL_VERSION);
-
+       rds_ib_cm_fill_conn_param(conn, &conn_param, &dp, RDS_PROTOCOL_VERSION,
+               UINT_MAX, UINT_MAX);
        ret = rdma_connect(cm_id, &conn_param);
        if (ret)
                rds_ib_conn_error(conn, "rdma_connect failed (%d)\n", ret);

diff --git a/net/rds/rdma.c b/net/rds/rdma.c
index a7019df38c7031dbfe4fa349d25383443e84a607..abbc2979e7e5c142b858b7d8f77d8ec864b8ea71 100644 (file)
--- a/net/rds/rdma.c
+++ b/net/rds/rdma.c
@@ -745,7 +745,6 @@ int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
                rm->atomic.op_swap_add = args->fadd.add;
        }
 
-       rm->m_rdma_cookie = args->cookie;
        rm->atomic.op_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);
        rm->atomic.op_recverr = rs->rs_recverr;
        rm->atomic.op_sg = rds_message_alloc_sgs(rm, 1);
@@ -779,7 +778,7 @@ int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
                rm->atomic.op_notifier->n_status = RDS_RDMA_SUCCESS;
        }
 
-       rm->atomic.op_rkey = rds_rdma_cookie_key(rm->m_rdma_cookie);
+       rm->atomic.op_rkey = rds_rdma_cookie_key(args->cookie);
        rm->atomic.op_remote_addr = args->remote_addr + rds_rdma_cookie_offset(args->cookie);
 
        rm->atomic.op_active = 1;