xprtrdma: Eliminate rpcrdma_ia::ri_device

Clean up.

Since commit 54cbd6b0c6b9 ("xprtrdma: Delay DMA mapping Send and
Receive buffers"), a pointer to the device is now saved in each
regbuf when it is DMA mapped.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>

diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c
index a2a2e01cb5dd47d7dbd214c48bf6eb1b1829cf87..7cd27184ecd10abe5d6208244244bb4f00dcd1e2 100644 (file)
--- a/net/sunrpc/xprtrdma/frwr_ops.c
+++ b/net/sunrpc/xprtrdma/frwr_ops.c
@@ -82,13 +82,13 @@
 
 /**
  * frwr_is_supported - Check if device supports FRWR
- * @ia: interface adapter to check
+ * @device: interface adapter to check
  *
  * Returns true if device supports FRWR, otherwise false
  */
-bool frwr_is_supported(struct rpcrdma_ia *ia)
+bool frwr_is_supported(struct ib_device *device)
 {
-       struct ib_device_attr *attrs = &ia->ri_device->attrs;
+       struct ib_device_attr *attrs = &device->attrs;
 
        if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS))
                goto out_not_supported;
@@ -98,7 +98,7 @@ bool frwr_is_supported(struct rpcrdma_ia *ia)
 
 out_not_supported:
        pr_info("rpcrdma: 'frwr' mode is not supported by device %s\n",
-               ia->ri_device->name);
+               device->name);
        return false;
 }
 
@@ -131,7 +131,7 @@ frwr_mr_recycle_worker(struct work_struct *work)
 
        if (mr->mr_dir != DMA_NONE) {
                trace_xprtrdma_mr_unmap(mr);
-               ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,
+               ib_dma_unmap_sg(r_xprt->rx_ia.ri_id->device,
                                mr->mr_sg, mr->mr_nents, mr->mr_dir);
                mr->mr_dir = DMA_NONE;
        }
@@ -211,7 +211,7 @@ out_list_err:
 int frwr_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
              struct rpcrdma_create_data_internal *cdata)
 {
-       struct ib_device_attr *attrs = &ia->ri_device->attrs;
+       struct ib_device_attr *attrs = &ia->ri_id->device->attrs;
        int max_qp_wr, depth, delta;
 
        ia->ri_mrtype = IB_MR_TYPE_MEM_REG;
@@ -253,7 +253,7 @@ int frwr_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
                } while (delta > 0);
        }
 
-       max_qp_wr = ia->ri_device->attrs.max_qp_wr;
+       max_qp_wr = ia->ri_id->device->attrs.max_qp_wr;
        max_qp_wr -= RPCRDMA_BACKWARD_WRS;
        max_qp_wr -= 1;
        if (max_qp_wr < RPCRDMA_MIN_SLOT_TABLE)
@@ -436,7 +436,8 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt,
        }
        mr->mr_dir = rpcrdma_data_dir(writing);
 
-       mr->mr_nents = ib_dma_map_sg(ia->ri_device, mr->mr_sg, i, mr->mr_dir);
+       mr->mr_nents =
+               ib_dma_map_sg(ia->ri_id->device, mr->mr_sg, i, mr->mr_dir);
        if (!mr->mr_nents)
                goto out_dmamap_err;
 

diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 1ed761a12f863df79db449cafba2a49c5ca70837..672993cee70d672e9c75b6b4b0a6414c4d9741aa 100644 (file)
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -250,7 +250,7 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event)
        case RDMA_CM_EVENT_DEVICE_REMOVAL:
 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
                pr_info("rpcrdma: removing device %s for %s:%s\n",
-                       ia->ri_device->name,
+                       ia->ri_id->device->name,
                        rpcrdma_addrstr(r_xprt), rpcrdma_portstr(r_xprt));
 #endif
                set_bit(RPCRDMA_IAF_REMOVING, &ia->ri_flags);
@@ -259,7 +259,6 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event)
                wait_for_completion(&ia->ri_remove_done);
 
                ia->ri_id = NULL;
-               ia->ri_device = NULL;
                /* Return 1 to ensure the core destroys the id. */
                return 1;
        case RDMA_CM_EVENT_ESTABLISHED:
@@ -294,7 +293,7 @@ disconnected:
 
        dprintk("RPC:       %s: %s:%s on %s/frwr: %s\n", __func__,
                rpcrdma_addrstr(r_xprt), rpcrdma_portstr(r_xprt),
-               ia->ri_device->name, rdma_event_msg(event->event));
+               ia->ri_id->device->name, rdma_event_msg(event->event));
        return 0;
 }
 
@@ -373,9 +372,8 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt)
                rc = PTR_ERR(ia->ri_id);
                goto out_err;
        }
-       ia->ri_device = ia->ri_id->device;
 
-       ia->ri_pd = ib_alloc_pd(ia->ri_device, 0);
+       ia->ri_pd = ib_alloc_pd(ia->ri_id->device, 0);
        if (IS_ERR(ia->ri_pd)) {
                rc = PTR_ERR(ia->ri_pd);
                pr_err("rpcrdma: ib_alloc_pd() returned %d\n", rc);
@@ -384,12 +382,12 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt)
 
        switch (xprt_rdma_memreg_strategy) {
        case RPCRDMA_FRWR:
-               if (frwr_is_supported(ia))
+               if (frwr_is_supported(ia->ri_id->device))
                        break;
                /*FALLTHROUGH*/
        default:
                pr_err("rpcrdma: Device %s does not support memreg mode %d\n",
-                      ia->ri_device->name, xprt_rdma_memreg_strategy);
+                      ia->ri_id->device->name, xprt_rdma_memreg_strategy);
                rc = -EINVAL;
                goto out_err;
        }
@@ -471,7 +469,6 @@ rpcrdma_ia_close(struct rpcrdma_ia *ia)
                rdma_destroy_id(ia->ri_id);
        }
        ia->ri_id = NULL;
-       ia->ri_device = NULL;
 
        /* If the pd is still busy, xprtrdma missed freeing a resource */
        if (ia->ri_pd && !IS_ERR(ia->ri_pd))
@@ -491,7 +488,7 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
        unsigned int max_sge;
        int rc;
 
-       max_sge = min_t(unsigned int, ia->ri_device->attrs.max_send_sge,
+       max_sge = min_t(unsigned int, ia->ri_id->device->attrs.max_send_sge,
                        RPCRDMA_MAX_SEND_SGES);
        if (max_sge < RPCRDMA_MIN_SEND_SGES) {
                pr_warn("rpcrdma: HCA provides only %d send SGEs\n", max_sge);
@@ -526,16 +523,16 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
        init_waitqueue_head(&ep->rep_connect_wait);
        ep->rep_receive_count = 0;
 
-       sendcq = ib_alloc_cq(ia->ri_device, NULL,
+       sendcq = ib_alloc_cq(ia->ri_id->device, NULL,
                             ep->rep_attr.cap.max_send_wr + 1,
-                            ia->ri_device->num_comp_vectors > 1 ? 1 : 0,
+                            ia->ri_id->device->num_comp_vectors > 1 ? 1 : 0,
                             IB_POLL_WORKQUEUE);
        if (IS_ERR(sendcq)) {
                rc = PTR_ERR(sendcq);
                goto out1;
        }
 
-       recvcq = ib_alloc_cq(ia->ri_device, NULL,
+       recvcq = ib_alloc_cq(ia->ri_id->device, NULL,
                             ep->rep_attr.cap.max_recv_wr + 1,
                             0, IB_POLL_WORKQUEUE);
        if (IS_ERR(recvcq)) {
@@ -561,7 +558,7 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
        /* Client offers RDMA Read but does not initiate */
        ep->rep_remote_cma.initiator_depth = 0;
        ep->rep_remote_cma.responder_resources =
-               min_t(int, U8_MAX, ia->ri_device->attrs.max_qp_rd_atom);
+               min_t(int, U8_MAX, ia->ri_id->device->attrs.max_qp_rd_atom);
 
        /* Limit transport retries so client can detect server
         * GID changes quickly. RPC layer handles re-establishing
@@ -673,7 +670,7 @@ rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt, struct rpcrdma_ep *ep,
         */
        old = id;
        rc = -ENETUNREACH;
-       if (ia->ri_device != id->device) {
+       if (ia->ri_id->device != id->device) {
                pr_err("rpcrdma: can't reconnect on different device!\n");
                goto out_destroy;
        }
@@ -1296,7 +1293,7 @@ rpcrdma_mr_unmap_and_put(struct rpcrdma_mr *mr)
 
        if (mr->mr_dir != DMA_NONE) {
                trace_xprtrdma_mr_unmap(mr);
-               ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,
+               ib_dma_unmap_sg(r_xprt->rx_ia.ri_id->device,
                                mr->mr_sg, mr->mr_nents, mr->mr_dir);
                mr->mr_dir = DMA_NONE;
        }
@@ -1429,7 +1426,7 @@ bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags)
 bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,
                              struct rpcrdma_regbuf *rb)
 {
-       struct ib_device *device = r_xprt->rx_ia.ri_device;
+       struct ib_device *device = r_xprt->rx_ia.ri_id->device;
 
        if (rb->rg_direction == DMA_NONE)
                return false;

diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index f8563937c8c66479b3327c2307996391fbc761aa..40912bb34b648f05edb3502aa9fd1f7e6c2c3a0c 100644 (file)
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -66,11 +66,8 @@
  * Interface Adapter -- one per transport instance
  */
 struct rpcrdma_ia {
-       struct ib_device        *ri_device;
        struct rdma_cm_id       *ri_id;
        struct ib_pd            *ri_pd;
-       struct completion       ri_done;
-       struct completion       ri_remove_done;
        int                     ri_async_rc;
        unsigned int            ri_max_segs;
        unsigned int            ri_max_frwr_depth;
@@ -80,6 +77,8 @@ struct rpcrdma_ia {
        bool                    ri_implicit_roundup;
        enum ib_mr_type         ri_mrtype;
        unsigned long           ri_flags;
+       struct completion       ri_done;
+       struct completion       ri_remove_done;
 };
 
 enum {
@@ -585,7 +584,7 @@ rpcrdma_data_dir(bool writing)
 
 /* Memory registration calls xprtrdma/frwr_ops.c
  */
-bool frwr_is_supported(struct rpcrdma_ia *);
+bool frwr_is_supported(struct ib_device *device);
 int frwr_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
              struct rpcrdma_create_data_internal *cdata);
 int frwr_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr);