IB/uverbs: Add WQ support

User space applications which use RSS functionality need to create
a work queue object (WQ). The lifetime of such an object is:
 * Create a WQ
 * Modify the WQ from reset to init state.
 * Use the WQ (by downstream patches).
 * Destroy the WQ.

These commands are added to the uverbs API.

Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
Signed-off-by: Matan Barak <matanb@mellanox.com>
Reviewed-by: Sagi Grimberg <sagi@rimberg.me>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
index 612ccfd39bf98181693f07c5dfcc661dc8191e39..74776c6531f4a6cd88ad108a23ac7e98dcb8e27a 100644 (file)
--- a/drivers/infiniband/core/uverbs.h
+++ b/drivers/infiniband/core/uverbs.h
@@ -162,6 +162,10 @@ struct ib_uqp_object {
        struct ib_uxrcd_object *uxrcd;
 };
 
+struct ib_uwq_object {
+       struct ib_uevent_object uevent;
+};
+
 struct ib_ucq_object {
        struct ib_uobject       uobject;
        struct ib_uverbs_file  *uverbs_file;
@@ -181,6 +185,7 @@ extern struct idr ib_uverbs_qp_idr;
 extern struct idr ib_uverbs_srq_idr;
 extern struct idr ib_uverbs_xrcd_idr;
 extern struct idr ib_uverbs_rule_idr;
+extern struct idr ib_uverbs_wq_idr;
 
 void idr_remove_uobj(struct idr *idp, struct ib_uobject *uobj);
 
@@ -199,6 +204,7 @@ void ib_uverbs_release_uevent(struct ib_uverbs_file *file,
 void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context);
 void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr);
 void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr);
+void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr);
 void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr);
 void ib_uverbs_event_handler(struct ib_event_handler *handler,
                             struct ib_event *event);
@@ -275,5 +281,8 @@ IB_UVERBS_DECLARE_EX_CMD(destroy_flow);
 IB_UVERBS_DECLARE_EX_CMD(query_device);
 IB_UVERBS_DECLARE_EX_CMD(create_cq);
 IB_UVERBS_DECLARE_EX_CMD(create_qp);
+IB_UVERBS_DECLARE_EX_CMD(create_wq);
+IB_UVERBS_DECLARE_EX_CMD(modify_wq);
+IB_UVERBS_DECLARE_EX_CMD(destroy_wq);
 
 #endif /* UVERBS_H */

diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 1a8babb8ee3c4d328af7e08063cef8f147086a10..22e6173916579d44c033609154a1a151e0f95d49 100644 (file)
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -57,6 +57,7 @@ static struct uverbs_lock_class ah_lock_class = { .name = "AH-uobj" };
 static struct uverbs_lock_class srq_lock_class = { .name = "SRQ-uobj" };
 static struct uverbs_lock_class xrcd_lock_class = { .name = "XRCD-uobj" };
 static struct uverbs_lock_class rule_lock_class = { .name = "RULE-uobj" };
+static struct uverbs_lock_class wq_lock_class = { .name = "WQ-uobj" };
 
 /*
  * The ib_uobject locking scheme is as follows:
@@ -243,6 +244,16 @@ static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context)
        return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0);
 }
 
+static struct ib_wq *idr_read_wq(int wq_handle, struct ib_ucontext *context)
+{
+       return idr_read_obj(&ib_uverbs_wq_idr, wq_handle, context, 0);
+}
+
+static void put_wq_read(struct ib_wq *wq)
+{
+       put_uobj_read(wq->uobject);
+}
+
 static struct ib_qp *idr_write_qp(int qp_handle, struct ib_ucontext *context)
 {
        struct ib_uobject *uobj;
@@ -326,6 +337,7 @@ ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file,
        INIT_LIST_HEAD(&ucontext->qp_list);
        INIT_LIST_HEAD(&ucontext->srq_list);
        INIT_LIST_HEAD(&ucontext->ah_list);
+       INIT_LIST_HEAD(&ucontext->wq_list);
        INIT_LIST_HEAD(&ucontext->xrcd_list);
        INIT_LIST_HEAD(&ucontext->rule_list);
        rcu_read_lock();
@@ -3056,6 +3068,237 @@ static int kern_spec_to_ib_spec(struct ib_uverbs_flow_spec *kern_spec,
        return 0;
 }
 
+int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file,
+                          struct ib_device *ib_dev,
+                          struct ib_udata *ucore,
+                          struct ib_udata *uhw)
+{
+       struct ib_uverbs_ex_create_wq     cmd = {};
+       struct ib_uverbs_ex_create_wq_resp resp = {};
+       struct ib_uwq_object           *obj;
+       int err = 0;
+       struct ib_cq *cq;
+       struct ib_pd *pd;
+       struct ib_wq *wq;
+       struct ib_wq_init_attr wq_init_attr = {};
+       size_t required_cmd_sz;
+       size_t required_resp_len;
+
+       required_cmd_sz = offsetof(typeof(cmd), max_sge) + sizeof(cmd.max_sge);
+       required_resp_len = offsetof(typeof(resp), wqn) + sizeof(resp.wqn);
+
+       if (ucore->inlen < required_cmd_sz)
+               return -EINVAL;
+
+       if (ucore->outlen < required_resp_len)
+               return -ENOSPC;
+
+       if (ucore->inlen > sizeof(cmd) &&
+           !ib_is_udata_cleared(ucore, sizeof(cmd),
+                                ucore->inlen - sizeof(cmd)))
+               return -EOPNOTSUPP;
+
+       err = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
+       if (err)
+               return err;
+
+       if (cmd.comp_mask)
+               return -EOPNOTSUPP;
+
+       obj = kmalloc(sizeof(*obj), GFP_KERNEL);
+       if (!obj)
+               return -ENOMEM;
+
+       init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext,
+                 &wq_lock_class);
+       down_write(&obj->uevent.uobject.mutex);
+       pd  = idr_read_pd(cmd.pd_handle, file->ucontext);
+       if (!pd) {
+               err = -EINVAL;
+               goto err_uobj;
+       }
+
+       cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
+       if (!cq) {
+               err = -EINVAL;
+               goto err_put_pd;
+       }
+
+       wq_init_attr.cq = cq;
+       wq_init_attr.max_sge = cmd.max_sge;
+       wq_init_attr.max_wr = cmd.max_wr;
+       wq_init_attr.wq_context = file;
+       wq_init_attr.wq_type = cmd.wq_type;
+       wq_init_attr.event_handler = ib_uverbs_wq_event_handler;
+       obj->uevent.events_reported = 0;
+       INIT_LIST_HEAD(&obj->uevent.event_list);
+       wq = pd->device->create_wq(pd, &wq_init_attr, uhw);
+       if (IS_ERR(wq)) {
+               err = PTR_ERR(wq);
+               goto err_put_cq;
+       }
+
+       wq->uobject = &obj->uevent.uobject;
+       obj->uevent.uobject.object = wq;
+       wq->wq_type = wq_init_attr.wq_type;
+       wq->cq = cq;
+       wq->pd = pd;
+       wq->device = pd->device;
+       wq->wq_context = wq_init_attr.wq_context;
+       atomic_set(&wq->usecnt, 0);
+       atomic_inc(&pd->usecnt);
+       atomic_inc(&cq->usecnt);
+       wq->uobject = &obj->uevent.uobject;
+       obj->uevent.uobject.object = wq;
+       err = idr_add_uobj(&ib_uverbs_wq_idr, &obj->uevent.uobject);
+       if (err)
+               goto destroy_wq;
+
+       memset(&resp, 0, sizeof(resp));
+       resp.wq_handle = obj->uevent.uobject.id;
+       resp.max_sge = wq_init_attr.max_sge;
+       resp.max_wr = wq_init_attr.max_wr;
+       resp.wqn = wq->wq_num;
+       resp.response_length = required_resp_len;
+       err = ib_copy_to_udata(ucore,
+                              &resp, resp.response_length);
+       if (err)
+               goto err_copy;
+
+       put_pd_read(pd);
+       put_cq_read(cq);
+
+       mutex_lock(&file->mutex);
+       list_add_tail(&obj->uevent.uobject.list, &file->ucontext->wq_list);
+       mutex_unlock(&file->mutex);
+
+       obj->uevent.uobject.live = 1;
+       up_write(&obj->uevent.uobject.mutex);
+       return 0;
+
+err_copy:
+       idr_remove_uobj(&ib_uverbs_wq_idr, &obj->uevent.uobject);
+destroy_wq:
+       ib_destroy_wq(wq);
+err_put_cq:
+       put_cq_read(cq);
+err_put_pd:
+       put_pd_read(pd);
+err_uobj:
+       put_uobj_write(&obj->uevent.uobject);
+
+       return err;
+}
+
+int ib_uverbs_ex_destroy_wq(struct ib_uverbs_file *file,
+                           struct ib_device *ib_dev,
+                           struct ib_udata *ucore,
+                           struct ib_udata *uhw)
+{
+       struct ib_uverbs_ex_destroy_wq  cmd = {};
+       struct ib_uverbs_ex_destroy_wq_resp     resp = {};
+       struct ib_wq                    *wq;
+       struct ib_uobject               *uobj;
+       struct ib_uwq_object            *obj;
+       size_t required_cmd_sz;
+       size_t required_resp_len;
+       int                             ret;
+
+       required_cmd_sz = offsetof(typeof(cmd), wq_handle) + sizeof(cmd.wq_handle);
+       required_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
+
+       if (ucore->inlen < required_cmd_sz)
+               return -EINVAL;
+
+       if (ucore->outlen < required_resp_len)
+               return -ENOSPC;
+
+       if (ucore->inlen > sizeof(cmd) &&
+           !ib_is_udata_cleared(ucore, sizeof(cmd),
+                                ucore->inlen - sizeof(cmd)))
+               return -EOPNOTSUPP;
+
+       ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
+       if (ret)
+               return ret;
+
+       if (cmd.comp_mask)
+               return -EOPNOTSUPP;
+
+       resp.response_length = required_resp_len;
+       uobj = idr_write_uobj(&ib_uverbs_wq_idr, cmd.wq_handle,
+                             file->ucontext);
+       if (!uobj)
+               return -EINVAL;
+
+       wq = uobj->object;
+       obj = container_of(uobj, struct ib_uwq_object, uevent.uobject);
+       ret = ib_destroy_wq(wq);
+       if (!ret)
+               uobj->live = 0;
+
+       put_uobj_write(uobj);
+       if (ret)
+               return ret;
+
+       idr_remove_uobj(&ib_uverbs_wq_idr, uobj);
+
+       mutex_lock(&file->mutex);
+       list_del(&uobj->list);
+       mutex_unlock(&file->mutex);
+
+       ib_uverbs_release_uevent(file, &obj->uevent);
+       resp.events_reported = obj->uevent.events_reported;
+       put_uobj(uobj);
+
+       ret = ib_copy_to_udata(ucore, &resp, resp.response_length);
+       if (ret)
+               return ret;
+
+       return 0;
+}
+
+int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file,
+                          struct ib_device *ib_dev,
+                          struct ib_udata *ucore,
+                          struct ib_udata *uhw)
+{
+       struct ib_uverbs_ex_modify_wq cmd = {};
+       struct ib_wq *wq;
+       struct ib_wq_attr wq_attr = {};
+       size_t required_cmd_sz;
+       int ret;
+
+       required_cmd_sz = offsetof(typeof(cmd), curr_wq_state) + sizeof(cmd.curr_wq_state);
+       if (ucore->inlen < required_cmd_sz)
+               return -EINVAL;
+
+       if (ucore->inlen > sizeof(cmd) &&
+           !ib_is_udata_cleared(ucore, sizeof(cmd),
+                                ucore->inlen - sizeof(cmd)))
+               return -EOPNOTSUPP;
+
+       ret = ib_copy_from_udata(&cmd, ucore, min(sizeof(cmd), ucore->inlen));
+       if (ret)
+               return ret;
+
+       if (!cmd.attr_mask)
+               return -EINVAL;
+
+       if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE))
+               return -EINVAL;
+
+       wq = idr_read_wq(cmd.wq_handle, file->ucontext);
+       if (!wq)
+               return -EINVAL;
+
+       wq_attr.curr_wq_state = cmd.curr_wq_state;
+       wq_attr.wq_state = cmd.wq_state;
+       ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw);
+       put_wq_read(wq);
+       return ret;
+}
+
 int ib_uverbs_ex_create_flow(struct ib_uverbs_file *file,
                             struct ib_device *ib_dev,
                             struct ib_udata *ucore,

diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 31f422a70623a5df98f94acc90a6f6a2d598c2bc..91cb36f66ea7b6560425b0f3a3c51920570905de 100644 (file)
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -76,6 +76,7 @@ DEFINE_IDR(ib_uverbs_qp_idr);
 DEFINE_IDR(ib_uverbs_srq_idr);
 DEFINE_IDR(ib_uverbs_xrcd_idr);
 DEFINE_IDR(ib_uverbs_rule_idr);
+DEFINE_IDR(ib_uverbs_wq_idr);
 
 static DEFINE_SPINLOCK(map_lock);
 static DECLARE_BITMAP(dev_map, IB_UVERBS_MAX_DEVICES);
@@ -130,6 +131,9 @@ static int (*uverbs_ex_cmd_table[])(struct ib_uverbs_file *file,
        [IB_USER_VERBS_EX_CMD_QUERY_DEVICE]     = ib_uverbs_ex_query_device,
        [IB_USER_VERBS_EX_CMD_CREATE_CQ]        = ib_uverbs_ex_create_cq,
        [IB_USER_VERBS_EX_CMD_CREATE_QP]        = ib_uverbs_ex_create_qp,
+       [IB_USER_VERBS_EX_CMD_CREATE_WQ]        = ib_uverbs_ex_create_wq,
+       [IB_USER_VERBS_EX_CMD_MODIFY_WQ]        = ib_uverbs_ex_modify_wq,
+       [IB_USER_VERBS_EX_CMD_DESTROY_WQ]       = ib_uverbs_ex_destroy_wq,
 };
 
 static void ib_uverbs_add_one(struct ib_device *device);
@@ -265,6 +269,17 @@ static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file,
                kfree(uqp);
        }
 
+       list_for_each_entry_safe(uobj, tmp, &context->wq_list, list) {
+               struct ib_wq *wq = uobj->object;
+               struct ib_uwq_object *uwq =
+                       container_of(uobj, struct ib_uwq_object, uevent.uobject);
+
+               idr_remove_uobj(&ib_uverbs_wq_idr, uobj);
+               ib_destroy_wq(wq);
+               ib_uverbs_release_uevent(file, &uwq->uevent);
+               kfree(uwq);
+       }
+
        list_for_each_entry_safe(uobj, tmp, &context->srq_list, list) {
                struct ib_srq *srq = uobj->object;
                struct ib_uevent_object *uevent =
@@ -568,6 +583,16 @@ void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr)
                                &uobj->events_reported);
 }
 
+void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr)
+{
+       struct ib_uevent_object *uobj = container_of(event->element.wq->uobject,
+                                                 struct ib_uevent_object, uobject);
+
+       ib_uverbs_async_handler(context_ptr, uobj->uobject.user_handle,
+                               event->event, &uobj->event_list,
+                               &uobj->events_reported);
+}
+
 void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr)
 {
        struct ib_uevent_object *uobj;

diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index f2d954ac42f68218e9130718d0b42c783df3d46c..0c1956a985735c2f3ef8def107b0f93e238e92d6 100644 (file)
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -562,6 +562,7 @@ enum ib_event_type {
        IB_EVENT_QP_LAST_WQE_REACHED,
        IB_EVENT_CLIENT_REREGISTER,
        IB_EVENT_GID_CHANGE,
+       IB_EVENT_WQ_FATAL,
 };
 
 const char *__attribute_const__ ib_event_msg(enum ib_event_type event);
@@ -572,6 +573,7 @@ struct ib_event {
                struct ib_cq    *cq;
                struct ib_qp    *qp;
                struct ib_srq   *srq;
+               struct ib_wq    *wq;
                u8              port_num;
        } element;
        enum ib_event_type      event;
@@ -1323,6 +1325,7 @@ struct ib_ucontext {
        struct list_head        ah_list;
        struct list_head        xrcd_list;
        struct list_head        rule_list;
+       struct list_head        wq_list;
        int                     closing;
 
        struct pid             *tgid;

diff --git a/include/uapi/rdma/ib_user_verbs.h b/include/uapi/rdma/ib_user_verbs.h
index b6543d73d20a7fd03a0e77a2615679bad978d74a..c9470e511e03b36f40a3733199b3215e4cb24141 100644 (file)
--- a/include/uapi/rdma/ib_user_verbs.h
+++ b/include/uapi/rdma/ib_user_verbs.h
@@ -95,6 +95,9 @@ enum {
        IB_USER_VERBS_EX_CMD_CREATE_QP = IB_USER_VERBS_CMD_CREATE_QP,
        IB_USER_VERBS_EX_CMD_CREATE_FLOW = IB_USER_VERBS_CMD_THRESHOLD,
        IB_USER_VERBS_EX_CMD_DESTROY_FLOW,
+       IB_USER_VERBS_EX_CMD_CREATE_WQ,
+       IB_USER_VERBS_EX_CMD_MODIFY_WQ,
+       IB_USER_VERBS_EX_CMD_DESTROY_WQ,
 };
 
 /*
@@ -946,4 +949,42 @@ struct ib_uverbs_destroy_srq_resp {
        __u32 events_reported;
 };
 
+struct ib_uverbs_ex_create_wq  {
+       __u32 comp_mask;
+       __u32 wq_type;
+       __u64 user_handle;
+       __u32 pd_handle;
+       __u32 cq_handle;
+       __u32 max_wr;
+       __u32 max_sge;
+};
+
+struct ib_uverbs_ex_create_wq_resp {
+       __u32 comp_mask;
+       __u32 response_length;
+       __u32 wq_handle;
+       __u32 max_wr;
+       __u32 max_sge;
+       __u32 wqn;
+};
+
+struct ib_uverbs_ex_destroy_wq  {
+       __u32 comp_mask;
+       __u32 wq_handle;
+};
+
+struct ib_uverbs_ex_destroy_wq_resp {
+       __u32 comp_mask;
+       __u32 response_length;
+       __u32 events_reported;
+       __u32 reserved;
+};
+
+struct ib_uverbs_ex_modify_wq  {
+       __u32 attr_mask;
+       __u32 wq_handle;
+       __u32 wq_state;
+       __u32 curr_wq_state;
+};
+
 #endif /* IB_USER_VERBS_H */