xdp: support simultaneous driver and hw XDP attachment

Split the query of HW-attached program from the software one.
Introduce new .ndo_bpf command to query HW-attached program.
This will allow drivers to install different programs in HW
and SW at the same time.  Netlink can now also carry multiple
programs on dump (in which case mode will be set to
XDP_ATTACHED_MULTI and user has to check per-attachment point
attributes, IFLA_XDP_PROG_ID will not be present).  We reuse
IFLA_XDP_PROG_ID skb space for second mode, so rtnl_xdp_size()
doesn't need to be updated.

Note that the installation side is still not there, since all
drivers currently reject installing more than one program at
the time.

Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Reviewed-by: Quentin Monnet <quentin.monnet@netronome.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index 4bb589dbffbcce07e0e0afb58577e1db5fd3d9da..bb1e72e8dbc26f921a25d5dfa6e9f4229ec1cc57 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -3453,6 +3453,12 @@ static int nfp_net_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
        case XDP_SETUP_PROG_HW:
                return nfp_net_xdp_setup(nn, xdp);
        case XDP_QUERY_PROG:
+               if (nn->dp.bpf_offload_xdp)
+                       return 0;
+               return xdp_attachment_query(&nn->xdp, xdp);
+       case XDP_QUERY_PROG_HW:
+               if (!nn->dp.bpf_offload_xdp)
+                       return 0;
                return xdp_attachment_query(&nn->xdp, xdp);
        default:
                return nfp_app_bpf(nn->app, nn, xdp);

diff --git a/drivers/net/netdevsim/bpf.c b/drivers/net/netdevsim/bpf.c
index c485d97b5df44834c9f2ddd49ea41d04939bc99d..5544c9b51173e48759b99bee584eb81a888e8097 100644 (file)
--- a/drivers/net/netdevsim/bpf.c
+++ b/drivers/net/netdevsim/bpf.c
@@ -561,6 +561,12 @@ int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf)
                nsim_bpf_destroy_prog(bpf->offload.prog);
                return 0;
        case XDP_QUERY_PROG:
+               if (ns->xdp_prog_mode != XDP_ATTACHED_DRV)
+                       return 0;
+               return xdp_attachment_query(&ns->xdp, bpf);
+       case XDP_QUERY_PROG_HW:
+               if (ns->xdp_prog_mode != XDP_ATTACHED_HW)
+                       return 0;
                return xdp_attachment_query(&ns->xdp, bpf);
        case XDP_SETUP_PROG:
                err = nsim_setup_prog_checks(ns, bpf);

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 69a664789b332496684d48e158c4582295cee9d1..2422c0e88f5c9da4cd49fd4a9a351591cab86c9f 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -820,6 +820,7 @@ enum bpf_netdev_command {
        XDP_SETUP_PROG,
        XDP_SETUP_PROG_HW,
        XDP_QUERY_PROG,
+       XDP_QUERY_PROG_HW,
        /* BPF program for offload callbacks, invoked at program load time. */
        BPF_OFFLOAD_VERIFIER_PREP,
        BPF_OFFLOAD_TRANSLATE,
@@ -843,7 +844,7 @@ struct netdev_bpf {
                        struct bpf_prog *prog;
                        struct netlink_ext_ack *extack;
                };
-               /* XDP_QUERY_PROG */
+               /* XDP_QUERY_PROG, XDP_QUERY_PROG_HW */
                struct {
                        u32 prog_id;
                        /* flags with which program was installed */
@@ -3533,8 +3534,8 @@ struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
 typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf);
 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
                      int fd, u32 flags);
-void __dev_xdp_query(struct net_device *dev, bpf_op_t xdp_op,
-                    struct netdev_bpf *xdp);
+u32 __dev_xdp_query(struct net_device *dev, bpf_op_t xdp_op,
+                   enum bpf_netdev_command cmd);
 
 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);

diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index bc86c2b105ece0e82c37bdace42aaf469c37ba0c..8759cfb8aa2eebabfb247aace11a50a304f07eb1 100644 (file)
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -920,6 +920,7 @@ enum {
        XDP_ATTACHED_DRV,
        XDP_ATTACHED_SKB,
        XDP_ATTACHED_HW,
+       XDP_ATTACHED_MULTI,
 };
 
 enum {

diff --git a/net/core/dev.c b/net/core/dev.c
index 9fa3b3705a8e391b1ac063c92a047c73d6caf193..993cdc3cd086149affebb66ceb4f2a1ac963b177 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -7582,21 +7582,19 @@ int dev_change_proto_down(struct net_device *dev, bool proto_down)
 }
 EXPORT_SYMBOL(dev_change_proto_down);
 
-void __dev_xdp_query(struct net_device *dev, bpf_op_t bpf_op,
-                    struct netdev_bpf *xdp)
+u32 __dev_xdp_query(struct net_device *dev, bpf_op_t bpf_op,
+                   enum bpf_netdev_command cmd)
 {
-       memset(xdp, 0, sizeof(*xdp));
-       xdp->command = XDP_QUERY_PROG;
+       struct netdev_bpf xdp;
 
-       /* Query must always succeed. */
-       WARN_ON(bpf_op(dev, xdp) < 0);
-}
+       if (!bpf_op)
+               return 0;
 
-static bool __dev_xdp_attached(struct net_device *dev, bpf_op_t bpf_op)
-{
-       struct netdev_bpf xdp;
+       memset(&xdp, 0, sizeof(xdp));
+       xdp.command = cmd;
 
-       __dev_xdp_query(dev, bpf_op, &xdp);
+       /* Query must always succeed. */
+       WARN_ON(bpf_op(dev, &xdp) < 0 && cmd == XDP_QUERY_PROG);
 
        return xdp.prog_id;
 }
@@ -7632,12 +7630,19 @@ static void dev_xdp_uninstall(struct net_device *dev)
        if (!ndo_bpf)
                return;
 
-       __dev_xdp_query(dev, ndo_bpf, &xdp);
-       if (!xdp.prog_id)
-               return;
+       memset(&xdp, 0, sizeof(xdp));
+       xdp.command = XDP_QUERY_PROG;
+       WARN_ON(ndo_bpf(dev, &xdp));
+       if (xdp.prog_id)
+               WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags,
+                                       NULL));
 
-       /* Program removal should always succeed */
-       WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags, NULL));
+       /* Remove HW offload */
+       memset(&xdp, 0, sizeof(xdp));
+       xdp.command = XDP_QUERY_PROG_HW;
+       if (!ndo_bpf(dev, &xdp) && xdp.prog_id)
+               WARN_ON(dev_xdp_install(dev, ndo_bpf, NULL, xdp.prog_flags,
+                                       NULL));
 }
 
 /**
@@ -7653,12 +7658,15 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
                      int fd, u32 flags)
 {
        const struct net_device_ops *ops = dev->netdev_ops;
+       enum bpf_netdev_command query;
        struct bpf_prog *prog = NULL;
        bpf_op_t bpf_op, bpf_chk;
        int err;
 
        ASSERT_RTNL();
 
+       query = flags & XDP_FLAGS_HW_MODE ? XDP_QUERY_PROG_HW : XDP_QUERY_PROG;
+
        bpf_op = bpf_chk = ops->ndo_bpf;
        if (!bpf_op && (flags & (XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE)))
                return -EOPNOTSUPP;
@@ -7668,10 +7676,11 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
                bpf_chk = generic_xdp_install;
 
        if (fd >= 0) {
-               if (bpf_chk && __dev_xdp_attached(dev, bpf_chk))
+               if (__dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG) ||
+                   __dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG_HW))
                        return -EEXIST;
                if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) &&
-                   __dev_xdp_attached(dev, bpf_op))
+                   __dev_xdp_query(dev, bpf_op, query))
                        return -EBUSY;
 
                prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,

diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 02ebc056a68843daa6bf93b1d9c6270142687b7f..c9929ef175390a3022bd1209d3e1f6ab740fbe6e 100644 (file)
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -964,7 +964,7 @@ static size_t rtnl_xdp_size(void)
 {
        size_t xdp_size = nla_total_size(0) +   /* nest IFLA_XDP */
                          nla_total_size(1) +   /* XDP_ATTACHED */
-                         nla_total_size(4) +   /* XDP_PROG_ID */
+                         nla_total_size(4) +   /* XDP_PROG_ID (or 1st mode) */
                          nla_total_size(4);    /* XDP_<mode>_PROG_ID */
 
        return xdp_size;
@@ -1354,37 +1354,57 @@ static int rtnl_fill_link_ifmap(struct sk_buff *skb, struct net_device *dev)
        return 0;
 }
 
-static u8 rtnl_xdp_attached_mode(struct net_device *dev, u32 *prog_id)
+static u32 rtnl_xdp_prog_skb(struct net_device *dev)
 {
-       const struct net_device_ops *ops = dev->netdev_ops;
        const struct bpf_prog *generic_xdp_prog;
-       struct netdev_bpf xdp;
 
        ASSERT_RTNL();
 
-       *prog_id = 0;
        generic_xdp_prog = rtnl_dereference(dev->xdp_prog);
-       if (generic_xdp_prog) {
-               *prog_id = generic_xdp_prog->aux->id;
-               return XDP_ATTACHED_SKB;
-       }
-       if (!ops->ndo_bpf)
-               return XDP_ATTACHED_NONE;
+       if (!generic_xdp_prog)
+               return 0;
+       return generic_xdp_prog->aux->id;
+}
+
+static u32 rtnl_xdp_prog_drv(struct net_device *dev)
+{
+       return __dev_xdp_query(dev, dev->netdev_ops->ndo_bpf, XDP_QUERY_PROG);
+}
+
+static u32 rtnl_xdp_prog_hw(struct net_device *dev)
+{
+       return __dev_xdp_query(dev, dev->netdev_ops->ndo_bpf,
+                              XDP_QUERY_PROG_HW);
+}
+
+static int rtnl_xdp_report_one(struct sk_buff *skb, struct net_device *dev,
+                              u32 *prog_id, u8 *mode, u8 tgt_mode, u32 attr,
+                              u32 (*get_prog_id)(struct net_device *dev))
+{
+       u32 curr_id;
+       int err;
+
+       curr_id = get_prog_id(dev);
+       if (!curr_id)
+               return 0;
+
+       *prog_id = curr_id;
+       err = nla_put_u32(skb, attr, curr_id);
+       if (err)
+               return err;
 
-       __dev_xdp_query(dev, ops->ndo_bpf, &xdp);
-       if (!xdp.prog_id)
-               return XDP_ATTACHED_NONE;
+       if (*mode != XDP_ATTACHED_NONE)
+               *mode = XDP_ATTACHED_MULTI;
+       else
+               *mode = tgt_mode;
 
-       *prog_id = xdp.prog_id;
-       if (xdp.prog_flags & XDP_FLAGS_HW_MODE)
-               return XDP_ATTACHED_HW;
-       return XDP_ATTACHED_DRV;
+       return 0;
 }
 
 static int rtnl_xdp_fill(struct sk_buff *skb, struct net_device *dev)
 {
-       u32 prog_attr, prog_id;
        struct nlattr *xdp;
+       u32 prog_id;
        int err;
        u8 mode;
 
@@ -1392,35 +1412,26 @@ static int rtnl_xdp_fill(struct sk_buff *skb, struct net_device *dev)
        if (!xdp)
                return -EMSGSIZE;
 
-       mode = rtnl_xdp_attached_mode(dev, &prog_id);
+       prog_id = 0;
+       mode = XDP_ATTACHED_NONE;
+       if (rtnl_xdp_report_one(skb, dev, &prog_id, &mode, XDP_ATTACHED_SKB,
+                               IFLA_XDP_SKB_PROG_ID, rtnl_xdp_prog_skb))
+               goto err_cancel;
+       if (rtnl_xdp_report_one(skb, dev, &prog_id, &mode, XDP_ATTACHED_DRV,
+                               IFLA_XDP_DRV_PROG_ID, rtnl_xdp_prog_drv))
+               goto err_cancel;
+       if (rtnl_xdp_report_one(skb, dev, &prog_id, &mode, XDP_ATTACHED_HW,
+                               IFLA_XDP_HW_PROG_ID, rtnl_xdp_prog_hw))
+               goto err_cancel;
+
        err = nla_put_u8(skb, IFLA_XDP_ATTACHED, mode);
        if (err)
                goto err_cancel;
 
-       if (prog_id) {
+       if (prog_id && mode != XDP_ATTACHED_MULTI) {
                err = nla_put_u32(skb, IFLA_XDP_PROG_ID, prog_id);
                if (err)
                        goto err_cancel;
-
-               switch (mode) {
-               case XDP_ATTACHED_DRV:
-                       prog_attr = IFLA_XDP_DRV_PROG_ID;
-                       break;
-               case XDP_ATTACHED_SKB:
-                       prog_attr = IFLA_XDP_SKB_PROG_ID;
-                       break;
-               case XDP_ATTACHED_HW:
-                       prog_attr = IFLA_XDP_HW_PROG_ID;
-                       break;
-               case XDP_ATTACHED_NONE:
-               default:
-                       err = -EINVAL;
-                       goto err_cancel;
-               }
-
-               err = nla_put_u32(skb, prog_attr, prog_id);
-               if (err)
-                       goto err_cancel;
        }
 
        nla_nest_end(skb, xdp);