ixgbe: Reconfigure SR-IOV Init
authorGreg Rose <gregory.v.rose@intel.com>
Wed, 24 Aug 2011 02:37:55 +0000 (02:37 +0000)
committerJeff Kirsher <jeffrey.t.kirsher@intel.com>
Fri, 23 Sep 2011 16:05:49 +0000 (09:05 -0700)
Use the PCI device flag indicating if a VF is assigned to a guest VM
to guard against destroying VFs upon driver removal.  Implement
additional feature to detect if VFs already exist when the driver
is loaded and if so configure them and set the driver state to
SR-IOV enabled.

Signed-off-by: Greg Rose <gregory.v.rose@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
drivers/net/ethernet/intel/ixgbe/ixgbe.h
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
drivers/net/ethernet/intel/ixgbe/ixgbe_type.h

index 2c9fdf8ef5f18ad81f48dc2e7ae6fe8643f08244..b43b2cde49d2815c4e7bcaa4fde384efd84ddedc 100644 (file)
@@ -128,6 +128,7 @@ struct vf_data_storage {
        u16 pf_vlan; /* When set, guest VLAN config not allowed. */
        u16 pf_qos;
        u16 tx_rate;
+       struct pci_dev *vfdev;
 };
 
 struct vf_macvlans {
index 49e82de136a7c51dd8300758b26610de62f46c5c..8b86c41f2967c04796747cc89bc8c1652343bad2 100644 (file)
@@ -134,42 +134,6 @@ MODULE_VERSION(DRV_VERSION);
 
 #define DEFAULT_DEBUG_LEVEL_SHIFT 3
 
-static inline void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
-{
-       struct ixgbe_hw *hw = &adapter->hw;
-       u32 gcr;
-       u32 gpie;
-       u32 vmdctl;
-
-#ifdef CONFIG_PCI_IOV
-       /* disable iov and allow time for transactions to clear */
-       pci_disable_sriov(adapter->pdev);
-#endif
-
-       /* turn off device IOV mode */
-       gcr = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
-       gcr &= ~(IXGBE_GCR_EXT_SRIOV);
-       IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr);
-       gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
-       gpie &= ~IXGBE_GPIE_VTMODE_MASK;
-       IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
-
-       /* set default pool back to 0 */
-       vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
-       vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
-       IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
-       IXGBE_WRITE_FLUSH(hw);
-
-       /* take a breather then clean up driver data */
-       msleep(100);
-
-       kfree(adapter->vfinfo);
-       adapter->vfinfo = NULL;
-
-       adapter->num_vfs = 0;
-       adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
-}
-
 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
 {
        if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
@@ -7064,11 +7028,8 @@ static void __devinit ixgbe_probe_vf(struct ixgbe_adapter *adapter,
 {
 #ifdef CONFIG_PCI_IOV
        struct ixgbe_hw *hw = &adapter->hw;
-       int err;
-       int num_vf_macvlans, i;
-       struct vf_macvlans *mv_list;
 
-       if (hw->mac.type == ixgbe_mac_82598EB || !max_vfs)
+       if (hw->mac.type == ixgbe_mac_82598EB)
                return;
 
        /* The 82599 supports up to 64 VFs per physical function
@@ -7077,60 +7038,7 @@ static void __devinit ixgbe_probe_vf(struct ixgbe_adapter *adapter,
         * physical function
         */
        adapter->num_vfs = (max_vfs > 63) ? 63 : max_vfs;
-       adapter->flags |= IXGBE_FLAG_SRIOV_ENABLED;
-       err = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
-       if (err) {
-               e_err(probe, "Failed to enable PCI sriov: %d\n", err);
-               goto err_novfs;
-       }
-
-       num_vf_macvlans = hw->mac.num_rar_entries -
-               (IXGBE_MAX_PF_MACVLANS + 1 + adapter->num_vfs);
-
-       adapter->mv_list = mv_list = kcalloc(num_vf_macvlans,
-                                            sizeof(struct vf_macvlans),
-                                            GFP_KERNEL);
-       if (mv_list) {
-               /* Initialize list of VF macvlans */
-               INIT_LIST_HEAD(&adapter->vf_mvs.l);
-               for (i = 0; i < num_vf_macvlans; i++) {
-                       mv_list->vf = -1;
-                       mv_list->free = true;
-                       mv_list->rar_entry = hw->mac.num_rar_entries -
-                               (i + adapter->num_vfs + 1);
-                       list_add(&mv_list->l, &adapter->vf_mvs.l);
-                       mv_list++;
-               }
-       }
-
-       /* If call to enable VFs succeeded then allocate memory
-        * for per VF control structures.
-        */
-       adapter->vfinfo =
-               kcalloc(adapter->num_vfs,
-                       sizeof(struct vf_data_storage), GFP_KERNEL);
-       if (adapter->vfinfo) {
-               /* Now that we're sure SR-IOV is enabled
-                * and memory allocated set up the mailbox parameters
-                */
-               ixgbe_init_mbx_params_pf(hw);
-               memcpy(&hw->mbx.ops, ii->mbx_ops,
-                      sizeof(hw->mbx.ops));
-
-               /* Disable RSC when in SR-IOV mode */
-               adapter->flags2 &= ~(IXGBE_FLAG2_RSC_CAPABLE |
-                                    IXGBE_FLAG2_RSC_ENABLED);
-               return;
-       }
-
-       /* Oh oh */
-       e_err(probe, "Unable to allocate memory for VF Data Storage - "
-             "SRIOV disabled\n");
-       pci_disable_sriov(adapter->pdev);
-
-err_novfs:
-       adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
-       adapter->num_vfs = 0;
+       ixgbe_enable_sriov(adapter, ii);
 #endif /* CONFIG_PCI_IOV */
 }
 
@@ -7580,8 +7488,13 @@ static void __devexit ixgbe_remove(struct pci_dev *pdev)
        if (netdev->reg_state == NETREG_REGISTERED)
                unregister_netdev(netdev);
 
-       if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
-               ixgbe_disable_sriov(adapter);
+       if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
+               if (!(ixgbe_check_vf_assignment(adapter)))
+                       ixgbe_disable_sriov(adapter);
+               else
+                       e_dev_warn("Unloading driver while VFs are assigned "
+                                  "- VFs will not be deallocated\n");
+       }
 
        ixgbe_clear_interrupt_scheme(adapter);
 
index d99d01e21326dff47e67c60c8f3ead92c7ed1e82..468ddd0873dab0eb679367d0f4d92dba17a2e21d 100644 (file)
 #endif
 
 #include "ixgbe.h"
-
+#include "ixgbe_type.h"
 #include "ixgbe_sriov.h"
 
+#ifdef CONFIG_PCI_IOV
+static int ixgbe_find_enabled_vfs(struct ixgbe_adapter *adapter)
+{
+       struct pci_dev *pdev = adapter->pdev;
+       struct pci_dev *pvfdev;
+       u16 vf_devfn = 0;
+       int device_id;
+       int vfs_found = 0;
+
+       switch (adapter->hw.mac.type) {
+       case ixgbe_mac_82599EB:
+               device_id = IXGBE_DEV_ID_82599_VF;
+               break;
+       case ixgbe_mac_X540:
+               device_id = IXGBE_DEV_ID_X540_VF;
+               break;
+       default:
+               device_id = 0;
+               break;
+       }
+
+       vf_devfn = pdev->devfn + 0x80;
+       pvfdev = pci_get_device(IXGBE_INTEL_VENDOR_ID, device_id, NULL);
+       while (pvfdev) {
+               if (pvfdev->devfn == vf_devfn)
+                       vfs_found++;
+               vf_devfn += 2;
+               pvfdev = pci_get_device(IXGBE_INTEL_VENDOR_ID,
+                                       device_id, pvfdev);
+       }
+
+       return vfs_found;
+}
+
+void ixgbe_enable_sriov(struct ixgbe_adapter *adapter,
+                        const struct ixgbe_info *ii)
+{
+       struct ixgbe_hw *hw = &adapter->hw;
+       int err = 0;
+       int num_vf_macvlans, i;
+       struct vf_macvlans *mv_list;
+       int pre_existing_vfs = 0;
+
+       pre_existing_vfs = ixgbe_find_enabled_vfs(adapter);
+       if (!pre_existing_vfs && !adapter->num_vfs)
+               return;
+
+       /* If there are pre-existing VFs then we have to force
+        * use of that many because they were not deleted the last
+        * time someone removed the PF driver.  That would have
+        * been because they were allocated to guest VMs and can't
+        * be removed.  Go ahead and just re-enable the old amount.
+        * If the user wants to change the number of VFs they can
+        * use ethtool while making sure no VFs are allocated to
+        * guest VMs... i.e. the right way.
+        */
+       if (pre_existing_vfs) {
+               adapter->num_vfs = pre_existing_vfs;
+               dev_warn(&adapter->pdev->dev, "Virtual Functions already "
+                        "enabled for this device - Please reload all "
+                        "VF drivers to avoid spoofed packet errors\n");
+       } else {
+               err = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
+       }
+       if (err) {
+               e_err(probe, "Failed to enable PCI sriov: %d\n", err);
+               goto err_novfs;
+       }
+       adapter->flags |= IXGBE_FLAG_SRIOV_ENABLED;
+
+       e_info(probe, "SR-IOV enabled with %d VFs\n", adapter->num_vfs);
+
+       num_vf_macvlans = hw->mac.num_rar_entries -
+       (IXGBE_MAX_PF_MACVLANS + 1 + adapter->num_vfs);
+
+       adapter->mv_list = mv_list = kcalloc(num_vf_macvlans,
+                                            sizeof(struct vf_macvlans),
+                                            GFP_KERNEL);
+       if (mv_list) {
+               /* Initialize list of VF macvlans */
+               INIT_LIST_HEAD(&adapter->vf_mvs.l);
+               for (i = 0; i < num_vf_macvlans; i++) {
+                       mv_list->vf = -1;
+                       mv_list->free = true;
+                       mv_list->rar_entry = hw->mac.num_rar_entries -
+                               (i + adapter->num_vfs + 1);
+                       list_add(&mv_list->l, &adapter->vf_mvs.l);
+                       mv_list++;
+               }
+       }
+
+       /* If call to enable VFs succeeded then allocate memory
+        * for per VF control structures.
+        */
+       adapter->vfinfo =
+               kcalloc(adapter->num_vfs,
+                       sizeof(struct vf_data_storage), GFP_KERNEL);
+       if (adapter->vfinfo) {
+               /* Now that we're sure SR-IOV is enabled
+                * and memory allocated set up the mailbox parameters
+                */
+               ixgbe_init_mbx_params_pf(hw);
+               memcpy(&hw->mbx.ops, ii->mbx_ops,
+                      sizeof(hw->mbx.ops));
+
+               /* Disable RSC when in SR-IOV mode */
+               adapter->flags2 &= ~(IXGBE_FLAG2_RSC_CAPABLE |
+                                    IXGBE_FLAG2_RSC_ENABLED);
+               return;
+       }
+
+       /* Oh oh */
+       e_err(probe, "Unable to allocate memory for VF Data Storage - "
+             "SRIOV disabled\n");
+       pci_disable_sriov(adapter->pdev);
+
+err_novfs:
+       adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
+       adapter->num_vfs = 0;
+}
+#endif /* #ifdef CONFIG_PCI_IOV */
+
+void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
+{
+       struct ixgbe_hw *hw = &adapter->hw;
+       u32 gcr;
+       u32 gpie;
+       u32 vmdctl;
+       int i;
+
+#ifdef CONFIG_PCI_IOV
+       /* disable iov and allow time for transactions to clear */
+       pci_disable_sriov(adapter->pdev);
+#endif
+
+       /* turn off device IOV mode */
+       gcr = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
+       gcr &= ~(IXGBE_GCR_EXT_SRIOV);
+       IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr);
+       gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
+       gpie &= ~IXGBE_GPIE_VTMODE_MASK;
+       IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
+
+       /* set default pool back to 0 */
+       vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
+       vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
+       IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
+       IXGBE_WRITE_FLUSH(hw);
+
+       /* take a breather then clean up driver data */
+       msleep(100);
+
+       /* Release reference to VF devices */
+       for (i = 0; i < adapter->num_vfs; i++) {
+               if (adapter->vfinfo[i].vfdev)
+                       pci_dev_put(adapter->vfinfo[i].vfdev);
+       }
+       kfree(adapter->vfinfo);
+       kfree(adapter->mv_list);
+       adapter->vfinfo = NULL;
+
+       adapter->num_vfs = 0;
+       adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
+}
+
 static int ixgbe_set_vf_multicasts(struct ixgbe_adapter *adapter,
                                   int entries, u16 *hash_list, u32 vf)
 {
@@ -273,11 +438,26 @@ static int ixgbe_set_vf_macvlan(struct ixgbe_adapter *adapter,
        return 0;
 }
 
+int ixgbe_check_vf_assignment(struct ixgbe_adapter *adapter)
+{
+       int i;
+       for (i = 0; i < adapter->num_vfs; i++) {
+               if (adapter->vfinfo[i].vfdev->dev_flags &
+                               PCI_DEV_FLAGS_ASSIGNED)
+                       return true;
+       }
+       return false;
+}
+
 int ixgbe_vf_configuration(struct pci_dev *pdev, unsigned int event_mask)
 {
        unsigned char vf_mac_addr[6];
        struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
        unsigned int vfn = (event_mask & 0x3f);
+       struct pci_dev *pvfdev;
+       unsigned int device_id;
+       u16 thisvf_devfn = (pdev->devfn + 0x80 + (vfn << 1)) |
+                               (pdev->devfn & 1);
 
        bool enable = ((event_mask & 0x10000000U) != 0);
 
@@ -290,6 +470,31 @@ int ixgbe_vf_configuration(struct pci_dev *pdev, unsigned int event_mask)
                 * for it later.
                 */
                memcpy(adapter->vfinfo[vfn].vf_mac_addresses, vf_mac_addr, 6);
+
+               switch (adapter->hw.mac.type) {
+               case ixgbe_mac_82599EB:
+                       device_id = IXGBE_DEV_ID_82599_VF;
+                       break;
+               case ixgbe_mac_X540:
+                       device_id = IXGBE_DEV_ID_X540_VF;
+                       break;
+               default:
+                       device_id = 0;
+                       break;
+               }
+
+               pvfdev = pci_get_device(IXGBE_INTEL_VENDOR_ID, device_id, NULL);
+               while (pvfdev) {
+                       if (pvfdev->devfn == thisvf_devfn)
+                               break;
+                       pvfdev = pci_get_device(IXGBE_INTEL_VENDOR_ID,
+                                               device_id, pvfdev);
+               }
+               if (pvfdev)
+                       adapter->vfinfo[vfn].vfdev = pvfdev;
+               else
+                       e_err(drv, "Couldn't find pci dev ptr for VF %4.4x\n",
+                             thisvf_devfn);
        }
 
        return 0;
index 34175564bb78b75e034bceda257f7a3646994b23..278184757b690dfe6e36965889ecdcdbfcb54f4e 100644 (file)
@@ -41,6 +41,11 @@ int ixgbe_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
 int ixgbe_ndo_get_vf_config(struct net_device *netdev,
                            int vf, struct ifla_vf_info *ivi);
 void ixgbe_check_vf_rate_limit(struct ixgbe_adapter *adapter);
+void ixgbe_disable_sriov(struct ixgbe_adapter *adapter);
+void ixgbe_enable_sriov(struct ixgbe_adapter *adapter,
+                       const struct ixgbe_info *ii);
+int ixgbe_check_vf_assignment(struct ixgbe_adapter *adapter);
+
 
 #endif /* _IXGBE_SRIOV_H_ */
 
index a9f8839bffb9c94e855584850fcb4b5f3a813b80..56a7adfa95d73f5f69d326bb92aa90b81012c0b5 100644 (file)
 #define IXGBE_DEV_ID_82599_LS            0x154F
 #define IXGBE_DEV_ID_X540T               0x1528
 
+/* VF Device IDs */
+#define IXGBE_DEV_ID_82599_VF           0x10ED
+#define IXGBE_DEV_ID_X540_VF            0x1515
+
 /* General Registers */
 #define IXGBE_CTRL      0x00000
 #define IXGBE_STATUS    0x00008