net/mlx5: Handle SW reset of FW in error flow

New mlx5 adapters allow the driver to reset the FW in the event of an
error, this action called "SW Reset". When an SW reset is issued on any
PF all PFs enter reset state which is a recoverable condition. The
existing recovery flow was designed to allow the recovery of a VF after
a PF driver reload. This patch adds the sw reset to the NIC states
as a preparation for sw reset handling.

When a software reset is issued the following occurs:
1. The NIC interface mode is set to 7 while the reset is in progress.
2. Once the reset completes the NIC interface mode is set to 1.

Signed-off-by: Feras Daoud <ferasda@mellanox.com>
Signed-off-by: Moshe Shemesh <moshe@mellanox.com>
Signed-off-by: Daniel Jurgens <danielj@mellanox.com>
Reviewed-by: Alex Vesker <valex@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c b/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
index 4382ef85488c5b6e936424dc902fd590d8030294..840ec945ccbab466863fb6f43a29bbfd23d08c4a 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
@@ -64,7 +64,7 @@ static int mlx5e_test_health_info(struct mlx5e_priv *priv)
 {
        struct mlx5_core_health *health = &priv->mdev->priv.health;
 
-       return health->sick ? 1 : 0;
+       return health->fatal_error ? 1 : 0;
 }
 
 static int mlx5e_test_link_state(struct mlx5e_priv *priv)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
index a2656f4008d9162b87ff2fc1f5639fcf8d3d4dda..737e6d55077598565794bf8247f0af2f21f8e9f3 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
@@ -62,12 +62,18 @@ enum {
 
 enum {
        MLX5_DROP_NEW_HEALTH_WORK,
-       MLX5_DROP_NEW_RECOVERY_WORK,
+};
+
+enum  {
+       MLX5_SENSOR_NO_ERR              = 0,
+       MLX5_SENSOR_PCI_COMM_ERR        = 1,
+       MLX5_SENSOR_NIC_DISABLED        = 2,
+       MLX5_SENSOR_NIC_SW_RESET        = 3,
 };
 
 u8 mlx5_get_nic_state(struct mlx5_core_dev *dev)
 {
-       return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3;
+       return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 7;
 }
 
 void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state)
@@ -80,18 +86,25 @@ void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state)
                    &dev->iseg->cmdq_addr_l_sz);
 }
 
-static int in_fatal(struct mlx5_core_dev *dev)
+static bool sensor_pci_not_working(struct mlx5_core_dev *dev)
 {
        struct mlx5_core_health *health = &dev->priv.health;
        struct health_buffer __iomem *h = health->health;
 
-       if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
-               return 1;
+       /* Offline PCI reads return 0xffffffff */
+       return (ioread32be(&h->fw_ver) == 0xffffffff);
+}
 
-       if (ioread32be(&h->fw_ver) == 0xffffffff)
-               return 1;
+static u32 check_fatal_sensors(struct mlx5_core_dev *dev)
+{
+       if (sensor_pci_not_working(dev))
+               return MLX5_SENSOR_PCI_COMM_ERR;
+       if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
+               return MLX5_SENSOR_NIC_DISABLED;
+       if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_SW_RESET)
+               return MLX5_SENSOR_NIC_SW_RESET;
 
-       return 0;
+       return MLX5_SENSOR_NO_ERR;
 }
 
 void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
@@ -101,7 +114,8 @@ void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
                goto unlock;
 
        mlx5_core_err(dev, "start\n");
-       if (pci_channel_offline(dev->pdev) || in_fatal(dev) || force) {
+       if (pci_channel_offline(dev->pdev) ||
+           dev->priv.health.fatal_error != MLX5_SENSOR_NO_ERR || force) {
                dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
                mlx5_cmd_flush(dev);
        }
@@ -137,38 +151,14 @@ static void mlx5_handle_bad_state(struct mlx5_core_dev *dev)
        mlx5_disable_device(dev);
 }
 
-static void health_recover(struct work_struct *work)
-{
-       struct mlx5_core_health *health;
-       struct delayed_work *dwork;
-       struct mlx5_core_dev *dev;
-       struct mlx5_priv *priv;
-       u8 nic_state;
-
-       dwork = container_of(work, struct delayed_work, work);
-       health = container_of(dwork, struct mlx5_core_health, recover_work);
-       priv = container_of(health, struct mlx5_priv, health);
-       dev = container_of(priv, struct mlx5_core_dev, priv);
-
-       nic_state = mlx5_get_nic_state(dev);
-       if (nic_state == MLX5_NIC_IFC_INVALID) {
-               mlx5_core_err(dev, "health recovery flow aborted since the nic state is invalid\n");
-               return;
-       }
-
-       mlx5_core_err(dev, "starting health recovery flow\n");
-       mlx5_recover_device(dev);
-}
-
 /* How much time to wait until health resetting the driver (in msecs) */
-#define MLX5_RECOVERY_DELAY_MSECS 60000
+#define MLX5_RECOVERY_WAIT_MSECS 60000
 static void health_care(struct work_struct *work)
 {
-       unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS);
        struct mlx5_core_health *health;
        struct mlx5_core_dev *dev;
        struct mlx5_priv *priv;
-       unsigned long flags;
+       unsigned long end;
 
        health = container_of(work, struct mlx5_core_health, work);
        priv = container_of(health, struct mlx5_priv, health);
@@ -176,13 +166,18 @@ static void health_care(struct work_struct *work)
        mlx5_core_warn(dev, "handling bad device here\n");
        mlx5_handle_bad_state(dev);
 
-       spin_lock_irqsave(&health->wq_lock, flags);
-       if (!test_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags))
-               schedule_delayed_work(&health->recover_work, recover_delay);
-       else
-               mlx5_core_err(dev,
-                             "new health works are not permitted at this stage\n");
-       spin_unlock_irqrestore(&health->wq_lock, flags);
+       end = jiffies + msecs_to_jiffies(MLX5_RECOVERY_WAIT_MSECS);
+       while (sensor_pci_not_working(dev)) {
+               if (time_after(jiffies, end)) {
+                       mlx5_core_err(dev,
+                                     "health recovery flow aborted, PCI reads still not working\n");
+                       return;
+               }
+               msleep(100);
+       }
+
+       mlx5_core_err(dev, "starting health recovery flow\n");
+       mlx5_recover_device(dev);
 }
 
 static const char *hsynd_str(u8 synd)
@@ -274,6 +269,7 @@ static void poll_health(struct timer_list *t)
 {
        struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer);
        struct mlx5_core_health *health = &dev->priv.health;
+       u32 fatal_error;
        u32 count;
 
        if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
@@ -291,8 +287,11 @@ static void poll_health(struct timer_list *t)
                print_health_info(dev);
        }
 
-       if (in_fatal(dev) && !health->sick) {
-               health->sick = true;
+       fatal_error = check_fatal_sensors(dev);
+
+       if (fatal_error && !health->fatal_error) {
+               mlx5_core_err(dev, "Fatal error %u detected\n", fatal_error);
+               dev->priv.health.fatal_error = fatal_error;
                print_health_info(dev);
                mlx5_trigger_health_work(dev);
        }
@@ -306,9 +305,8 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev)
        struct mlx5_core_health *health = &dev->priv.health;
 
        timer_setup(&health->timer, poll_health, 0);
-       health->sick = 0;
+       health->fatal_error = MLX5_SENSOR_NO_ERR;
        clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
-       clear_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
        health->health = &dev->iseg->health;
        health->health_counter = &dev->iseg->health_counter;
 
@@ -324,7 +322,6 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health)
        if (disable_health) {
                spin_lock_irqsave(&health->wq_lock, flags);
                set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
-               set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
                spin_unlock_irqrestore(&health->wq_lock, flags);
        }
 
@@ -338,23 +335,10 @@ void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
 
        spin_lock_irqsave(&health->wq_lock, flags);
        set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
-       set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
        spin_unlock_irqrestore(&health->wq_lock, flags);
-       cancel_delayed_work_sync(&health->recover_work);
        cancel_work_sync(&health->work);
 }
 
-void mlx5_drain_health_recovery(struct mlx5_core_dev *dev)
-{
-       struct mlx5_core_health *health = &dev->priv.health;
-       unsigned long flags;
-
-       spin_lock_irqsave(&health->wq_lock, flags);
-       set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
-       spin_unlock_irqrestore(&health->wq_lock, flags);
-       cancel_delayed_work_sync(&dev->priv.health.recover_work);
-}
-
 void mlx5_health_flush(struct mlx5_core_dev *dev)
 {
        struct mlx5_core_health *health = &dev->priv.health;
@@ -387,7 +371,6 @@ int mlx5_health_init(struct mlx5_core_dev *dev)
                return -ENOMEM;
        spin_lock_init(&health->wq_lock);
        INIT_WORK(&health->work, health_care);
-       INIT_DELAYED_WORK(&health->recover_work, health_recover);
 
        return 0;
 }

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index c70e97071b872e4afdbe9f8b004c3947d2f470dc..fd0e2949c4f2355047fc979b701348b66d36fad0 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -1191,7 +1191,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup)
        int err = 0;
 
        if (cleanup)
-               mlx5_drain_health_recovery(dev);
+               mlx5_drain_health_wq(dev);
 
        mutex_lock(&dev->intf_state_mutex);
        if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index d4dd8c1ae55c668abb379ddadee350127165a5f3..97f8cf67ced091106bf1040074642f6855e67212 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -214,7 +214,7 @@ enum {
        MLX5_NIC_IFC_FULL               = 0,
        MLX5_NIC_IFC_DISABLED           = 1,
        MLX5_NIC_IFC_NO_DRAM_NIC        = 2,
-       MLX5_NIC_IFC_INVALID            = 3
+       MLX5_NIC_IFC_SW_RESET           = 7
 };
 
 u8 mlx5_get_nic_state(struct mlx5_core_dev *dev);

diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 4ae533b3da0748452a513feb65734f5aa6d86244..cc7fd8e62844376c10371a51f6bdb095c869ed9e 100644 (file)
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -435,6 +435,7 @@ struct mlx5_core_health {
        u32                             prev;
        int                             miss_counter;
        bool                            sick;
+       u32                             fatal_error;
        u32                             crdump_size;
        /* wq spinlock to synchronize draining */
        spinlock_t                      wq_lock;
@@ -906,7 +907,6 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev);
 void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health);
 void mlx5_drain_health_wq(struct mlx5_core_dev *dev);
 void mlx5_trigger_health_work(struct mlx5_core_dev *dev);
-void mlx5_drain_health_recovery(struct mlx5_core_dev *dev);
 int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size,
                        struct mlx5_frag_buf *buf, int node);
 int mlx5_buf_alloc(struct mlx5_core_dev *dev,