io-wq: fix IO_WQ_WORK_NO_CANCEL cancellation

To cancel a work, io-wq sets IO_WQ_WORK_CANCEL and executes the
callback. However, IO_WQ_WORK_NO_CANCEL works will just execute and may
return next work, which will be ignored and lost.

Cancel the whole link.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.c b/fs/io-wq.c
index bf8ed1b0b90a01ce356588ed8d9c1184ca2510f7..9a7aacc96d8414942b4aa60d0233cf288d5703d9 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -747,6 +747,17 @@ static bool io_wq_can_queue(struct io_wqe *wqe, struct io_wqe_acct *acct,
        return true;
 }
 
+static void io_run_cancel(struct io_wq_work *work)
+{
+       do {
+               struct io_wq_work *old_work = work;
+
+               work->flags |= IO_WQ_WORK_CANCEL;
+               work->func(&work);
+               work = (work == old_work) ? NULL : work;
+       } while (work);
+}
+
 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)
 {
        struct io_wqe_acct *acct = io_work_get_acct(wqe, work);
@@ -760,8 +771,7 @@ static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work)
         * It's close enough to not be an issue, fork() has the same delay.
         */
        if (unlikely(!io_wq_can_queue(wqe, acct, work))) {
-               work->flags |= IO_WQ_WORK_CANCEL;
-               work->func(&work);
+               io_run_cancel(work);
                return;
        }
 
@@ -900,8 +910,7 @@ static enum io_wq_cancel io_wqe_cancel_cb_work(struct io_wqe *wqe,
        spin_unlock_irqrestore(&wqe->lock, flags);
 
        if (found) {
-               work->flags |= IO_WQ_WORK_CANCEL;
-               work->func(&work);
+               io_run_cancel(work);
                return IO_WQ_CANCEL_OK;
        }
 
@@ -976,8 +985,7 @@ static enum io_wq_cancel io_wqe_cancel_work(struct io_wqe *wqe,
        spin_unlock_irqrestore(&wqe->lock, flags);
 
        if (found) {
-               work->flags |= IO_WQ_WORK_CANCEL;
-               work->func(&work);
+               io_run_cancel(work);
                return IO_WQ_CANCEL_OK;
        }