block: enable multipage bvecs

This patch pulls the trigger for multi-page bvecs.

Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bio.c b/block/bio.c
index 968b12fea5645d92bc9bcc02c1fb7813791d1c4c..83a2dfa417caaddf31289090488f41de6096a6b8 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -753,6 +753,8 @@ EXPORT_SYMBOL(bio_add_pc_page);
  * @page: page to add
  * @len: length of the data to add
  * @off: offset of the data in @page
+ * @same_page: if %true only merge if the new data is in the same physical
+ *             page as the last segment of the bio.
  *
  * Try to add the data at @page + @off to the last bvec of @bio.  This is a
  * a useful optimisation for file systems with a block size smaller than the
@@ -761,19 +763,25 @@ EXPORT_SYMBOL(bio_add_pc_page);
  * Return %true on success or %false on failure.
  */
 bool __bio_try_merge_page(struct bio *bio, struct page *page,
-               unsigned int len, unsigned int off)
+               unsigned int len, unsigned int off, bool same_page)
 {
        if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
                return false;
 
        if (bio->bi_vcnt > 0) {
                struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
+               phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) +
+                       bv->bv_offset + bv->bv_len - 1;
+               phys_addr_t page_addr = page_to_phys(page);
 
-               if (page == bv->bv_page && off == bv->bv_offset + bv->bv_len) {
-                       bv->bv_len += len;
-                       bio->bi_iter.bi_size += len;
-                       return true;
-               }
+               if (vec_end_addr + 1 != page_addr + off)
+                       return false;
+               if (same_page && (vec_end_addr & PAGE_MASK) != page_addr)
+                       return false;
+
+               bv->bv_len += len;
+               bio->bi_iter.bi_size += len;
+               return true;
        }
        return false;
 }
@@ -819,7 +827,7 @@ EXPORT_SYMBOL_GPL(__bio_add_page);
 int bio_add_page(struct bio *bio, struct page *page,
                 unsigned int len, unsigned int offset)
 {
-       if (!__bio_try_merge_page(bio, page, len, offset)) {
+       if (!__bio_try_merge_page(bio, page, len, offset, false)) {
                if (bio_full(bio))
                        return 0;
                __bio_add_page(bio, page, len, offset);

diff --git a/fs/iomap.c b/fs/iomap.c
index af736acd9006196eeb49db6bd9290a209c393b2e..0c350e658b7f1d2d9c106e369089dc4ce8a8ed83 100644 (file)
--- a/fs/iomap.c
+++ b/fs/iomap.c
@@ -318,7 +318,7 @@ iomap_readpage_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
         */
        sector = iomap_sector(iomap, pos);
        if (ctx->bio && bio_end_sector(ctx->bio) == sector) {
-               if (__bio_try_merge_page(ctx->bio, page, plen, poff))
+               if (__bio_try_merge_page(ctx->bio, page, plen, poff, true))
                        goto done;
                is_contig = true;
        }
@@ -349,7 +349,7 @@ iomap_readpage_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
                ctx->bio->bi_end_io = iomap_read_end_io;
        }
 
-       __bio_add_page(ctx->bio, page, plen, poff);
+       bio_add_page(ctx->bio, page, plen, poff);
 done:
        /*
         * Move the caller beyond our range so that it keeps making progress.

diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c
index 1f1829e506e8d5904c2e60d81eeaa1a17e31bc00..b9fd44168f61d46a0cfa986c0436e885bd6275a1 100644 (file)
--- a/fs/xfs/xfs_aops.c
+++ b/fs/xfs/xfs_aops.c
@@ -616,12 +616,12 @@ xfs_add_to_ioend(
                                bdev, sector);
        }
 
-       if (!__bio_try_merge_page(wpc->ioend->io_bio, page, len, poff)) {
+       if (!__bio_try_merge_page(wpc->ioend->io_bio, page, len, poff, true)) {
                if (iop)
                        atomic_inc(&iop->write_count);
                if (bio_full(wpc->ioend->io_bio))
                        xfs_chain_bio(wpc->ioend, wbc, bdev, sector);
-               __bio_add_page(wpc->ioend->io_bio, page, len, poff);
+               bio_add_page(wpc->ioend->io_bio, page, len, poff);
        }
 
        wpc->ioend->io_size += len;

diff --git a/include/linux/bio.h b/include/linux/bio.h
index 089370eb84d97bd1b471d5789bc0010d6adbe911..9f77adcfde823acd4b98a01d94716cdad286ac1b 100644 (file)
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -441,7 +441,7 @@ extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
 extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
                           unsigned int, unsigned int);
 bool __bio_try_merge_page(struct bio *bio, struct page *page,
-               unsigned int len, unsigned int off);
+               unsigned int len, unsigned int off, bool same_page);
 void __bio_add_page(struct bio *bio, struct page *page,
                unsigned int len, unsigned int off);
 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);