block: replace bio->bi_issue_stat with bio-specific type

struct blk_issue_stat is going away, and bio->bi_issue_stat doesn't even
use the blk-stats interface, so we can provide a separate implementation
specific for bios. The helpers work the same way as the blk-stats
helpers.

Signed-off-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index c5a1316737331ba785a0c569aac1615994cb1996..35f9b8ff40d734bf2ac89c2316c4b204a70262b9 100644 (file)
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -36,8 +36,6 @@ static int throtl_quantum = 32;
  */
 #define LATENCY_FILTERED_HD (1000L) /* 1ms */
 
-#define SKIP_LATENCY (((u64)1) << BLK_STAT_RES_SHIFT)
-
 static struct blkcg_policy blkcg_policy_throtl;
 
 /* A workqueue to queue throttle related work */
@@ -2139,7 +2137,7 @@ static void blk_throtl_assoc_bio(struct throtl_grp *tg, struct bio *bio)
                bio->bi_cg_private = tg;
                blkg_get(tg_to_blkg(tg));
        }
-       blk_stat_set_issue(&bio->bi_issue_stat, bio_sectors(bio));
+       bio_issue_init(&bio->bi_issue, bio_sectors(bio));
 #endif
 }
 
@@ -2251,7 +2249,7 @@ out:
 
 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
        if (throttled || !td->track_bio_latency)
-               bio->bi_issue_stat.stat |= SKIP_LATENCY;
+               bio->bi_issue.value |= BIO_ISSUE_THROTL_SKIP_LATENCY;
 #endif
        return throttled;
 }
@@ -2302,8 +2300,8 @@ void blk_throtl_bio_endio(struct bio *bio)
        finish_time_ns = ktime_get_ns();
        tg->last_finish_time = finish_time_ns >> 10;
 
-       start_time = blk_stat_time(&bio->bi_issue_stat) >> 10;
-       finish_time = __blk_stat_time(finish_time_ns) >> 10;
+       start_time = bio_issue_time(&bio->bi_issue) >> 10;
+       finish_time = __bio_issue_time(finish_time_ns) >> 10;
        if (!start_time || finish_time <= start_time) {
                blkg_put(tg_to_blkg(tg));
                return;
@@ -2311,16 +2309,15 @@ void blk_throtl_bio_endio(struct bio *bio)
 
        lat = finish_time - start_time;
        /* this is only for bio based driver */
-       if (!(bio->bi_issue_stat.stat & SKIP_LATENCY))
-               throtl_track_latency(tg->td, blk_stat_size(&bio->bi_issue_stat),
-                       bio_op(bio), lat);
+       if (!(bio->bi_issue.value & BIO_ISSUE_THROTL_SKIP_LATENCY))
+               throtl_track_latency(tg->td, bio_issue_size(&bio->bi_issue),
+                                    bio_op(bio), lat);
 
        if (tg->latency_target && lat >= tg->td->filtered_latency) {
                int bucket;
                unsigned int threshold;
 
-               bucket = request_bucket_index(
-                       blk_stat_size(&bio->bi_issue_stat));
+               bucket = request_bucket_index(bio_issue_size(&bio->bi_issue));
                threshold = tg->td->avg_buckets[rw][bucket].latency +
                        tg->latency_target;
                if (lat > threshold)

diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 17b18b91ebac91a68244b51ca57cd086fd2d60a5..b6f1d53cf11395817e6607361bcf632cdc626a7a 100644 (file)
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -8,6 +8,7 @@
 
 #include <linux/types.h>
 #include <linux/bvec.h>
+#include <linux/ktime.h>
 
 struct bio_set;
 struct bio;
@@ -94,6 +95,52 @@ struct blk_issue_stat {
        u64 stat;
 };
 
+/*
+ * From most significant bit:
+ * 1 bit: reserved for other usage, see below
+ * 12 bits: original size of bio
+ * 51 bits: issue time of bio
+ */
+#define BIO_ISSUE_RES_BITS      1
+#define BIO_ISSUE_SIZE_BITS     12
+#define BIO_ISSUE_RES_SHIFT     (64 - BIO_ISSUE_RES_BITS)
+#define BIO_ISSUE_SIZE_SHIFT    (BIO_ISSUE_RES_SHIFT - BIO_ISSUE_SIZE_BITS)
+#define BIO_ISSUE_TIME_MASK     ((1ULL << BIO_ISSUE_SIZE_SHIFT) - 1)
+#define BIO_ISSUE_SIZE_MASK     \
+       (((1ULL << BIO_ISSUE_SIZE_BITS) - 1) << BIO_ISSUE_SIZE_SHIFT)
+#define BIO_ISSUE_RES_MASK      (~((1ULL << BIO_ISSUE_RES_SHIFT) - 1))
+
+/* Reserved bit for blk-throtl */
+#define BIO_ISSUE_THROTL_SKIP_LATENCY (1ULL << 63)
+
+struct bio_issue {
+       u64 value;
+};
+
+static inline u64 __bio_issue_time(u64 time)
+{
+       return time & BIO_ISSUE_TIME_MASK;
+}
+
+static inline u64 bio_issue_time(struct bio_issue *issue)
+{
+       return __bio_issue_time(issue->value);
+}
+
+static inline sector_t bio_issue_size(struct bio_issue *issue)
+{
+       return ((issue->value & BIO_ISSUE_SIZE_MASK) >> BIO_ISSUE_SIZE_SHIFT);
+}
+
+static inline void bio_issue_init(struct bio_issue *issue,
+                                      sector_t size)
+{
+       size &= (1ULL << BIO_ISSUE_SIZE_BITS) - 1;
+       issue->value = ((issue->value & BIO_ISSUE_RES_MASK) |
+                       (ktime_get_ns() & BIO_ISSUE_TIME_MASK) |
+                       ((u64)size << BIO_ISSUE_SIZE_SHIFT));
+}
+
 /*
  * main unit of I/O for the block layer and lower layers (ie drivers and
  * stacking drivers)
@@ -138,7 +185,7 @@ struct bio {
        struct cgroup_subsys_state *bi_css;
 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
        void                    *bi_cg_private;
-       struct blk_issue_stat   bi_issue_stat;
+       struct bio_issue        bi_issue;
 #endif
 #endif
        union {