From 5238dcf4136fd7287be8e7d38752645bfa5782ec Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Wed, 9 May 2018 02:08:49 -0700 Subject: [PATCH] block: replace bio->bi_issue_stat with bio-specific type struct blk_issue_stat is going away, and bio->bi_issue_stat doesn't even use the blk-stats interface, so we can provide a separate implementation specific for bios. The helpers work the same way as the blk-stats helpers. Signed-off-by: Omar Sandoval Signed-off-by: Jens Axboe --- block/blk-throttle.c | 19 +++++++-------- include/linux/blk_types.h | 49 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/block/blk-throttle.c b/block/blk-throttle.c index c5a131673733..35f9b8ff40d7 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -36,8 +36,6 @@ static int throtl_quantum = 32; */ #define LATENCY_FILTERED_HD (1000L) /* 1ms */ -#define SKIP_LATENCY (((u64)1) << BLK_STAT_RES_SHIFT) - static struct blkcg_policy blkcg_policy_throtl; /* A workqueue to queue throttle related work */ @@ -2139,7 +2137,7 @@ static void blk_throtl_assoc_bio(struct throtl_grp *tg, struct bio *bio) bio->bi_cg_private = tg; blkg_get(tg_to_blkg(tg)); } - blk_stat_set_issue(&bio->bi_issue_stat, bio_sectors(bio)); + bio_issue_init(&bio->bi_issue, bio_sectors(bio)); #endif } @@ -2251,7 +2249,7 @@ out: #ifdef CONFIG_BLK_DEV_THROTTLING_LOW if (throttled || !td->track_bio_latency) - bio->bi_issue_stat.stat |= SKIP_LATENCY; + bio->bi_issue.value |= BIO_ISSUE_THROTL_SKIP_LATENCY; #endif return throttled; } @@ -2302,8 +2300,8 @@ void blk_throtl_bio_endio(struct bio *bio) finish_time_ns = ktime_get_ns(); tg->last_finish_time = finish_time_ns >> 10; - start_time = blk_stat_time(&bio->bi_issue_stat) >> 10; - finish_time = __blk_stat_time(finish_time_ns) >> 10; + start_time = bio_issue_time(&bio->bi_issue) >> 10; + finish_time = __bio_issue_time(finish_time_ns) >> 10; if (!start_time || finish_time <= start_time) { blkg_put(tg_to_blkg(tg)); return; @@ -2311,16 +2309,15 @@ void blk_throtl_bio_endio(struct bio *bio) lat = finish_time - start_time; /* this is only for bio based driver */ - if (!(bio->bi_issue_stat.stat & SKIP_LATENCY)) - throtl_track_latency(tg->td, blk_stat_size(&bio->bi_issue_stat), - bio_op(bio), lat); + if (!(bio->bi_issue.value & BIO_ISSUE_THROTL_SKIP_LATENCY)) + throtl_track_latency(tg->td, bio_issue_size(&bio->bi_issue), + bio_op(bio), lat); if (tg->latency_target && lat >= tg->td->filtered_latency) { int bucket; unsigned int threshold; - bucket = request_bucket_index( - blk_stat_size(&bio->bi_issue_stat)); + bucket = request_bucket_index(bio_issue_size(&bio->bi_issue)); threshold = tg->td->avg_buckets[rw][bucket].latency + tg->latency_target; if (lat > threshold) diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 17b18b91ebac..b6f1d53cf113 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -8,6 +8,7 @@ #include #include +#include struct bio_set; struct bio; @@ -94,6 +95,52 @@ struct blk_issue_stat { u64 stat; }; +/* + * From most significant bit: + * 1 bit: reserved for other usage, see below + * 12 bits: original size of bio + * 51 bits: issue time of bio + */ +#define BIO_ISSUE_RES_BITS 1 +#define BIO_ISSUE_SIZE_BITS 12 +#define BIO_ISSUE_RES_SHIFT (64 - BIO_ISSUE_RES_BITS) +#define BIO_ISSUE_SIZE_SHIFT (BIO_ISSUE_RES_SHIFT - BIO_ISSUE_SIZE_BITS) +#define BIO_ISSUE_TIME_MASK ((1ULL << BIO_ISSUE_SIZE_SHIFT) - 1) +#define BIO_ISSUE_SIZE_MASK \ + (((1ULL << BIO_ISSUE_SIZE_BITS) - 1) << BIO_ISSUE_SIZE_SHIFT) +#define BIO_ISSUE_RES_MASK (~((1ULL << BIO_ISSUE_RES_SHIFT) - 1)) + +/* Reserved bit for blk-throtl */ +#define BIO_ISSUE_THROTL_SKIP_LATENCY (1ULL << 63) + +struct bio_issue { + u64 value; +}; + +static inline u64 __bio_issue_time(u64 time) +{ + return time & BIO_ISSUE_TIME_MASK; +} + +static inline u64 bio_issue_time(struct bio_issue *issue) +{ + return __bio_issue_time(issue->value); +} + +static inline sector_t bio_issue_size(struct bio_issue *issue) +{ + return ((issue->value & BIO_ISSUE_SIZE_MASK) >> BIO_ISSUE_SIZE_SHIFT); +} + +static inline void bio_issue_init(struct bio_issue *issue, + sector_t size) +{ + size &= (1ULL << BIO_ISSUE_SIZE_BITS) - 1; + issue->value = ((issue->value & BIO_ISSUE_RES_MASK) | + (ktime_get_ns() & BIO_ISSUE_TIME_MASK) | + ((u64)size << BIO_ISSUE_SIZE_SHIFT)); +} + /* * main unit of I/O for the block layer and lower layers (ie drivers and * stacking drivers) @@ -138,7 +185,7 @@ struct bio { struct cgroup_subsys_state *bi_css; #ifdef CONFIG_BLK_DEV_THROTTLING_LOW void *bi_cg_private; - struct blk_issue_stat bi_issue_stat; + struct bio_issue bi_issue; #endif #endif union { -- 2.30.2