2 * buffered writeback throttling. loosely based on CoDel. We can't drop
3 * packets for IO scheduling, so the logic is something like this:
5 * - Monitor latencies in a defined window of time.
6 * - If the minimum latency in the above window exceeds some target, increment
7 * scaling step and scale down queue depth by a factor of 2x. The monitoring
8 * window is then shrunk to 100 / sqrt(scaling step + 1).
9 * - For any window where we don't have solid data on what the latencies
10 * look like, retain status quo.
11 * - If latencies look good, decrement scaling step.
12 * - If we're only doing writes, allow the scaling step to go negative. This
13 * will temporarily boost write performance, snapping back to a stable
14 * scaling step of 0 if reads show up or the heavy writers finish. Unlike
15 * positive scaling steps where we shrink the monitoring window, a negative
16 * scaling step retains the default step==0 window size.
18 * Copyright (C) 2016 Jens Axboe
21 #include <linux/kernel.h>
22 #include <linux/blk_types.h>
23 #include <linux/slab.h>
24 #include <linux/backing-dev.h>
25 #include <linux/swap.h>
28 #include "blk-rq-qos.h"
30 #define CREATE_TRACE_POINTS
31 #include <trace/events/wbt.h>
33 static inline void wbt_clear_state(struct request *rq)
38 static inline enum wbt_flags wbt_flags(struct request *rq)
43 static inline bool wbt_is_tracked(struct request *rq)
45 return rq->wbt_flags & WBT_TRACKED;
48 static inline bool wbt_is_read(struct request *rq)
50 return rq->wbt_flags & WBT_READ;
55 * Default setting, we'll scale up (to 75% of QD max) or down (min 1)
56 * from here depending on device stats
63 RWB_WINDOW_NSEC = 100 * 1000 * 1000ULL,
66 * Disregard stats, if we don't meet this minimum
68 RWB_MIN_WRITE_SAMPLES = 3,
71 * If we have this number of consecutive windows with not enough
72 * information to scale up or down, scale up.
77 static inline bool rwb_enabled(struct rq_wb *rwb)
79 return rwb && rwb->wb_normal != 0;
82 static void wb_timestamp(struct rq_wb *rwb, unsigned long *var)
84 if (rwb_enabled(rwb)) {
85 const unsigned long cur = jiffies;
93 * If a task was rate throttled in balance_dirty_pages() within the last
94 * second or so, use that to indicate a higher cleaning rate.
96 static bool wb_recent_wait(struct rq_wb *rwb)
98 struct bdi_writeback *wb = &rwb->rqos.q->backing_dev_info->wb;
100 return time_before(jiffies, wb->dirty_sleep + HZ);
103 static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb,
104 enum wbt_flags wb_acct)
106 if (wb_acct & WBT_KSWAPD)
107 return &rwb->rq_wait[WBT_RWQ_KSWAPD];
108 else if (wb_acct & WBT_DISCARD)
109 return &rwb->rq_wait[WBT_RWQ_DISCARD];
111 return &rwb->rq_wait[WBT_RWQ_BG];
114 static void rwb_wake_all(struct rq_wb *rwb)
118 for (i = 0; i < WBT_NUM_RWQ; i++) {
119 struct rq_wait *rqw = &rwb->rq_wait[i];
121 if (wq_has_sleeper(&rqw->wait))
122 wake_up_all(&rqw->wait);
126 static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw,
127 enum wbt_flags wb_acct)
131 inflight = atomic_dec_return(&rqw->inflight);
134 * wbt got disabled with IO in flight. Wake up any potential
135 * waiters, we don't have to do more than that.
137 if (unlikely(!rwb_enabled(rwb))) {
143 * For discards, our limit is always the background. For writes, if
144 * the device does write back caching, drop further down before we
147 if (wb_acct & WBT_DISCARD)
148 limit = rwb->wb_background;
149 else if (rwb->wc && !wb_recent_wait(rwb))
152 limit = rwb->wb_normal;
155 * Don't wake anyone up if we are above the normal limit.
157 if (inflight && inflight >= limit)
160 if (wq_has_sleeper(&rqw->wait)) {
161 int diff = limit - inflight;
163 if (!inflight || diff >= rwb->wb_background / 2)
164 wake_up_all(&rqw->wait);
168 static void __wbt_done(struct rq_qos *rqos, enum wbt_flags wb_acct)
170 struct rq_wb *rwb = RQWB(rqos);
173 if (!(wb_acct & WBT_TRACKED))
176 rqw = get_rq_wait(rwb, wb_acct);
177 wbt_rqw_done(rwb, rqw, wb_acct);
181 * Called on completion of a request. Note that it's also called when
182 * a request is merged, when the request gets freed.
184 static void wbt_done(struct rq_qos *rqos, struct request *rq)
186 struct rq_wb *rwb = RQWB(rqos);
188 if (!wbt_is_tracked(rq)) {
189 if (rwb->sync_cookie == rq) {
191 rwb->sync_cookie = NULL;
195 wb_timestamp(rwb, &rwb->last_comp);
197 WARN_ON_ONCE(rq == rwb->sync_cookie);
198 __wbt_done(rqos, wbt_flags(rq));
203 static inline bool stat_sample_valid(struct blk_rq_stat *stat)
206 * We need at least one read sample, and a minimum of
207 * RWB_MIN_WRITE_SAMPLES. We require some write samples to know
208 * that it's writes impacting us, and not just some sole read on
209 * a device that is in a lower power state.
211 return (stat[READ].nr_samples >= 1 &&
212 stat[WRITE].nr_samples >= RWB_MIN_WRITE_SAMPLES);
215 static u64 rwb_sync_issue_lat(struct rq_wb *rwb)
217 u64 now, issue = READ_ONCE(rwb->sync_issue);
219 if (!issue || !rwb->sync_cookie)
222 now = ktime_to_ns(ktime_get());
233 static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat)
235 struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info;
236 struct rq_depth *rqd = &rwb->rq_depth;
240 * If our stored sync issue exceeds the window size, or it
241 * exceeds our min target AND we haven't logged any entries,
242 * flag the latency as exceeded. wbt works off completion latencies,
243 * but for a flooded device, a single sync IO can take a long time
244 * to complete after being issued. If this time exceeds our
245 * monitoring window AND we didn't see any other completions in that
246 * window, then count that sync IO as a violation of the latency.
248 thislat = rwb_sync_issue_lat(rwb);
249 if (thislat > rwb->cur_win_nsec ||
250 (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) {
251 trace_wbt_lat(bdi, thislat);
256 * No read/write mix, if stat isn't valid
258 if (!stat_sample_valid(stat)) {
260 * If we had writes in this stat window and the window is
261 * current, we're only doing writes. If a task recently
262 * waited or still has writes in flights, consider us doing
263 * just writes as well.
265 if (stat[WRITE].nr_samples || wb_recent_wait(rwb) ||
267 return LAT_UNKNOWN_WRITES;
272 * If the 'min' latency exceeds our target, step down.
274 if (stat[READ].min > rwb->min_lat_nsec) {
275 trace_wbt_lat(bdi, stat[READ].min);
276 trace_wbt_stat(bdi, stat);
281 trace_wbt_stat(bdi, stat);
286 static void rwb_trace_step(struct rq_wb *rwb, const char *msg)
288 struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info;
289 struct rq_depth *rqd = &rwb->rq_depth;
291 trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec,
292 rwb->wb_background, rwb->wb_normal, rqd->max_depth);
295 static void calc_wb_limits(struct rq_wb *rwb)
297 if (rwb->min_lat_nsec == 0) {
298 rwb->wb_normal = rwb->wb_background = 0;
299 } else if (rwb->rq_depth.max_depth <= 2) {
300 rwb->wb_normal = rwb->rq_depth.max_depth;
301 rwb->wb_background = 1;
303 rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2;
304 rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4;
308 static void scale_up(struct rq_wb *rwb)
310 if (!rq_depth_scale_up(&rwb->rq_depth))
313 rwb->unknown_cnt = 0;
315 rwb_trace_step(rwb, "scale up");
318 static void scale_down(struct rq_wb *rwb, bool hard_throttle)
320 if (!rq_depth_scale_down(&rwb->rq_depth, hard_throttle))
323 rwb->unknown_cnt = 0;
324 rwb_trace_step(rwb, "scale down");
327 static void rwb_arm_timer(struct rq_wb *rwb)
329 struct rq_depth *rqd = &rwb->rq_depth;
331 if (rqd->scale_step > 0) {
333 * We should speed this up, using some variant of a fast
334 * integer inverse square root calculation. Since we only do
335 * this for every window expiration, it's not a huge deal,
338 rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4,
339 int_sqrt((rqd->scale_step + 1) << 8));
342 * For step < 0, we don't want to increase/decrease the
345 rwb->cur_win_nsec = rwb->win_nsec;
348 blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec);
351 static void wb_timer_fn(struct blk_stat_callback *cb)
353 struct rq_wb *rwb = cb->data;
354 struct rq_depth *rqd = &rwb->rq_depth;
355 unsigned int inflight = wbt_inflight(rwb);
358 status = latency_exceeded(rwb, cb->stat);
360 trace_wbt_timer(rwb->rqos.q->backing_dev_info, status, rqd->scale_step,
364 * If we exceeded the latency target, step down. If we did not,
365 * step one level up. If we don't know enough to say either exceeded
366 * or ok, then don't do anything.
370 scale_down(rwb, true);
375 case LAT_UNKNOWN_WRITES:
377 * We started a the center step, but don't have a valid
378 * read/write sample, but we do have writes going on.
379 * Allow step to go negative, to increase write perf.
384 if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP)
387 * We get here when previously scaled reduced depth, and we
388 * currently don't have a valid read/write sample. For that
389 * case, slowly return to center state (step == 0).
391 if (rqd->scale_step > 0)
393 else if (rqd->scale_step < 0)
394 scale_down(rwb, false);
401 * Re-arm timer, if we have IO in flight
403 if (rqd->scale_step || inflight)
407 static void __wbt_update_limits(struct rq_wb *rwb)
409 struct rq_depth *rqd = &rwb->rq_depth;
412 rqd->scaled_max = false;
414 rq_depth_calc_max_depth(rqd);
420 void wbt_update_limits(struct request_queue *q)
422 struct rq_qos *rqos = wbt_rq_qos(q);
425 __wbt_update_limits(RQWB(rqos));
428 u64 wbt_get_min_lat(struct request_queue *q)
430 struct rq_qos *rqos = wbt_rq_qos(q);
433 return RQWB(rqos)->min_lat_nsec;
436 void wbt_set_min_lat(struct request_queue *q, u64 val)
438 struct rq_qos *rqos = wbt_rq_qos(q);
441 RQWB(rqos)->min_lat_nsec = val;
442 RQWB(rqos)->enable_state = WBT_STATE_ON_MANUAL;
443 __wbt_update_limits(RQWB(rqos));
447 static bool close_io(struct rq_wb *rwb)
449 const unsigned long now = jiffies;
451 return time_before(now, rwb->last_issue + HZ / 10) ||
452 time_before(now, rwb->last_comp + HZ / 10);
455 #define REQ_HIPRIO (REQ_SYNC | REQ_META | REQ_PRIO)
457 static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw)
462 * If we got disabled, just return UINT_MAX. This ensures that
463 * we'll properly inc a new IO, and dec+wakeup at the end.
465 if (!rwb_enabled(rwb))
468 if ((rw & REQ_OP_MASK) == REQ_OP_DISCARD)
469 return rwb->wb_background;
472 * At this point we know it's a buffered write. If this is
473 * kswapd trying to free memory, or REQ_SYNC is set, then
474 * it's WB_SYNC_ALL writeback, and we'll use the max limit for
475 * that. If the write is marked as a background write, then use
476 * the idle limit, or go to normal if we haven't had competing
479 if ((rw & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd())
480 limit = rwb->rq_depth.max_depth;
481 else if ((rw & REQ_BACKGROUND) || close_io(rwb)) {
483 * If less than 100ms since we completed unrelated IO,
484 * limit us to half the depth for background writeback.
486 limit = rwb->wb_background;
488 limit = rwb->wb_normal;
493 struct wbt_wait_data {
494 struct wait_queue_entry wq;
495 struct task_struct *task;
502 static int wbt_wake_function(struct wait_queue_entry *curr, unsigned int mode,
503 int wake_flags, void *key)
505 struct wbt_wait_data *data = container_of(curr, struct wbt_wait_data,
509 * If we fail to get a budget, return -1 to interrupt the wake up
510 * loop in __wake_up_common.
512 if (!rq_wait_inc_below(data->rqw, get_limit(data->rwb, data->rw)))
515 data->got_token = true;
516 list_del_init(&curr->entry);
517 wake_up_process(data->task);
522 * Block if we will exceed our limit, or if we are currently waiting for
523 * the timer to kick off queuing again.
525 static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct,
526 unsigned long rw, spinlock_t *lock)
530 struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);
531 struct wbt_wait_data data = {
533 .func = wbt_wake_function,
534 .entry = LIST_HEAD_INIT(data.wq.entry),
543 has_sleeper = wq_has_sleeper(&rqw->wait);
544 if (!has_sleeper && rq_wait_inc_below(rqw, get_limit(rwb, rw)))
547 prepare_to_wait_exclusive(&rqw->wait, &data.wq, TASK_UNINTERRUPTIBLE);
553 rq_wait_inc_below(rqw, get_limit(rwb, rw))) {
554 finish_wait(&rqw->wait, &data.wq);
557 * We raced with wbt_wake_function() getting a token,
558 * which means we now have two. Put our local token
559 * and wake anyone else potentially waiting for one.
562 wbt_rqw_done(rwb, rqw, wb_acct);
567 spin_unlock_irq(lock);
576 finish_wait(&rqw->wait, &data.wq);
579 static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)
581 switch (bio_op(bio)) {
584 * Don't throttle WRITE_ODIRECT
586 if ((bio->bi_opf & (REQ_SYNC | REQ_IDLE)) ==
587 (REQ_SYNC | REQ_IDLE))
597 static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio)
599 enum wbt_flags flags = 0;
601 if (!rwb_enabled(rwb))
604 if (bio_op(bio) == REQ_OP_READ) {
606 } else if (wbt_should_throttle(rwb, bio)) {
607 if (current_is_kswapd())
609 if (bio_op(bio) == REQ_OP_DISCARD)
610 flags |= WBT_DISCARD;
611 flags |= WBT_TRACKED;
616 static void wbt_cleanup(struct rq_qos *rqos, struct bio *bio)
618 struct rq_wb *rwb = RQWB(rqos);
619 enum wbt_flags flags = bio_to_wbt_flags(rwb, bio);
620 __wbt_done(rqos, flags);
624 * Returns true if the IO request should be accounted, false if not.
625 * May sleep, if we have exceeded the writeback limits. Caller can pass
626 * in an irq held spinlock, if it holds one when calling this function.
627 * If we do sleep, we'll release and re-grab it.
629 static void wbt_wait(struct rq_qos *rqos, struct bio *bio, spinlock_t *lock)
631 struct rq_wb *rwb = RQWB(rqos);
632 enum wbt_flags flags;
634 flags = bio_to_wbt_flags(rwb, bio);
635 if (!(flags & WBT_TRACKED)) {
636 if (flags & WBT_READ)
637 wb_timestamp(rwb, &rwb->last_issue);
641 __wbt_wait(rwb, flags, bio->bi_opf, lock);
643 if (!blk_stat_is_active(rwb->cb))
647 static void wbt_track(struct rq_qos *rqos, struct request *rq, struct bio *bio)
649 struct rq_wb *rwb = RQWB(rqos);
650 rq->wbt_flags |= bio_to_wbt_flags(rwb, bio);
653 void wbt_issue(struct rq_qos *rqos, struct request *rq)
655 struct rq_wb *rwb = RQWB(rqos);
657 if (!rwb_enabled(rwb))
661 * Track sync issue, in case it takes a long time to complete. Allows us
662 * to react quicker, if a sync IO takes a long time to complete. Note
663 * that this is just a hint. The request can go away when it completes,
664 * so it's important we never dereference it. We only use the address to
665 * compare with, which is why we store the sync_issue time locally.
667 if (wbt_is_read(rq) && !rwb->sync_issue) {
668 rwb->sync_cookie = rq;
669 rwb->sync_issue = rq->io_start_time_ns;
673 void wbt_requeue(struct rq_qos *rqos, struct request *rq)
675 struct rq_wb *rwb = RQWB(rqos);
676 if (!rwb_enabled(rwb))
678 if (rq == rwb->sync_cookie) {
680 rwb->sync_cookie = NULL;
684 void wbt_set_queue_depth(struct request_queue *q, unsigned int depth)
686 struct rq_qos *rqos = wbt_rq_qos(q);
688 RQWB(rqos)->rq_depth.queue_depth = depth;
689 __wbt_update_limits(RQWB(rqos));
693 void wbt_set_write_cache(struct request_queue *q, bool write_cache_on)
695 struct rq_qos *rqos = wbt_rq_qos(q);
697 RQWB(rqos)->wc = write_cache_on;
701 * Enable wbt if defaults are configured that way
703 void wbt_enable_default(struct request_queue *q)
705 struct rq_qos *rqos = wbt_rq_qos(q);
706 /* Throttling already enabled? */
710 /* Queue not registered? Maybe shutting down... */
711 if (!test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
714 if ((q->mq_ops && IS_ENABLED(CONFIG_BLK_WBT_MQ)) ||
715 (q->request_fn && IS_ENABLED(CONFIG_BLK_WBT_SQ)))
718 EXPORT_SYMBOL_GPL(wbt_enable_default);
720 u64 wbt_default_latency_nsec(struct request_queue *q)
723 * We default to 2msec for non-rotational storage, and 75msec
724 * for rotational storage.
726 if (blk_queue_nonrot(q))
732 static int wbt_data_dir(const struct request *rq)
734 const int op = req_op(rq);
736 if (op == REQ_OP_READ)
738 else if (op_is_write(op))
745 static void wbt_exit(struct rq_qos *rqos)
747 struct rq_wb *rwb = RQWB(rqos);
748 struct request_queue *q = rqos->q;
750 blk_stat_remove_callback(q, rwb->cb);
751 blk_stat_free_callback(rwb->cb);
756 * Disable wbt, if enabled by default.
758 void wbt_disable_default(struct request_queue *q)
760 struct rq_qos *rqos = wbt_rq_qos(q);
765 if (rwb->enable_state == WBT_STATE_ON_DEFAULT) {
766 blk_stat_deactivate(rwb->cb);
770 EXPORT_SYMBOL_GPL(wbt_disable_default);
773 static struct rq_qos_ops wbt_rqos_ops = {
774 .throttle = wbt_wait,
777 .requeue = wbt_requeue,
779 .cleanup = wbt_cleanup,
783 int wbt_init(struct request_queue *q)
788 rwb = kzalloc(sizeof(*rwb), GFP_KERNEL);
792 rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb);
798 for (i = 0; i < WBT_NUM_RWQ; i++)
799 rq_wait_init(&rwb->rq_wait[i]);
801 rwb->rqos.id = RQ_QOS_WBT;
802 rwb->rqos.ops = &wbt_rqos_ops;
804 rwb->last_comp = rwb->last_issue = jiffies;
805 rwb->win_nsec = RWB_WINDOW_NSEC;
806 rwb->enable_state = WBT_STATE_ON_DEFAULT;
808 rwb->rq_depth.default_depth = RWB_DEF_DEPTH;
809 __wbt_update_limits(rwb);
812 * Assign rwb and add the stats callback.
814 rq_qos_add(q, &rwb->rqos);
815 blk_stat_add_callback(q, rwb->cb);
817 rwb->min_lat_nsec = wbt_default_latency_nsec(q);
819 wbt_set_queue_depth(q, blk_queue_depth(q));
820 wbt_set_write_cache(q, test_bit(QUEUE_FLAG_WC, &q->queue_flags));