2 * Common Block IO controller cgroup interface
4 * Based on ideas and code from CFQ, CFS and BFQ:
5 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
7 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
8 * Paolo Valente <paolo.valente@unimore.it>
10 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
11 * Nauman Rafique <nauman@google.com>
13 * For policy-specific per-blkcg data:
14 * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
15 * Arianna Avanzini <avanzini.arianna@gmail.com>
17 #include <linux/ioprio.h>
18 #include <linux/kdev_t.h>
19 #include <linux/module.h>
20 #include <linux/sched/signal.h>
21 #include <linux/err.h>
22 #include <linux/blkdev.h>
23 #include <linux/backing-dev.h>
24 #include <linux/slab.h>
25 #include <linux/genhd.h>
26 #include <linux/delay.h>
27 #include <linux/atomic.h>
28 #include <linux/ctype.h>
29 #include <linux/blk-cgroup.h>
30 #include <linux/tracehook.h>
33 #define MAX_KEY_LEN 100
36 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
37 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
38 * policy [un]register operations including cgroup file additions /
39 * removals. Putting cgroup file registration outside blkcg_pol_mutex
40 * allows grabbing it from cgroup callbacks.
42 static DEFINE_MUTEX(blkcg_pol_register_mutex);
43 static DEFINE_MUTEX(blkcg_pol_mutex);
45 struct blkcg blkcg_root;
46 EXPORT_SYMBOL_GPL(blkcg_root);
48 struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
50 static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
52 static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
54 static bool blkcg_debug_stats = false;
56 static bool blkcg_policy_enabled(struct request_queue *q,
57 const struct blkcg_policy *pol)
59 return pol && test_bit(pol->plid, q->blkcg_pols);
63 * blkg_free - free a blkg
66 * Free @blkg which may be partially allocated.
68 static void blkg_free(struct blkcg_gq *blkg)
75 for (i = 0; i < BLKCG_MAX_POLS; i++)
77 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
79 blkg_rwstat_exit(&blkg->stat_ios);
80 blkg_rwstat_exit(&blkg->stat_bytes);
85 * blkg_alloc - allocate a blkg
86 * @blkcg: block cgroup the new blkg is associated with
87 * @q: request_queue the new blkg is associated with
88 * @gfp_mask: allocation mask to use
90 * Allocate a new blkg assocating @blkcg and @q.
92 static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
95 struct blkcg_gq *blkg;
98 /* alloc and init base part */
99 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
103 if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) ||
104 blkg_rwstat_init(&blkg->stat_ios, gfp_mask))
108 INIT_LIST_HEAD(&blkg->q_node);
110 atomic_set(&blkg->refcnt, 1);
112 for (i = 0; i < BLKCG_MAX_POLS; i++) {
113 struct blkcg_policy *pol = blkcg_policy[i];
114 struct blkg_policy_data *pd;
116 if (!blkcg_policy_enabled(q, pol))
119 /* alloc per-policy data and attach it to blkg */
120 pd = pol->pd_alloc_fn(gfp_mask, q->node);
136 struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
137 struct request_queue *q, bool update_hint)
139 struct blkcg_gq *blkg;
142 * Hint didn't match. Look up from the radix tree. Note that the
143 * hint can only be updated under queue_lock as otherwise @blkg
144 * could have already been removed from blkg_tree. The caller is
145 * responsible for grabbing queue_lock if @update_hint.
147 blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
148 if (blkg && blkg->q == q) {
150 lockdep_assert_held(&q->queue_lock);
151 rcu_assign_pointer(blkcg->blkg_hint, blkg);
158 EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
161 * If @new_blkg is %NULL, this function tries to allocate a new one as
162 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
164 static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
165 struct request_queue *q,
166 struct blkcg_gq *new_blkg)
168 struct blkcg_gq *blkg;
169 struct bdi_writeback_congested *wb_congested;
172 WARN_ON_ONCE(!rcu_read_lock_held());
173 lockdep_assert_held(&q->queue_lock);
175 /* blkg holds a reference to blkcg */
176 if (!css_tryget_online(&blkcg->css)) {
181 wb_congested = wb_congested_get_create(q->backing_dev_info,
183 GFP_NOWAIT | __GFP_NOWARN);
191 new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
192 if (unlikely(!new_blkg)) {
194 goto err_put_congested;
198 blkg->wb_congested = wb_congested;
201 if (blkcg_parent(blkcg)) {
202 blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
203 if (WARN_ON_ONCE(!blkg->parent)) {
205 goto err_put_congested;
207 blkg_get(blkg->parent);
210 /* invoke per-policy init */
211 for (i = 0; i < BLKCG_MAX_POLS; i++) {
212 struct blkcg_policy *pol = blkcg_policy[i];
214 if (blkg->pd[i] && pol->pd_init_fn)
215 pol->pd_init_fn(blkg->pd[i]);
219 spin_lock(&blkcg->lock);
220 ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
222 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
223 list_add(&blkg->q_node, &q->blkg_list);
225 for (i = 0; i < BLKCG_MAX_POLS; i++) {
226 struct blkcg_policy *pol = blkcg_policy[i];
228 if (blkg->pd[i] && pol->pd_online_fn)
229 pol->pd_online_fn(blkg->pd[i]);
233 spin_unlock(&blkcg->lock);
238 /* @blkg failed fully initialized, use the usual release path */
243 wb_congested_put(wb_congested);
245 css_put(&blkcg->css);
252 * __blkg_lookup_create - lookup blkg, try to create one if not there
253 * @blkcg: blkcg of interest
254 * @q: request_queue of interest
256 * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to
257 * create one. blkg creation is performed recursively from blkcg_root such
258 * that all non-root blkg's have access to the parent blkg. This function
259 * should be called under RCU read lock and @q->queue_lock.
261 * Returns pointer to the looked up or created blkg on success, ERR_PTR()
262 * value on error. If @q is dead, returns ERR_PTR(-EINVAL). If @q is not
263 * dead and bypassing, returns ERR_PTR(-EBUSY).
265 struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,
266 struct request_queue *q)
268 struct blkcg_gq *blkg;
270 WARN_ON_ONCE(!rcu_read_lock_held());
271 lockdep_assert_held(&q->queue_lock);
273 blkg = __blkg_lookup(blkcg, q, true);
278 * Create blkgs walking down from blkcg_root to @blkcg, so that all
279 * non-root blkgs have access to their parents.
282 struct blkcg *pos = blkcg;
283 struct blkcg *parent = blkcg_parent(blkcg);
285 while (parent && !__blkg_lookup(parent, q, false)) {
287 parent = blkcg_parent(parent);
290 blkg = blkg_create(pos, q, NULL);
291 if (pos == blkcg || IS_ERR(blkg))
297 * blkg_lookup_create - find or create a blkg
298 * @blkcg: target block cgroup
299 * @q: target request_queue
301 * This looks up or creates the blkg representing the unique pair
302 * of the blkcg and the request_queue.
304 struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
305 struct request_queue *q)
307 struct blkcg_gq *blkg = blkg_lookup(blkcg, q);
309 if (unlikely(!blkg)) {
310 spin_lock_irq(&q->queue_lock);
311 blkg = __blkg_lookup_create(blkcg, q);
312 spin_unlock_irq(&q->queue_lock);
318 static void blkg_destroy(struct blkcg_gq *blkg)
320 struct blkcg *blkcg = blkg->blkcg;
321 struct blkcg_gq *parent = blkg->parent;
324 lockdep_assert_held(&blkg->q->queue_lock);
325 lockdep_assert_held(&blkcg->lock);
327 /* Something wrong if we are trying to remove same group twice */
328 WARN_ON_ONCE(list_empty(&blkg->q_node));
329 WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
331 for (i = 0; i < BLKCG_MAX_POLS; i++) {
332 struct blkcg_policy *pol = blkcg_policy[i];
334 if (blkg->pd[i] && pol->pd_offline_fn)
335 pol->pd_offline_fn(blkg->pd[i]);
339 blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes);
340 blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios);
343 blkg->online = false;
345 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
346 list_del_init(&blkg->q_node);
347 hlist_del_init_rcu(&blkg->blkcg_node);
350 * Both setting lookup hint to and clearing it from @blkg are done
351 * under queue_lock. If it's not pointing to @blkg now, it never
352 * will. Hint assignment itself can race safely.
354 if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
355 rcu_assign_pointer(blkcg->blkg_hint, NULL);
358 * Put the reference taken at the time of creation so that when all
359 * queues are gone, group can be destroyed.
365 * blkg_destroy_all - destroy all blkgs associated with a request_queue
366 * @q: request_queue of interest
368 * Destroy all blkgs associated with @q.
370 static void blkg_destroy_all(struct request_queue *q)
372 struct blkcg_gq *blkg, *n;
374 spin_lock_irq(&q->queue_lock);
375 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
376 struct blkcg *blkcg = blkg->blkcg;
378 spin_lock(&blkcg->lock);
380 spin_unlock(&blkcg->lock);
384 spin_unlock_irq(&q->queue_lock);
388 * A group is RCU protected, but having an rcu lock does not mean that one
389 * can access all the fields of blkg and assume these are valid. For
390 * example, don't try to follow throtl_data and request queue links.
392 * Having a reference to blkg under an rcu allows accesses to only values
393 * local to groups like group stats and group rate limits.
395 void __blkg_release_rcu(struct rcu_head *rcu_head)
397 struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head);
399 /* release the blkcg and parent blkg refs this blkg has been holding */
400 css_put(&blkg->blkcg->css);
402 blkg_put(blkg->parent);
404 wb_congested_put(blkg->wb_congested);
408 EXPORT_SYMBOL_GPL(__blkg_release_rcu);
410 static int blkcg_reset_stats(struct cgroup_subsys_state *css,
411 struct cftype *cftype, u64 val)
413 struct blkcg *blkcg = css_to_blkcg(css);
414 struct blkcg_gq *blkg;
417 mutex_lock(&blkcg_pol_mutex);
418 spin_lock_irq(&blkcg->lock);
421 * Note that stat reset is racy - it doesn't synchronize against
422 * stat updates. This is a debug feature which shouldn't exist
423 * anyway. If you get hit by a race, retry.
425 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
426 blkg_rwstat_reset(&blkg->stat_bytes);
427 blkg_rwstat_reset(&blkg->stat_ios);
429 for (i = 0; i < BLKCG_MAX_POLS; i++) {
430 struct blkcg_policy *pol = blkcg_policy[i];
432 if (blkg->pd[i] && pol->pd_reset_stats_fn)
433 pol->pd_reset_stats_fn(blkg->pd[i]);
437 spin_unlock_irq(&blkcg->lock);
438 mutex_unlock(&blkcg_pol_mutex);
442 const char *blkg_dev_name(struct blkcg_gq *blkg)
444 /* some drivers (floppy) instantiate a queue w/o disk registered */
445 if (blkg->q->backing_dev_info->dev)
446 return dev_name(blkg->q->backing_dev_info->dev);
451 * blkcg_print_blkgs - helper for printing per-blkg data
452 * @sf: seq_file to print to
453 * @blkcg: blkcg of interest
454 * @prfill: fill function to print out a blkg
455 * @pol: policy in question
456 * @data: data to be passed to @prfill
457 * @show_total: to print out sum of prfill return values or not
459 * This function invokes @prfill on each blkg of @blkcg if pd for the
460 * policy specified by @pol exists. @prfill is invoked with @sf, the
461 * policy data and @data and the matching queue lock held. If @show_total
462 * is %true, the sum of the return values from @prfill is printed with
463 * "Total" label at the end.
465 * This is to be used to construct print functions for
466 * cftype->read_seq_string method.
468 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
469 u64 (*prfill)(struct seq_file *,
470 struct blkg_policy_data *, int),
471 const struct blkcg_policy *pol, int data,
474 struct blkcg_gq *blkg;
478 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
479 spin_lock_irq(&blkg->q->queue_lock);
480 if (blkcg_policy_enabled(blkg->q, pol))
481 total += prfill(sf, blkg->pd[pol->plid], data);
482 spin_unlock_irq(&blkg->q->queue_lock);
487 seq_printf(sf, "Total %llu\n", (unsigned long long)total);
489 EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
492 * __blkg_prfill_u64 - prfill helper for a single u64 value
493 * @sf: seq_file to print to
494 * @pd: policy private data of interest
497 * Print @v to @sf for the device assocaited with @pd.
499 u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
501 const char *dname = blkg_dev_name(pd->blkg);
506 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
509 EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
512 * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
513 * @sf: seq_file to print to
514 * @pd: policy private data of interest
515 * @rwstat: rwstat to print
517 * Print @rwstat to @sf for the device assocaited with @pd.
519 u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
520 const struct blkg_rwstat *rwstat)
522 static const char *rwstr[] = {
523 [BLKG_RWSTAT_READ] = "Read",
524 [BLKG_RWSTAT_WRITE] = "Write",
525 [BLKG_RWSTAT_SYNC] = "Sync",
526 [BLKG_RWSTAT_ASYNC] = "Async",
527 [BLKG_RWSTAT_DISCARD] = "Discard",
529 const char *dname = blkg_dev_name(pd->blkg);
536 for (i = 0; i < BLKG_RWSTAT_NR; i++)
537 seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
538 (unsigned long long)atomic64_read(&rwstat->aux_cnt[i]));
540 v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) +
541 atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]) +
542 atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_DISCARD]);
543 seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
546 EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
549 * blkg_prfill_stat - prfill callback for blkg_stat
550 * @sf: seq_file to print to
551 * @pd: policy private data of interest
552 * @off: offset to the blkg_stat in @pd
554 * prfill callback for printing a blkg_stat.
556 u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
558 return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
560 EXPORT_SYMBOL_GPL(blkg_prfill_stat);
563 * blkg_prfill_rwstat - prfill callback for blkg_rwstat
564 * @sf: seq_file to print to
565 * @pd: policy private data of interest
566 * @off: offset to the blkg_rwstat in @pd
568 * prfill callback for printing a blkg_rwstat.
570 u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
573 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
575 return __blkg_prfill_rwstat(sf, pd, &rwstat);
577 EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
579 static u64 blkg_prfill_rwstat_field(struct seq_file *sf,
580 struct blkg_policy_data *pd, int off)
582 struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off);
584 return __blkg_prfill_rwstat(sf, pd, &rwstat);
588 * blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes
589 * @sf: seq_file to print to
592 * To be used as cftype->seq_show to print blkg->stat_bytes.
593 * cftype->private must be set to the blkcg_policy.
595 int blkg_print_stat_bytes(struct seq_file *sf, void *v)
597 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
598 blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
599 offsetof(struct blkcg_gq, stat_bytes), true);
602 EXPORT_SYMBOL_GPL(blkg_print_stat_bytes);
605 * blkg_print_stat_bytes - seq_show callback for blkg->stat_ios
606 * @sf: seq_file to print to
609 * To be used as cftype->seq_show to print blkg->stat_ios. cftype->private
610 * must be set to the blkcg_policy.
612 int blkg_print_stat_ios(struct seq_file *sf, void *v)
614 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
615 blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private,
616 offsetof(struct blkcg_gq, stat_ios), true);
619 EXPORT_SYMBOL_GPL(blkg_print_stat_ios);
621 static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf,
622 struct blkg_policy_data *pd,
625 struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg,
627 return __blkg_prfill_rwstat(sf, pd, &rwstat);
631 * blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes
632 * @sf: seq_file to print to
635 int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v)
637 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
638 blkg_prfill_rwstat_field_recursive,
639 (void *)seq_cft(sf)->private,
640 offsetof(struct blkcg_gq, stat_bytes), true);
643 EXPORT_SYMBOL_GPL(blkg_print_stat_bytes_recursive);
646 * blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios
647 * @sf: seq_file to print to
650 int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v)
652 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
653 blkg_prfill_rwstat_field_recursive,
654 (void *)seq_cft(sf)->private,
655 offsetof(struct blkcg_gq, stat_ios), true);
658 EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive);
661 * blkg_stat_recursive_sum - collect hierarchical blkg_stat
662 * @blkg: blkg of interest
663 * @pol: blkcg_policy which contains the blkg_stat
664 * @off: offset to the blkg_stat in blkg_policy_data or @blkg
666 * Collect the blkg_stat specified by @blkg, @pol and @off and all its
667 * online descendants and their aux counts. The caller must be holding the
668 * queue lock for online tests.
670 * If @pol is NULL, blkg_stat is at @off bytes into @blkg; otherwise, it is
671 * at @off bytes into @blkg's blkg_policy_data of the policy.
673 u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg,
674 struct blkcg_policy *pol, int off)
676 struct blkcg_gq *pos_blkg;
677 struct cgroup_subsys_state *pos_css;
680 lockdep_assert_held(&blkg->q->queue_lock);
683 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
684 struct blkg_stat *stat;
686 if (!pos_blkg->online)
690 stat = (void *)blkg_to_pd(pos_blkg, pol) + off;
692 stat = (void *)blkg + off;
694 sum += blkg_stat_read(stat) + atomic64_read(&stat->aux_cnt);
700 EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum);
703 * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat
704 * @blkg: blkg of interest
705 * @pol: blkcg_policy which contains the blkg_rwstat
706 * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg
708 * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its
709 * online descendants and their aux counts. The caller must be holding the
710 * queue lock for online tests.
712 * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it
713 * is at @off bytes into @blkg's blkg_policy_data of the policy.
715 struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg,
716 struct blkcg_policy *pol, int off)
718 struct blkcg_gq *pos_blkg;
719 struct cgroup_subsys_state *pos_css;
720 struct blkg_rwstat sum = { };
723 lockdep_assert_held(&blkg->q->queue_lock);
726 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
727 struct blkg_rwstat *rwstat;
729 if (!pos_blkg->online)
733 rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off;
735 rwstat = (void *)pos_blkg + off;
737 for (i = 0; i < BLKG_RWSTAT_NR; i++)
738 atomic64_add(atomic64_read(&rwstat->aux_cnt[i]) +
739 percpu_counter_sum_positive(&rwstat->cpu_cnt[i]),
746 EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum);
748 /* Performs queue bypass and policy enabled checks then looks up blkg. */
749 static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg,
750 const struct blkcg_policy *pol,
751 struct request_queue *q)
753 WARN_ON_ONCE(!rcu_read_lock_held());
754 lockdep_assert_held(&q->queue_lock);
756 if (!blkcg_policy_enabled(q, pol))
757 return ERR_PTR(-EOPNOTSUPP);
758 return __blkg_lookup(blkcg, q, true /* update_hint */);
762 * blkg_conf_prep - parse and prepare for per-blkg config update
763 * @blkcg: target block cgroup
764 * @pol: target policy
765 * @input: input string
766 * @ctx: blkg_conf_ctx to be filled
768 * Parse per-blkg config update from @input and initialize @ctx with the
769 * result. @ctx->blkg points to the blkg to be updated and @ctx->body the
770 * part of @input following MAJ:MIN. This function returns with RCU read
771 * lock and queue lock held and must be paired with blkg_conf_finish().
773 int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
774 char *input, struct blkg_conf_ctx *ctx)
775 __acquires(rcu) __acquires(&disk->queue->queue_lock)
777 struct gendisk *disk;
778 struct request_queue *q;
779 struct blkcg_gq *blkg;
780 unsigned int major, minor;
781 int key_len, part, ret;
784 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
787 body = input + key_len;
790 body = skip_spaces(body);
792 disk = get_gendisk(MKDEV(major, minor), &part);
803 spin_lock_irq(&q->queue_lock);
805 blkg = blkg_lookup_check(blkcg, pol, q);
815 * Create blkgs walking down from blkcg_root to @blkcg, so that all
816 * non-root blkgs have access to their parents.
819 struct blkcg *pos = blkcg;
820 struct blkcg *parent;
821 struct blkcg_gq *new_blkg;
823 parent = blkcg_parent(blkcg);
824 while (parent && !__blkg_lookup(parent, q, false)) {
826 parent = blkcg_parent(parent);
829 /* Drop locks to do new blkg allocation with GFP_KERNEL. */
830 spin_unlock_irq(&q->queue_lock);
833 new_blkg = blkg_alloc(pos, q, GFP_KERNEL);
834 if (unlikely(!new_blkg)) {
840 spin_lock_irq(&q->queue_lock);
842 blkg = blkg_lookup_check(pos, pol, q);
851 blkg = blkg_create(pos, q, new_blkg);
852 if (unlikely(IS_ERR(blkg))) {
868 spin_unlock_irq(&q->queue_lock);
871 put_disk_and_module(disk);
873 * If queue was bypassing, we should retry. Do so after a
874 * short msleep(). It isn't strictly necessary but queue
875 * can be bypassing for some time and it's always nice to
876 * avoid busy looping.
880 ret = restart_syscall();
886 * blkg_conf_finish - finish up per-blkg config update
887 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
889 * Finish up after per-blkg config update. This function must be paired
890 * with blkg_conf_prep().
892 void blkg_conf_finish(struct blkg_conf_ctx *ctx)
893 __releases(&ctx->disk->queue->queue_lock) __releases(rcu)
895 spin_unlock_irq(&ctx->disk->queue->queue_lock);
897 put_disk_and_module(ctx->disk);
900 static int blkcg_print_stat(struct seq_file *sf, void *v)
902 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
903 struct blkcg_gq *blkg;
907 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
910 struct blkg_rwstat rwstat;
911 u64 rbytes, wbytes, rios, wios, dbytes, dios;
912 size_t size = seq_get_buf(sf, &buf), off = 0;
914 bool has_stats = false;
916 dname = blkg_dev_name(blkg);
921 * Hooray string manipulation, count is the size written NOT
922 * INCLUDING THE \0, so size is now count+1 less than what we
923 * had before, but we want to start writing the next bit from
924 * the \0 so we only add count to buf.
926 off += scnprintf(buf+off, size-off, "%s ", dname);
928 spin_lock_irq(&blkg->q->queue_lock);
930 rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
931 offsetof(struct blkcg_gq, stat_bytes));
932 rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
933 wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
934 dbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_DISCARD]);
936 rwstat = blkg_rwstat_recursive_sum(blkg, NULL,
937 offsetof(struct blkcg_gq, stat_ios));
938 rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]);
939 wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]);
940 dios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_DISCARD]);
942 spin_unlock_irq(&blkg->q->queue_lock);
944 if (rbytes || wbytes || rios || wios) {
946 off += scnprintf(buf+off, size-off,
947 "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
948 rbytes, wbytes, rios, wios,
952 if (!blkcg_debug_stats)
955 if (atomic_read(&blkg->use_delay)) {
957 off += scnprintf(buf+off, size-off,
958 " use_delay=%d delay_nsec=%llu",
959 atomic_read(&blkg->use_delay),
960 (unsigned long long)atomic64_read(&blkg->delay_nsec));
963 for (i = 0; i < BLKCG_MAX_POLS; i++) {
964 struct blkcg_policy *pol = blkcg_policy[i];
967 if (!blkg->pd[i] || !pol->pd_stat_fn)
970 written = pol->pd_stat_fn(blkg->pd[i], buf+off, size-off);
977 off += scnprintf(buf+off, size-off, "\n");
986 static struct cftype blkcg_files[] = {
989 .flags = CFTYPE_NOT_ON_ROOT,
990 .seq_show = blkcg_print_stat,
995 static struct cftype blkcg_legacy_files[] = {
997 .name = "reset_stats",
998 .write_u64 = blkcg_reset_stats,
1004 * blkcg destruction is a three-stage process.
1006 * 1. Destruction starts. The blkcg_css_offline() callback is invoked
1007 * which offlines writeback. Here we tie the next stage of blkg destruction
1008 * to the completion of writeback associated with the blkcg. This lets us
1009 * avoid punting potentially large amounts of outstanding writeback to root
1010 * while maintaining any ongoing policies. The next stage is triggered when
1011 * the nr_cgwbs count goes to zero.
1013 * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called
1014 * and handles the destruction of blkgs. Here the css reference held by
1015 * the blkg is put back eventually allowing blkcg_css_free() to be called.
1016 * This work may occur in cgwb_release_workfn() on the cgwb_release
1017 * workqueue. Any submitted ios that fail to get the blkg ref will be
1018 * punted to the root_blkg.
1020 * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called.
1021 * This finally frees the blkcg.
1025 * blkcg_css_offline - cgroup css_offline callback
1026 * @css: css of interest
1028 * This function is called when @css is about to go away. Here the cgwbs are
1029 * offlined first and only once writeback associated with the blkcg has
1030 * finished do we start step 2 (see above).
1032 static void blkcg_css_offline(struct cgroup_subsys_state *css)
1034 struct blkcg *blkcg = css_to_blkcg(css);
1036 /* this prevents anyone from attaching or migrating to this blkcg */
1037 wb_blkcg_offline(blkcg);
1039 /* put the base cgwb reference allowing step 2 to be triggered */
1040 blkcg_cgwb_put(blkcg);
1044 * blkcg_destroy_blkgs - responsible for shooting down blkgs
1045 * @blkcg: blkcg of interest
1047 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock
1048 * is nested inside q lock, this function performs reverse double lock dancing.
1049 * Destroying the blkgs releases the reference held on the blkcg's css allowing
1050 * blkcg_css_free to eventually be called.
1052 * This is the blkcg counterpart of ioc_release_fn().
1054 void blkcg_destroy_blkgs(struct blkcg *blkcg)
1056 spin_lock_irq(&blkcg->lock);
1058 while (!hlist_empty(&blkcg->blkg_list)) {
1059 struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
1060 struct blkcg_gq, blkcg_node);
1061 struct request_queue *q = blkg->q;
1063 if (spin_trylock(&q->queue_lock)) {
1065 spin_unlock(&q->queue_lock);
1067 spin_unlock_irq(&blkcg->lock);
1069 spin_lock_irq(&blkcg->lock);
1073 spin_unlock_irq(&blkcg->lock);
1076 static void blkcg_css_free(struct cgroup_subsys_state *css)
1078 struct blkcg *blkcg = css_to_blkcg(css);
1081 mutex_lock(&blkcg_pol_mutex);
1083 list_del(&blkcg->all_blkcgs_node);
1085 for (i = 0; i < BLKCG_MAX_POLS; i++)
1087 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1089 mutex_unlock(&blkcg_pol_mutex);
1094 static struct cgroup_subsys_state *
1095 blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
1097 struct blkcg *blkcg;
1098 struct cgroup_subsys_state *ret;
1101 mutex_lock(&blkcg_pol_mutex);
1104 blkcg = &blkcg_root;
1106 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1108 ret = ERR_PTR(-ENOMEM);
1113 for (i = 0; i < BLKCG_MAX_POLS ; i++) {
1114 struct blkcg_policy *pol = blkcg_policy[i];
1115 struct blkcg_policy_data *cpd;
1118 * If the policy hasn't been attached yet, wait for it
1119 * to be attached before doing anything else. Otherwise,
1120 * check if the policy requires any specific per-cgroup
1121 * data: if it does, allocate and initialize it.
1123 if (!pol || !pol->cpd_alloc_fn)
1126 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1128 ret = ERR_PTR(-ENOMEM);
1131 blkcg->cpd[i] = cpd;
1134 if (pol->cpd_init_fn)
1135 pol->cpd_init_fn(cpd);
1138 spin_lock_init(&blkcg->lock);
1139 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
1140 INIT_HLIST_HEAD(&blkcg->blkg_list);
1141 #ifdef CONFIG_CGROUP_WRITEBACK
1142 INIT_LIST_HEAD(&blkcg->cgwb_list);
1143 refcount_set(&blkcg->cgwb_refcnt, 1);
1145 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
1147 mutex_unlock(&blkcg_pol_mutex);
1151 for (i--; i >= 0; i--)
1153 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1155 if (blkcg != &blkcg_root)
1158 mutex_unlock(&blkcg_pol_mutex);
1163 * blkcg_init_queue - initialize blkcg part of request queue
1164 * @q: request_queue to initialize
1166 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
1167 * part of new request_queue @q.
1170 * 0 on success, -errno on failure.
1172 int blkcg_init_queue(struct request_queue *q)
1174 struct blkcg_gq *new_blkg, *blkg;
1178 new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
1182 preloaded = !radix_tree_preload(GFP_KERNEL);
1184 /* Make sure the root blkg exists. */
1186 spin_lock_irq(&q->queue_lock);
1187 blkg = blkg_create(&blkcg_root, q, new_blkg);
1190 q->root_blkg = blkg;
1191 spin_unlock_irq(&q->queue_lock);
1195 radix_tree_preload_end();
1197 ret = blk_iolatency_init(q);
1199 goto err_destroy_all;
1201 ret = blk_throtl_init(q);
1203 goto err_destroy_all;
1207 blkg_destroy_all(q);
1210 spin_unlock_irq(&q->queue_lock);
1213 radix_tree_preload_end();
1214 return PTR_ERR(blkg);
1218 * blkcg_drain_queue - drain blkcg part of request_queue
1219 * @q: request_queue to drain
1221 * Called from blk_drain_queue(). Responsible for draining blkcg part.
1223 void blkcg_drain_queue(struct request_queue *q)
1225 lockdep_assert_held(&q->queue_lock);
1228 * @q could be exiting and already have destroyed all blkgs as
1229 * indicated by NULL root_blkg. If so, don't confuse policies.
1234 blk_throtl_drain(q);
1238 * blkcg_exit_queue - exit and release blkcg part of request_queue
1239 * @q: request_queue being released
1241 * Called from blk_release_queue(). Responsible for exiting blkcg part.
1243 void blkcg_exit_queue(struct request_queue *q)
1245 blkg_destroy_all(q);
1250 * We cannot support shared io contexts, as we have no mean to support
1251 * two tasks with the same ioc in two different groups without major rework
1252 * of the main cic data structures. For now we allow a task to change
1253 * its cgroup only if it's the only owner of its ioc.
1255 static int blkcg_can_attach(struct cgroup_taskset *tset)
1257 struct task_struct *task;
1258 struct cgroup_subsys_state *dst_css;
1259 struct io_context *ioc;
1262 /* task_lock() is needed to avoid races with exit_io_context() */
1263 cgroup_taskset_for_each(task, dst_css, tset) {
1265 ioc = task->io_context;
1266 if (ioc && atomic_read(&ioc->nr_tasks) > 1)
1275 static void blkcg_bind(struct cgroup_subsys_state *root_css)
1279 mutex_lock(&blkcg_pol_mutex);
1281 for (i = 0; i < BLKCG_MAX_POLS; i++) {
1282 struct blkcg_policy *pol = blkcg_policy[i];
1283 struct blkcg *blkcg;
1285 if (!pol || !pol->cpd_bind_fn)
1288 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)
1289 if (blkcg->cpd[pol->plid])
1290 pol->cpd_bind_fn(blkcg->cpd[pol->plid]);
1292 mutex_unlock(&blkcg_pol_mutex);
1295 static void blkcg_exit(struct task_struct *tsk)
1297 if (tsk->throttle_queue)
1298 blk_put_queue(tsk->throttle_queue);
1299 tsk->throttle_queue = NULL;
1302 struct cgroup_subsys io_cgrp_subsys = {
1303 .css_alloc = blkcg_css_alloc,
1304 .css_offline = blkcg_css_offline,
1305 .css_free = blkcg_css_free,
1306 .can_attach = blkcg_can_attach,
1308 .dfl_cftypes = blkcg_files,
1309 .legacy_cftypes = blkcg_legacy_files,
1310 .legacy_name = "blkio",
1314 * This ensures that, if available, memcg is automatically enabled
1315 * together on the default hierarchy so that the owner cgroup can
1316 * be retrieved from writeback pages.
1318 .depends_on = 1 << memory_cgrp_id,
1321 EXPORT_SYMBOL_GPL(io_cgrp_subsys);
1324 * blkcg_activate_policy - activate a blkcg policy on a request_queue
1325 * @q: request_queue of interest
1326 * @pol: blkcg policy to activate
1328 * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through
1329 * bypass mode to populate its blkgs with policy_data for @pol.
1331 * Activation happens with @q bypassed, so nobody would be accessing blkgs
1332 * from IO path. Update of each blkg is protected by both queue and blkcg
1333 * locks so that holding either lock and testing blkcg_policy_enabled() is
1334 * always enough for dereferencing policy data.
1336 * The caller is responsible for synchronizing [de]activations and policy
1337 * [un]registerations. Returns 0 on success, -errno on failure.
1339 int blkcg_activate_policy(struct request_queue *q,
1340 const struct blkcg_policy *pol)
1342 struct blkg_policy_data *pd_prealloc = NULL;
1343 struct blkcg_gq *blkg;
1346 if (blkcg_policy_enabled(q, pol))
1350 blk_mq_freeze_queue(q);
1353 pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q->node);
1356 goto out_bypass_end;
1360 spin_lock_irq(&q->queue_lock);
1362 list_for_each_entry(blkg, &q->blkg_list, q_node) {
1363 struct blkg_policy_data *pd;
1365 if (blkg->pd[pol->plid])
1368 pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q->node);
1370 swap(pd, pd_prealloc);
1372 spin_unlock_irq(&q->queue_lock);
1376 blkg->pd[pol->plid] = pd;
1378 pd->plid = pol->plid;
1379 if (pol->pd_init_fn)
1380 pol->pd_init_fn(pd);
1383 __set_bit(pol->plid, q->blkcg_pols);
1386 spin_unlock_irq(&q->queue_lock);
1389 blk_mq_unfreeze_queue(q);
1391 pol->pd_free_fn(pd_prealloc);
1394 EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1397 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
1398 * @q: request_queue of interest
1399 * @pol: blkcg policy to deactivate
1401 * Deactivate @pol on @q. Follows the same synchronization rules as
1402 * blkcg_activate_policy().
1404 void blkcg_deactivate_policy(struct request_queue *q,
1405 const struct blkcg_policy *pol)
1407 struct blkcg_gq *blkg;
1409 if (!blkcg_policy_enabled(q, pol))
1413 blk_mq_freeze_queue(q);
1415 spin_lock_irq(&q->queue_lock);
1417 __clear_bit(pol->plid, q->blkcg_pols);
1419 list_for_each_entry(blkg, &q->blkg_list, q_node) {
1420 if (blkg->pd[pol->plid]) {
1421 if (pol->pd_offline_fn)
1422 pol->pd_offline_fn(blkg->pd[pol->plid]);
1423 pol->pd_free_fn(blkg->pd[pol->plid]);
1424 blkg->pd[pol->plid] = NULL;
1428 spin_unlock_irq(&q->queue_lock);
1431 blk_mq_unfreeze_queue(q);
1433 EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1436 * blkcg_policy_register - register a blkcg policy
1437 * @pol: blkcg policy to register
1439 * Register @pol with blkcg core. Might sleep and @pol may be modified on
1440 * successful registration. Returns 0 on success and -errno on failure.
1442 int blkcg_policy_register(struct blkcg_policy *pol)
1444 struct blkcg *blkcg;
1447 mutex_lock(&blkcg_pol_register_mutex);
1448 mutex_lock(&blkcg_pol_mutex);
1450 /* find an empty slot */
1452 for (i = 0; i < BLKCG_MAX_POLS; i++)
1453 if (!blkcg_policy[i])
1455 if (i >= BLKCG_MAX_POLS) {
1456 pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");
1460 /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
1461 if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
1462 (!pol->pd_alloc_fn ^ !pol->pd_free_fn))
1467 blkcg_policy[pol->plid] = pol;
1469 /* allocate and install cpd's */
1470 if (pol->cpd_alloc_fn) {
1471 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1472 struct blkcg_policy_data *cpd;
1474 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1478 blkcg->cpd[pol->plid] = cpd;
1480 cpd->plid = pol->plid;
1481 pol->cpd_init_fn(cpd);
1485 mutex_unlock(&blkcg_pol_mutex);
1487 /* everything is in place, add intf files for the new policy */
1488 if (pol->dfl_cftypes)
1489 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
1491 if (pol->legacy_cftypes)
1492 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
1493 pol->legacy_cftypes));
1494 mutex_unlock(&blkcg_pol_register_mutex);
1498 if (pol->cpd_free_fn) {
1499 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1500 if (blkcg->cpd[pol->plid]) {
1501 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1502 blkcg->cpd[pol->plid] = NULL;
1506 blkcg_policy[pol->plid] = NULL;
1508 mutex_unlock(&blkcg_pol_mutex);
1509 mutex_unlock(&blkcg_pol_register_mutex);
1512 EXPORT_SYMBOL_GPL(blkcg_policy_register);
1515 * blkcg_policy_unregister - unregister a blkcg policy
1516 * @pol: blkcg policy to unregister
1518 * Undo blkcg_policy_register(@pol). Might sleep.
1520 void blkcg_policy_unregister(struct blkcg_policy *pol)
1522 struct blkcg *blkcg;
1524 mutex_lock(&blkcg_pol_register_mutex);
1526 if (WARN_ON(blkcg_policy[pol->plid] != pol))
1529 /* kill the intf files first */
1530 if (pol->dfl_cftypes)
1531 cgroup_rm_cftypes(pol->dfl_cftypes);
1532 if (pol->legacy_cftypes)
1533 cgroup_rm_cftypes(pol->legacy_cftypes);
1535 /* remove cpds and unregister */
1536 mutex_lock(&blkcg_pol_mutex);
1538 if (pol->cpd_free_fn) {
1539 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1540 if (blkcg->cpd[pol->plid]) {
1541 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1542 blkcg->cpd[pol->plid] = NULL;
1546 blkcg_policy[pol->plid] = NULL;
1548 mutex_unlock(&blkcg_pol_mutex);
1550 mutex_unlock(&blkcg_pol_register_mutex);
1552 EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
1555 * Scale the accumulated delay based on how long it has been since we updated
1556 * the delay. We only call this when we are adding delay, in case it's been a
1557 * while since we added delay, and when we are checking to see if we need to
1558 * delay a task, to account for any delays that may have occurred.
1560 static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now)
1562 u64 old = atomic64_read(&blkg->delay_start);
1565 * We only want to scale down every second. The idea here is that we
1566 * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain
1567 * time window. We only want to throttle tasks for recent delay that
1568 * has occurred, in 1 second time windows since that's the maximum
1569 * things can be throttled. We save the current delay window in
1570 * blkg->last_delay so we know what amount is still left to be charged
1571 * to the blkg from this point onward. blkg->last_use keeps track of
1572 * the use_delay counter. The idea is if we're unthrottling the blkg we
1573 * are ok with whatever is happening now, and we can take away more of
1574 * the accumulated delay as we've already throttled enough that
1575 * everybody is happy with their IO latencies.
1577 if (time_before64(old + NSEC_PER_SEC, now) &&
1578 atomic64_cmpxchg(&blkg->delay_start, old, now) == old) {
1579 u64 cur = atomic64_read(&blkg->delay_nsec);
1580 u64 sub = min_t(u64, blkg->last_delay, now - old);
1581 int cur_use = atomic_read(&blkg->use_delay);
1584 * We've been unthrottled, subtract a larger chunk of our
1585 * accumulated delay.
1587 if (cur_use < blkg->last_use)
1588 sub = max_t(u64, sub, blkg->last_delay >> 1);
1591 * This shouldn't happen, but handle it anyway. Our delay_nsec
1592 * should only ever be growing except here where we subtract out
1593 * min(last_delay, 1 second), but lord knows bugs happen and I'd
1594 * rather not end up with negative numbers.
1596 if (unlikely(cur < sub)) {
1597 atomic64_set(&blkg->delay_nsec, 0);
1598 blkg->last_delay = 0;
1600 atomic64_sub(sub, &blkg->delay_nsec);
1601 blkg->last_delay = cur - sub;
1603 blkg->last_use = cur_use;
1608 * This is called when we want to actually walk up the hierarchy and check to
1609 * see if we need to throttle, and then actually throttle if there is some
1610 * accumulated delay. This should only be called upon return to user space so
1611 * we're not holding some lock that would induce a priority inversion.
1613 static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
1615 u64 now = ktime_to_ns(ktime_get());
1620 while (blkg->parent) {
1621 if (atomic_read(&blkg->use_delay)) {
1622 blkcg_scale_delay(blkg, now);
1623 delay_nsec = max_t(u64, delay_nsec,
1624 atomic64_read(&blkg->delay_nsec));
1626 blkg = blkg->parent;
1633 * Let's not sleep for all eternity if we've amassed a huge delay.
1634 * Swapping or metadata IO can accumulate 10's of seconds worth of
1635 * delay, and we want userspace to be able to do _something_ so cap the
1636 * delays at 1 second. If there's 10's of seconds worth of delay then
1637 * the tasks will be delayed for 1 second for every syscall.
1639 delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
1642 * TODO: the use_memdelay flag is going to be for the upcoming psi stuff
1643 * that hasn't landed upstream yet. Once that stuff is in place we need
1644 * to do a psi_memstall_enter/leave if memdelay is set.
1647 exp = ktime_add_ns(now, delay_nsec);
1648 tok = io_schedule_prepare();
1650 __set_current_state(TASK_KILLABLE);
1651 if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS))
1653 } while (!fatal_signal_pending(current));
1654 io_schedule_finish(tok);
1658 * blkcg_maybe_throttle_current - throttle the current task if it has been marked
1660 * This is only called if we've been marked with set_notify_resume(). Obviously
1661 * we can be set_notify_resume() for reasons other than blkcg throttling, so we
1662 * check to see if current->throttle_queue is set and if not this doesn't do
1663 * anything. This should only ever be called by the resume code, it's not meant
1664 * to be called by people willy-nilly as it will actually do the work to
1665 * throttle the task if it is setup for throttling.
1667 void blkcg_maybe_throttle_current(void)
1669 struct request_queue *q = current->throttle_queue;
1670 struct cgroup_subsys_state *css;
1671 struct blkcg *blkcg;
1672 struct blkcg_gq *blkg;
1673 bool use_memdelay = current->use_memdelay;
1678 current->throttle_queue = NULL;
1679 current->use_memdelay = false;
1682 css = kthread_blkcg();
1684 blkcg = css_to_blkcg(css);
1686 blkcg = css_to_blkcg(task_css(current, io_cgrp_id));
1690 blkg = blkg_lookup(blkcg, q);
1693 blkg = blkg_try_get(blkg);
1698 blkcg_maybe_throttle_blkg(blkg, use_memdelay);
1708 * blkcg_schedule_throttle - this task needs to check for throttling
1709 * @q - the request queue IO was submitted on
1710 * @use_memdelay - do we charge this to memory delay for PSI
1712 * This is called by the IO controller when we know there's delay accumulated
1713 * for the blkg for this task. We do not pass the blkg because there are places
1714 * we call this that may not have that information, the swapping code for
1715 * instance will only have a request_queue at that point. This set's the
1716 * notify_resume for the task to check and see if it requires throttling before
1717 * returning to user space.
1719 * We will only schedule once per syscall. You can call this over and over
1720 * again and it will only do the check once upon return to user space, and only
1721 * throttle once. If the task needs to be throttled again it'll need to be
1722 * re-set at the next time we see the task.
1724 void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay)
1726 if (unlikely(current->flags & PF_KTHREAD))
1729 if (!blk_get_queue(q))
1732 if (current->throttle_queue)
1733 blk_put_queue(current->throttle_queue);
1734 current->throttle_queue = q;
1736 current->use_memdelay = use_memdelay;
1737 set_notify_resume(current);
1741 * blkcg_add_delay - add delay to this blkg
1742 * @now - the current time in nanoseconds
1743 * @delta - how many nanoseconds of delay to add
1745 * Charge @delta to the blkg's current delay accumulation. This is used to
1746 * throttle tasks if an IO controller thinks we need more throttling.
1748 void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
1750 blkcg_scale_delay(blkg, now);
1751 atomic64_add(delta, &blkg->delay_nsec);
1754 module_param(blkcg_debug_stats, bool, 0644);
1755 MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");