block/blk-iocost.c

   1 /* SPDX-License-Identifier: GPL-2.0
   2  *
   3  * IO cost model based controller.
   4  *
   5  * Copyright (C) 2019 Tejun Heo <tj@kernel.org>
   6  * Copyright (C) 2019 Andy Newell <newella@fb.com>
   7  * Copyright (C) 2019 Facebook
   8  *
   9  * One challenge of controlling IO resources is the lack of trivially
  10  * observable cost metric.  This is distinguished from CPU and memory where
  11  * wallclock time and the number of bytes can serve as accurate enough
  12  * approximations.
  13  *
  14  * Bandwidth and iops are the most commonly used metrics for IO devices but
  15  * depending on the type and specifics of the device, different IO patterns
  16  * easily lead to multiple orders of magnitude variations rendering them
  17  * useless for the purpose of IO capacity distribution.  While on-device
  18  * time, with a lot of clutches, could serve as a useful approximation for
  19  * non-queued rotational devices, this is no longer viable with modern
  20  * devices, even the rotational ones.
  21  *
  22  * While there is no cost metric we can trivially observe, it isn't a
  23  * complete mystery.  For example, on a rotational device, seek cost
  24  * dominates while a contiguous transfer contributes a smaller amount
  25  * proportional to the size.  If we can characterize at least the relative
  26  * costs of these different types of IOs, it should be possible to
  27  * implement a reasonable work-conserving proportional IO resource
  28  * distribution.
  29  *
  30  * 1. IO Cost Model
  31  *
  32  * IO cost model estimates the cost of an IO given its basic parameters and
  33  * history (e.g. the end sector of the last IO).  The cost is measured in
  34  * device time.  If a given IO is estimated to cost 10ms, the device should
  35  * be able to process ~100 of those IOs in a second.
  36  *
  37  * Currently, there's only one builtin cost model - linear.  Each IO is
  38  * classified as sequential or random and given a base cost accordingly.
  39  * On top of that, a size cost proportional to the length of the IO is
  40  * added.  While simple, this model captures the operational
  41  * characteristics of a wide varienty of devices well enough.  Default
  42  * paramters for several different classes of devices are provided and the
  43  * parameters can be configured from userspace via
  44  * /sys/fs/cgroup/io.cost.model.
  45  *
  46  * If needed, tools/cgroup/iocost_coef_gen.py can be used to generate
  47  * device-specific coefficients.
  48  *
  49  * 2. Control Strategy
  50  *
  51  * The device virtual time (vtime) is used as the primary control metric.
  52  * The control strategy is composed of the following three parts.
  53  *
  54  * 2-1. Vtime Distribution
  55  *
  56  * When a cgroup becomes active in terms of IOs, its hierarchical share is
  57  * calculated.  Please consider the following hierarchy where the numbers
  58  * inside parentheses denote the configured weights.
  59  *
  60  *           root
  61  *         /       \
  62  *      A (w:100)  B (w:300)
  63  *      /       \
  64  *  A0 (w:100)  A1 (w:100)
  65  *
  66  * If B is idle and only A0 and A1 are actively issuing IOs, as the two are
  67  * of equal weight, each gets 50% share.  If then B starts issuing IOs, B
  68  * gets 300/(100+300) or 75% share, and A0 and A1 equally splits the rest,
  69  * 12.5% each.  The distribution mechanism only cares about these flattened
  70  * shares.  They're called hweights (hierarchical weights) and always add
  71  * upto 1 (HWEIGHT_WHOLE).
  72  *
  73  * A given cgroup's vtime runs slower in inverse proportion to its hweight.
  74  * For example, with 12.5% weight, A0's time runs 8 times slower (100/12.5)
  75  * against the device vtime - an IO which takes 10ms on the underlying
  76  * device is considered to take 80ms on A0.
  77  *
  78  * This constitutes the basis of IO capacity distribution.  Each cgroup's
  79  * vtime is running at a rate determined by its hweight.  A cgroup tracks
  80  * the vtime consumed by past IOs and can issue a new IO iff doing so
  81  * wouldn't outrun the current device vtime.  Otherwise, the IO is
  82  * suspended until the vtime has progressed enough to cover it.
  83  *
  84  * 2-2. Vrate Adjustment
  85  *
  86  * It's unrealistic to expect the cost model to be perfect.  There are too
  87  * many devices and even on the same device the overall performance
  88  * fluctuates depending on numerous factors such as IO mixture and device
  89  * internal garbage collection.  The controller needs to adapt dynamically.
  90  *
  91  * This is achieved by adjusting the overall IO rate according to how busy
  92  * the device is.  If the device becomes overloaded, we're sending down too
  93  * many IOs and should generally slow down.  If there are waiting issuers
  94  * but the device isn't saturated, we're issuing too few and should
  95  * generally speed up.
  96  *
  97  * To slow down, we lower the vrate - the rate at which the device vtime
  98  * passes compared to the wall clock.  For example, if the vtime is running
  99  * at the vrate of 75%, all cgroups added up would only be able to issue
 100  * 750ms worth of IOs per second, and vice-versa for speeding up.
 101  *
 102  * Device business is determined using two criteria - rq wait and
 103  * completion latencies.
 104  *
 105  * When a device gets saturated, the on-device and then the request queues
 106  * fill up and a bio which is ready to be issued has to wait for a request
 107  * to become available.  When this delay becomes noticeable, it's a clear
 108  * indication that the device is saturated and we lower the vrate.  This
 109  * saturation signal is fairly conservative as it only triggers when both
 110  * hardware and software queues are filled up, and is used as the default
 111  * busy signal.
 112  *
 113  * As devices can have deep queues and be unfair in how the queued commands
 114  * are executed, soley depending on rq wait may not result in satisfactory
 115  * control quality.  For a better control quality, completion latency QoS
 116  * parameters can be configured so that the device is considered saturated
 117  * if N'th percentile completion latency rises above the set point.
 118  *
 119  * The completion latency requirements are a function of both the
 120  * underlying device characteristics and the desired IO latency quality of
 121  * service.  There is an inherent trade-off - the tighter the latency QoS,
 122  * the higher the bandwidth lossage.  Latency QoS is disabled by default
 123  * and can be set through /sys/fs/cgroup/io.cost.qos.
 124  *
 125  * 2-3. Work Conservation
 126  *
 127  * Imagine two cgroups A and B with equal weights.  A is issuing a small IO
 128  * periodically while B is sending out enough parallel IOs to saturate the
 129  * device on its own.  Let's say A's usage amounts to 100ms worth of IO
 130  * cost per second, i.e., 10% of the device capacity.  The naive
 131  * distribution of half and half would lead to 60% utilization of the
 132  * device, a significant reduction in the total amount of work done
 133  * compared to free-for-all competition.  This is too high a cost to pay
 134  * for IO control.
 135  *
 136  * To conserve the total amount of work done, we keep track of how much
 137  * each active cgroup is actually using and yield part of its weight if
 138  * there are other cgroups which can make use of it.  In the above case,
 139  * A's weight will be lowered so that it hovers above the actual usage and
 140  * B would be able to use the rest.
 141  *
 142  * As we don't want to penalize a cgroup for donating its weight, the
 143  * surplus weight adjustment factors in a margin and has an immediate
 144  * snapback mechanism in case the cgroup needs more IO vtime for itself.
 145  *
 146  * Note that adjusting down surplus weights has the same effects as
 147  * accelerating vtime for other cgroups and work conservation can also be
 148  * implemented by adjusting vrate dynamically.  However, squaring who can
 149  * donate and should take back how much requires hweight propagations
 150  * anyway making it easier to implement and understand as a separate
 151  * mechanism.
 152  *
 153  * 3. Monitoring
 154  *
 155  * Instead of debugfs or other clumsy monitoring mechanisms, this
 156  * controller uses a drgn based monitoring script -
 157  * tools/cgroup/iocost_monitor.py.  For details on drgn, please see
 158  * https://github.com/osandov/drgn.  The ouput looks like the following.
 159  *
 160  *  sdb RUN   per=300ms cur_per=234.218:v203.695 busy= +1 vrate= 62.12%
 161  *                 active      weight      hweight% inflt% dbt  delay usages%
 162  *  test/a              *    50/   50  33.33/ 33.33  27.65   2  0*041 033:033:033
 163  *  test/b              *   100/  100  66.67/ 66.67  17.56   0  0*000 066:079:077
 164  *
 165  * - per        : Timer period
 166  * - cur_per    : Internal wall and device vtime clock
 167  * - vrate      : Device virtual time rate against wall clock
 168  * - weight     : Surplus-adjusted and configured weights
 169  * - hweight    : Surplus-adjusted and configured hierarchical weights
 170  * - inflt      : The percentage of in-flight IO cost at the end of last period
 171  * - del_ms     : Deferred issuer delay induction level and duration
 172  * - usages     : Usage history
 173  */
 174
 175 #include <linux/kernel.h>
 176 #include <linux/module.h>
 177 #include <linux/timer.h>
 178 #include <linux/time64.h>
 179 #include <linux/parser.h>
 180 #include <linux/sched/signal.h>
 181 #include <linux/blk-cgroup.h>
 182 #include "blk-rq-qos.h"
 183 #include "blk-stat.h"
 184 #include "blk-wbt.h"
 185
 186 #ifdef CONFIG_TRACEPOINTS
 187
 188 /* copied from TRACE_CGROUP_PATH, see cgroup-internal.h */
 189 #define TRACE_IOCG_PATH_LEN 1024
 190 static DEFINE_SPINLOCK(trace_iocg_path_lock);
 191 static char trace_iocg_path[TRACE_IOCG_PATH_LEN];
 192
 193 #define TRACE_IOCG_PATH(type, iocg, ...)                                        \
 194         do {                                                                    \
 195                 unsigned long flags;                                            \
 196                 if (trace_iocost_##type##_enabled()) {                          \
 197                         spin_lock_irqsave(&trace_iocg_path_lock, flags);        \
 198                         cgroup_path(iocg_to_blkg(iocg)->blkcg->css.cgroup,      \
 199                                     trace_iocg_path, TRACE_IOCG_PATH_LEN);      \
 200                         trace_iocost_##type(iocg, trace_iocg_path,              \
 201                                               ##__VA_ARGS__);                   \
 202                         spin_unlock_irqrestore(&trace_iocg_path_lock, flags);   \
 203                 }                                                               \
 204         } while (0)
 205
 206 #else   /* CONFIG_TRACE_POINTS */
 207 #define TRACE_IOCG_PATH(type, iocg, ...)        do { } while (0)
 208 #endif  /* CONFIG_TRACE_POINTS */
 209
 210 enum {
 211         MILLION                 = 1000000,
 212
 213         /* timer period is calculated from latency requirements, bound it */
 214         MIN_PERIOD              = USEC_PER_MSEC,
 215         MAX_PERIOD              = USEC_PER_SEC,
 216
 217         /*
 218          * A cgroup's vtime can run 50% behind the device vtime, which
 219          * serves as its IO credit buffer.  Surplus weight adjustment is
 220          * immediately canceled if the vtime margin runs below 10%.
 221          */
 222         MARGIN_PCT              = 50,
 223         INUSE_MARGIN_PCT        = 10,
 224
 225         /* Have some play in waitq timer operations */
 226         WAITQ_TIMER_MARGIN_PCT  = 5,
 227
 228         /*
 229          * vtime can wrap well within a reasonable uptime when vrate is
 230          * consistently raised.  Don't trust recorded cgroup vtime if the
 231          * period counter indicates that it's older than 5mins.
 232          */
 233         VTIME_VALID_DUR         = 300 * USEC_PER_SEC,
 234
 235         /*
 236          * Remember the past three non-zero usages and use the max for
 237          * surplus calculation.  Three slots guarantee that we remember one
 238          * full period usage from the last active stretch even after
 239          * partial deactivation and re-activation periods.  Don't start
 240          * giving away weight before collecting two data points to prevent
 241          * hweight adjustments based on one partial activation period.
 242          */
 243         NR_USAGE_SLOTS          = 3,
 244         MIN_VALID_USAGES        = 2,
 245
 246         /* 1/64k is granular enough and can easily be handled w/ u32 */
 247         HWEIGHT_WHOLE           = 1 << 16,
 248
 249         /*
 250          * As vtime is used to calculate the cost of each IO, it needs to
 251          * be fairly high precision.  For example, it should be able to
 252          * represent the cost of a single page worth of discard with
 253          * suffificient accuracy.  At the same time, it should be able to
 254          * represent reasonably long enough durations to be useful and
 255          * convenient during operation.
 256          *
 257          * 1s worth of vtime is 2^37.  This gives us both sub-nanosecond
 258          * granularity and days of wrap-around time even at extreme vrates.
 259          */
 260         VTIME_PER_SEC_SHIFT     = 37,
 261         VTIME_PER_SEC           = 1LLU << VTIME_PER_SEC_SHIFT,
 262         VTIME_PER_USEC          = VTIME_PER_SEC / USEC_PER_SEC,
 263
 264         /* bound vrate adjustments within two orders of magnitude */
 265         VRATE_MIN_PPM           = 10000,        /* 1% */
 266         VRATE_MAX_PPM           = 100000000,    /* 10000% */
 267
 268         VRATE_MIN               = VTIME_PER_USEC * VRATE_MIN_PPM / MILLION,
 269         VRATE_CLAMP_ADJ_PCT     = 4,
 270
 271         /* if IOs end up waiting for requests, issue less */
 272         RQ_WAIT_BUSY_PCT        = 5,
 273
 274         /* unbusy hysterisis */
 275         UNBUSY_THR_PCT          = 75,
 276
 277         /* don't let cmds which take a very long time pin lagging for too long */
 278         MAX_LAGGING_PERIODS     = 10,
 279
 280         /*
 281          * If usage% * 1.25 + 2% is lower than hweight% by more than 3%,
 282          * donate the surplus.
 283          */
 284         SURPLUS_SCALE_PCT       = 125,                  /* * 125% */
 285         SURPLUS_SCALE_ABS       = HWEIGHT_WHOLE / 50,   /* + 2% */
 286         SURPLUS_MIN_ADJ_DELTA   = HWEIGHT_WHOLE / 33,   /* 3% */
 287
 288         /* switch iff the conditions are met for longer than this */
 289         AUTOP_CYCLE_NSEC        = 10LLU * NSEC_PER_SEC,
 290
 291         /*
 292          * Count IO size in 4k pages.  The 12bit shift helps keeping
 293          * size-proportional components of cost calculation in closer
 294          * numbers of digits to per-IO cost components.
 295          */
 296         IOC_PAGE_SHIFT          = 12,
 297         IOC_PAGE_SIZE           = 1 << IOC_PAGE_SHIFT,
 298         IOC_SECT_TO_PAGE_SHIFT  = IOC_PAGE_SHIFT - SECTOR_SHIFT,
 299
 300         /* if apart further than 16M, consider randio for linear model */
 301         LCOEF_RANDIO_PAGES      = 4096,
 302 };
 303
 304 enum ioc_running {
 305         IOC_IDLE,
 306         IOC_RUNNING,
 307         IOC_STOP,
 308 };
 309
 310 /* io.cost.qos controls including per-dev enable of the whole controller */
 311 enum {
 312         QOS_ENABLE,
 313         QOS_CTRL,
 314         NR_QOS_CTRL_PARAMS,
 315 };
 316
 317 /* io.cost.qos params */
 318 enum {
 319         QOS_RPPM,
 320         QOS_RLAT,
 321         QOS_WPPM,
 322         QOS_WLAT,
 323         QOS_MIN,
 324         QOS_MAX,
 325         NR_QOS_PARAMS,
 326 };
 327
 328 /* io.cost.model controls */
 329 enum {
 330         COST_CTRL,
 331         COST_MODEL,
 332         NR_COST_CTRL_PARAMS,
 333 };
 334
 335 /* builtin linear cost model coefficients */
 336 enum {
 337         I_LCOEF_RBPS,
 338         I_LCOEF_RSEQIOPS,
 339         I_LCOEF_RRANDIOPS,
 340         I_LCOEF_WBPS,
 341         I_LCOEF_WSEQIOPS,
 342         I_LCOEF_WRANDIOPS,
 343         NR_I_LCOEFS,
 344 };
 345
 346 enum {
 347         LCOEF_RPAGE,
 348         LCOEF_RSEQIO,
 349         LCOEF_RRANDIO,
 350         LCOEF_WPAGE,
 351         LCOEF_WSEQIO,
 352         LCOEF_WRANDIO,
 353         NR_LCOEFS,
 354 };
 355
 356 enum {
 357         AUTOP_INVALID,
 358         AUTOP_HDD,
 359         AUTOP_SSD_QD1,
 360         AUTOP_SSD_DFL,
 361         AUTOP_SSD_FAST,
 362 };
 363
 364 struct ioc_gq;
 365
 366 struct ioc_params {
 367         u32                             qos[NR_QOS_PARAMS];
 368         u64                             i_lcoefs[NR_I_LCOEFS];
 369         u64                             lcoefs[NR_LCOEFS];
 370         u32                             too_fast_vrate_pct;
 371         u32                             too_slow_vrate_pct;
 372 };
 373
 374 struct ioc_missed {
 375         u32                             nr_met;
 376         u32                             nr_missed;
 377         u32                             last_met;
 378         u32                             last_missed;
 379 };
 380
 381 struct ioc_pcpu_stat {
 382         struct ioc_missed               missed[2];
 383
 384         u64                             rq_wait_ns;
 385         u64                             last_rq_wait_ns;
 386 };
 387
 388 /* per device */
 389 struct ioc {
 390         struct rq_qos                   rqos;
 391
 392         bool                            enabled;
 393
 394         struct ioc_params               params;
 395         u32                             period_us;
 396         u32                             margin_us;
 397         u64                             vrate_min;
 398         u64                             vrate_max;
 399
 400         spinlock_t                      lock;
 401         struct timer_list               timer;
 402         struct list_head                active_iocgs;   /* active cgroups */
 403         struct ioc_pcpu_stat __percpu   *pcpu_stat;
 404
 405         enum ioc_running                running;
 406         atomic64_t                      vtime_rate;
 407
 408         seqcount_t                      period_seqcount;
 409         u32                             period_at;      /* wallclock starttime */
 410         u64                             period_at_vtime; /* vtime starttime */
 411
 412         atomic64_t                      cur_period;     /* inc'd each period */
 413         int                             busy_level;     /* saturation history */
 414
 415         u64                             inuse_margin_vtime;
 416         bool                            weights_updated;
 417         atomic_t                        hweight_gen;    /* for lazy hweights */
 418
 419         u64                             autop_too_fast_at;
 420         u64                             autop_too_slow_at;
 421         int                             autop_idx;
 422         bool                            user_qos_params:1;
 423         bool                            user_cost_model:1;
 424 };
 425
 426 /* per device-cgroup pair */
 427 struct ioc_gq {
 428         struct blkg_policy_data         pd;
 429         struct ioc                      *ioc;
 430
 431         /*
 432          * A iocg can get its weight from two sources - an explicit
 433          * per-device-cgroup configuration or the default weight of the
 434          * cgroup.  `cfg_weight` is the explicit per-device-cgroup
 435          * configuration.  `weight` is the effective considering both
 436          * sources.
 437          *
 438          * When an idle cgroup becomes active its `active` goes from 0 to
 439          * `weight`.  `inuse` is the surplus adjusted active weight.
 440          * `active` and `inuse` are used to calculate `hweight_active` and
 441          * `hweight_inuse`.
 442          *
 443          * `last_inuse` remembers `inuse` while an iocg is idle to persist
 444          * surplus adjustments.
 445          */
 446         u32                             cfg_weight;
 447         u32                             weight;
 448         u32                             active;
 449         u32                             inuse;
 450         u32                             last_inuse;
 451
 452         sector_t                        cursor;         /* to detect randio */
 453
 454         /*
 455          * `vtime` is this iocg's vtime cursor which progresses as IOs are
 456          * issued.  If lagging behind device vtime, the delta represents
 457          * the currently available IO budget.  If runnning ahead, the
 458          * overage.
 459          *
 460          * `vtime_done` is the same but progressed on completion rather
 461          * than issue.  The delta behind `vtime` represents the cost of
 462          * currently in-flight IOs.
 463          *
 464          * `last_vtime` is used to remember `vtime` at the end of the last
 465          * period to calculate utilization.
 466          */
 467         atomic64_t                      vtime;
 468         atomic64_t                      done_vtime;
 469         atomic64_t                      abs_vdebt;
 470         u64                             last_vtime;
 471
 472         /*
 473          * The period this iocg was last active in.  Used for deactivation
 474          * and invalidating `vtime`.
 475          */
 476         atomic64_t                      active_period;
 477         struct list_head                active_list;
 478
 479         /* see __propagate_active_weight() and current_hweight() for details */
 480         u64                             child_active_sum;
 481         u64                             child_inuse_sum;
 482         int                             hweight_gen;
 483         u32                             hweight_active;
 484         u32                             hweight_inuse;
 485         bool                            has_surplus;
 486
 487         struct wait_queue_head          waitq;
 488         struct hrtimer                  waitq_timer;
 489         struct hrtimer                  delay_timer;
 490
 491         /* usage is recorded as fractions of HWEIGHT_WHOLE */
 492         int                             usage_idx;
 493         u32                             usages[NR_USAGE_SLOTS];
 494
 495         /* this iocg's depth in the hierarchy and ancestors including self */
 496         int                             level;
 497         struct ioc_gq                   *ancestors[];
 498 };
 499
 500 /* per cgroup */
 501 struct ioc_cgrp {
 502         struct blkcg_policy_data        cpd;
 503         unsigned int                    dfl_weight;
 504 };
 505
 506 struct ioc_now {
 507         u64                             now_ns;
 508         u32                             now;
 509         u64                             vnow;
 510         u64                             vrate;
 511 };
 512
 513 struct iocg_wait {
 514         struct wait_queue_entry         wait;
 515         struct bio                      *bio;
 516         u64                             abs_cost;
 517         bool                            committed;
 518 };
 519
 520 struct iocg_wake_ctx {
 521         struct ioc_gq                   *iocg;
 522         u32                             hw_inuse;
 523         s64                             vbudget;
 524 };
 525
 526 static const struct ioc_params autop[] = {
 527         [AUTOP_HDD] = {
 528                 .qos                            = {
 529                         [QOS_RLAT]              =        250000, /* 250ms */
 530                         [QOS_WLAT]              =        250000,
 531                         [QOS_MIN]               = VRATE_MIN_PPM,
 532                         [QOS_MAX]               = VRATE_MAX_PPM,
 533                 },
 534                 .i_lcoefs                       = {
 535                         [I_LCOEF_RBPS]          =     174019176,
 536                         [I_LCOEF_RSEQIOPS]      =         41708,
 537                         [I_LCOEF_RRANDIOPS]     =           370,
 538                         [I_LCOEF_WBPS]          =     178075866,
 539                         [I_LCOEF_WSEQIOPS]      =         42705,
 540                         [I_LCOEF_WRANDIOPS]     =           378,
 541                 },
 542         },
 543         [AUTOP_SSD_QD1] = {
 544                 .qos                            = {
 545                         [QOS_RLAT]              =         25000, /* 25ms */
 546                         [QOS_WLAT]              =         25000,
 547                         [QOS_MIN]               = VRATE_MIN_PPM,
 548                         [QOS_MAX]               = VRATE_MAX_PPM,
 549                 },
 550                 .i_lcoefs                       = {
 551                         [I_LCOEF_RBPS]          =     245855193,
 552                         [I_LCOEF_RSEQIOPS]      =         61575,
 553                         [I_LCOEF_RRANDIOPS]     =          6946,
 554                         [I_LCOEF_WBPS]          =     141365009,
 555                         [I_LCOEF_WSEQIOPS]      =         33716,
 556                         [I_LCOEF_WRANDIOPS]     =         26796,
 557                 },
 558         },
 559         [AUTOP_SSD_DFL] = {
 560                 .qos                            = {
 561                         [QOS_RLAT]              =         25000, /* 25ms */
 562                         [QOS_WLAT]              =         25000,
 563                         [QOS_MIN]               = VRATE_MIN_PPM,
 564                         [QOS_MAX]               = VRATE_MAX_PPM,
 565                 },
 566                 .i_lcoefs                       = {
 567                         [I_LCOEF_RBPS]          =     488636629,
 568                         [I_LCOEF_RSEQIOPS]      =          8932,
 569                         [I_LCOEF_RRANDIOPS]     =          8518,
 570                         [I_LCOEF_WBPS]          =     427891549,
 571                         [I_LCOEF_WSEQIOPS]      =         28755,
 572                         [I_LCOEF_WRANDIOPS]     =         21940,
 573                 },
 574                 .too_fast_vrate_pct             =           500,
 575         },
 576         [AUTOP_SSD_FAST] = {
 577                 .qos                            = {
 578                         [QOS_RLAT]              =          5000, /* 5ms */
 579                         [QOS_WLAT]              =          5000,
 580                         [QOS_MIN]               = VRATE_MIN_PPM,
 581                         [QOS_MAX]               = VRATE_MAX_PPM,
 582                 },
 583                 .i_lcoefs                       = {
 584                         [I_LCOEF_RBPS]          =    3102524156LLU,
 585                         [I_LCOEF_RSEQIOPS]      =        724816,
 586                         [I_LCOEF_RRANDIOPS]     =        778122,
 587                         [I_LCOEF_WBPS]          =    1742780862LLU,
 588                         [I_LCOEF_WSEQIOPS]      =        425702,
 589                         [I_LCOEF_WRANDIOPS]     =        443193,
 590                 },
 591                 .too_slow_vrate_pct             =            10,
 592         },
 593 };
 594
 595 /*
 596  * vrate adjust percentages indexed by ioc->busy_level.  We adjust up on
 597  * vtime credit shortage and down on device saturation.
 598  */
 599 static u32 vrate_adj_pct[] =
 600         { 0, 0, 0, 0,
 601           1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
 602           2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 603           4, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 8, 8, 8, 8, 16 };
 604
 605 static struct blkcg_policy blkcg_policy_iocost;
 606
 607 /* accessors and helpers */
 608 static struct ioc *rqos_to_ioc(struct rq_qos *rqos)
 609 {
 610         return container_of(rqos, struct ioc, rqos);
 611 }
 612
 613 static struct ioc *q_to_ioc(struct request_queue *q)
 614 {
 615         return rqos_to_ioc(rq_qos_id(q, RQ_QOS_COST));
 616 }
 617
 618 static const char *q_name(struct request_queue *q)
 619 {
 620         if (test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
 621                 return kobject_name(q->kobj.parent);
 622         else
 623                 return "<unknown>";
 624 }
 625
 626 static const char __maybe_unused *ioc_name(struct ioc *ioc)
 627 {
 628         return q_name(ioc->rqos.q);
 629 }
 630
 631 static struct ioc_gq *pd_to_iocg(struct blkg_policy_data *pd)
 632 {
 633         return pd ? container_of(pd, struct ioc_gq, pd) : NULL;
 634 }
 635
 636 static struct ioc_gq *blkg_to_iocg(struct blkcg_gq *blkg)
 637 {
 638         return pd_to_iocg(blkg_to_pd(blkg, &blkcg_policy_iocost));
 639 }
 640
 641 static struct blkcg_gq *iocg_to_blkg(struct ioc_gq *iocg)
 642 {
 643         return pd_to_blkg(&iocg->pd);
 644 }
 645
 646 static struct ioc_cgrp *blkcg_to_iocc(struct blkcg *blkcg)
 647 {
 648         return container_of(blkcg_to_cpd(blkcg, &blkcg_policy_iocost),
 649                             struct ioc_cgrp, cpd);
 650 }
 651
 652 /*
 653  * Scale @abs_cost to the inverse of @hw_inuse.  The lower the hierarchical
 654  * weight, the more expensive each IO.  Must round up.
 655  */
 656 static u64 abs_cost_to_cost(u64 abs_cost, u32 hw_inuse)
 657 {
 658         return DIV64_U64_ROUND_UP(abs_cost * HWEIGHT_WHOLE, hw_inuse);
 659 }
 660
 661 /*
 662  * The inverse of abs_cost_to_cost().  Must round up.
 663  */
 664 static u64 cost_to_abs_cost(u64 cost, u32 hw_inuse)
 665 {
 666         return DIV64_U64_ROUND_UP(cost * hw_inuse, HWEIGHT_WHOLE);
 667 }
 668
 669 static void iocg_commit_bio(struct ioc_gq *iocg, struct bio *bio, u64 cost)
 670 {
 671         bio->bi_iocost_cost = cost;
 672         atomic64_add(cost, &iocg->vtime);
 673 }
 674
 675 #define CREATE_TRACE_POINTS
 676 #include <trace/events/iocost.h>
 677
 678 /* latency Qos params changed, update period_us and all the dependent params */
 679 static void ioc_refresh_period_us(struct ioc *ioc)
 680 {
 681         u32 ppm, lat, multi, period_us;
 682
 683         lockdep_assert_held(&ioc->lock);
 684
 685         /* pick the higher latency target */
 686         if (ioc->params.qos[QOS_RLAT] >= ioc->params.qos[QOS_WLAT]) {
 687                 ppm = ioc->params.qos[QOS_RPPM];
 688                 lat = ioc->params.qos[QOS_RLAT];
 689         } else {
 690                 ppm = ioc->params.qos[QOS_WPPM];
 691                 lat = ioc->params.qos[QOS_WLAT];
 692         }
 693
 694         /*
 695          * We want the period to be long enough to contain a healthy number
 696          * of IOs while short enough for granular control.  Define it as a
 697          * multiple of the latency target.  Ideally, the multiplier should
 698          * be scaled according to the percentile so that it would nominally
 699          * contain a certain number of requests.  Let's be simpler and
 700          * scale it linearly so that it's 2x >= pct(90) and 10x at pct(50).
 701          */
 702         if (ppm)
 703                 multi = max_t(u32, (MILLION - ppm) / 50000, 2);
 704         else
 705                 multi = 2;
 706         period_us = multi * lat;
 707         period_us = clamp_t(u32, period_us, MIN_PERIOD, MAX_PERIOD);
 708
 709         /* calculate dependent params */
 710         ioc->period_us = period_us;
 711         ioc->margin_us = period_us * MARGIN_PCT / 100;
 712         ioc->inuse_margin_vtime = DIV64_U64_ROUND_UP(
 713                         period_us * VTIME_PER_USEC * INUSE_MARGIN_PCT, 100);
 714 }
 715
 716 static int ioc_autop_idx(struct ioc *ioc)
 717 {
 718         int idx = ioc->autop_idx;
 719         const struct ioc_params *p = &autop[idx];
 720         u32 vrate_pct;
 721         u64 now_ns;
 722
 723         /* rotational? */
 724         if (!blk_queue_nonrot(ioc->rqos.q))
 725                 return AUTOP_HDD;
 726
 727         /* handle SATA SSDs w/ broken NCQ */
 728         if (blk_queue_depth(ioc->rqos.q) == 1)
 729                 return AUTOP_SSD_QD1;
 730
 731         /* use one of the normal ssd sets */
 732         if (idx < AUTOP_SSD_DFL)
 733                 return AUTOP_SSD_DFL;
 734
 735         /* if user is overriding anything, maintain what was there */
 736         if (ioc->user_qos_params || ioc->user_cost_model)
 737                 return idx;
 738
 739         /* step up/down based on the vrate */
 740         vrate_pct = div64_u64(atomic64_read(&ioc->vtime_rate) * 100,
 741                               VTIME_PER_USEC);
 742         now_ns = ktime_get_ns();
 743
 744         if (p->too_fast_vrate_pct && p->too_fast_vrate_pct <= vrate_pct) {
 745                 if (!ioc->autop_too_fast_at)
 746                         ioc->autop_too_fast_at = now_ns;
 747                 if (now_ns - ioc->autop_too_fast_at >= AUTOP_CYCLE_NSEC)
 748                         return idx + 1;
 749         } else {
 750                 ioc->autop_too_fast_at = 0;
 751         }
 752
 753         if (p->too_slow_vrate_pct && p->too_slow_vrate_pct >= vrate_pct) {
 754                 if (!ioc->autop_too_slow_at)
 755                         ioc->autop_too_slow_at = now_ns;
 756                 if (now_ns - ioc->autop_too_slow_at >= AUTOP_CYCLE_NSEC)
 757                         return idx - 1;
 758         } else {
 759                 ioc->autop_too_slow_at = 0;
 760         }
 761
 762         return idx;
 763 }
 764
 765 /*
 766  * Take the followings as input
 767  *
 768  *  @bps        maximum sequential throughput
 769  *  @seqiops    maximum sequential 4k iops
 770  *  @randiops   maximum random 4k iops
 771  *
 772  * and calculate the linear model cost coefficients.
 773  *
 774  *  *@page      per-page cost           1s / (@bps / 4096)
 775  *  *@seqio     base cost of a seq IO   max((1s / @seqiops) - *@page, 0)
 776  *  @randiops   base cost of a rand IO  max((1s / @randiops) - *@page, 0)
 777  */
 778 static void calc_lcoefs(u64 bps, u64 seqiops, u64 randiops,
 779                         u64 *page, u64 *seqio, u64 *randio)
 780 {
 781         u64 v;
 782
 783         *page = *seqio = *randio = 0;
 784
 785         if (bps)
 786                 *page = DIV64_U64_ROUND_UP(VTIME_PER_SEC,
 787                                            DIV_ROUND_UP_ULL(bps, IOC_PAGE_SIZE));
 788
 789         if (seqiops) {
 790                 v = DIV64_U64_ROUND_UP(VTIME_PER_SEC, seqiops);
 791                 if (v > *page)
 792                         *seqio = v - *page;
 793         }
 794
 795         if (randiops) {
 796                 v = DIV64_U64_ROUND_UP(VTIME_PER_SEC, randiops);
 797                 if (v > *page)
 798                         *randio = v - *page;
 799         }
 800 }
 801
 802 static void ioc_refresh_lcoefs(struct ioc *ioc)
 803 {
 804         u64 *u = ioc->params.i_lcoefs;
 805         u64 *c = ioc->params.lcoefs;
 806
 807         calc_lcoefs(u[I_LCOEF_RBPS], u[I_LCOEF_RSEQIOPS], u[I_LCOEF_RRANDIOPS],
 808                     &c[LCOEF_RPAGE], &c[LCOEF_RSEQIO], &c[LCOEF_RRANDIO]);
 809         calc_lcoefs(u[I_LCOEF_WBPS], u[I_LCOEF_WSEQIOPS], u[I_LCOEF_WRANDIOPS],
 810                     &c[LCOEF_WPAGE], &c[LCOEF_WSEQIO], &c[LCOEF_WRANDIO]);
 811 }
 812
 813 static bool ioc_refresh_params(struct ioc *ioc, bool force)
 814 {
 815         const struct ioc_params *p;
 816         int idx;
 817
 818         lockdep_assert_held(&ioc->lock);
 819
 820         idx = ioc_autop_idx(ioc);
 821         p = &autop[idx];
 822
 823         if (idx == ioc->autop_idx && !force)
 824                 return false;
 825
 826         if (idx != ioc->autop_idx)
 827                 atomic64_set(&ioc->vtime_rate, VTIME_PER_USEC);
 828
 829         ioc->autop_idx = idx;
 830         ioc->autop_too_fast_at = 0;
 831         ioc->autop_too_slow_at = 0;
 832
 833         if (!ioc->user_qos_params)
 834                 memcpy(ioc->params.qos, p->qos, sizeof(p->qos));
 835         if (!ioc->user_cost_model)
 836                 memcpy(ioc->params.i_lcoefs, p->i_lcoefs, sizeof(p->i_lcoefs));
 837
 838         ioc_refresh_period_us(ioc);
 839         ioc_refresh_lcoefs(ioc);
 840
 841         ioc->vrate_min = DIV64_U64_ROUND_UP((u64)ioc->params.qos[QOS_MIN] *
 842                                             VTIME_PER_USEC, MILLION);
 843         ioc->vrate_max = div64_u64((u64)ioc->params.qos[QOS_MAX] *
 844                                    VTIME_PER_USEC, MILLION);
 845
 846         return true;
 847 }
 848
 849 /* take a snapshot of the current [v]time and vrate */
 850 static void ioc_now(struct ioc *ioc, struct ioc_now *now)
 851 {
 852         unsigned seq;
 853
 854         now->now_ns = ktime_get();
 855         now->now = ktime_to_us(now->now_ns);
 856         now->vrate = atomic64_read(&ioc->vtime_rate);
 857
 858         /*
 859          * The current vtime is
 860          *
 861          *   vtime at period start + (wallclock time since the start) * vrate
 862          *
 863          * As a consistent snapshot of `period_at_vtime` and `period_at` is
 864          * needed, they're seqcount protected.
 865          */
 866         do {
 867                 seq = read_seqcount_begin(&ioc->period_seqcount);
 868                 now->vnow = ioc->period_at_vtime +
 869                         (now->now - ioc->period_at) * now->vrate;
 870         } while (read_seqcount_retry(&ioc->period_seqcount, seq));
 871 }
 872
 873 static void ioc_start_period(struct ioc *ioc, struct ioc_now *now)
 874 {
 875         lockdep_assert_held(&ioc->lock);
 876         WARN_ON_ONCE(ioc->running != IOC_RUNNING);
 877
 878         write_seqcount_begin(&ioc->period_seqcount);
 879         ioc->period_at = now->now;
 880         ioc->period_at_vtime = now->vnow;
 881         write_seqcount_end(&ioc->period_seqcount);
 882
 883         ioc->timer.expires = jiffies + usecs_to_jiffies(ioc->period_us);
 884         add_timer(&ioc->timer);
 885 }
 886
 887 /*
 888  * Update @iocg's `active` and `inuse` to @active and @inuse, update level
 889  * weight sums and propagate upwards accordingly.
 890  */
 891 static void __propagate_active_weight(struct ioc_gq *iocg, u32 active, u32 inuse)
 892 {
 893         struct ioc *ioc = iocg->ioc;
 894         int lvl;
 895
 896         lockdep_assert_held(&ioc->lock);
 897
 898         inuse = min(active, inuse);
 899
 900         for (lvl = iocg->level - 1; lvl >= 0; lvl--) {
 901                 struct ioc_gq *parent = iocg->ancestors[lvl];
 902                 struct ioc_gq *child = iocg->ancestors[lvl + 1];
 903                 u32 parent_active = 0, parent_inuse = 0;
 904
 905                 /* update the level sums */
 906                 parent->child_active_sum += (s32)(active - child->active);
 907                 parent->child_inuse_sum += (s32)(inuse - child->inuse);
 908                 /* apply the udpates */
 909                 child->active = active;
 910                 child->inuse = inuse;
 911
 912                 /*
 913                  * The delta between inuse and active sums indicates that
 914                  * that much of weight is being given away.  Parent's inuse
 915                  * and active should reflect the ratio.
 916                  */
 917                 if (parent->child_active_sum) {
 918                         parent_active = parent->weight;
 919                         parent_inuse = DIV64_U64_ROUND_UP(
 920                                 parent_active * parent->child_inuse_sum,
 921                                 parent->child_active_sum);
 922                 }
 923
 924                 /* do we need to keep walking up? */
 925                 if (parent_active == parent->active &&
 926                     parent_inuse == parent->inuse)
 927                         break;
 928
 929                 active = parent_active;
 930                 inuse = parent_inuse;
 931         }
 932
 933         ioc->weights_updated = true;
 934 }
 935
 936 static void commit_active_weights(struct ioc *ioc)
 937 {
 938         lockdep_assert_held(&ioc->lock);
 939
 940         if (ioc->weights_updated) {
 941                 /* paired with rmb in current_hweight(), see there */
 942                 smp_wmb();
 943                 atomic_inc(&ioc->hweight_gen);
 944                 ioc->weights_updated = false;
 945         }
 946 }
 947
 948 static void propagate_active_weight(struct ioc_gq *iocg, u32 active, u32 inuse)
 949 {
 950         __propagate_active_weight(iocg, active, inuse);
 951         commit_active_weights(iocg->ioc);
 952 }
 953
 954 static void current_hweight(struct ioc_gq *iocg, u32 *hw_activep, u32 *hw_inusep)
 955 {
 956         struct ioc *ioc = iocg->ioc;
 957         int lvl;
 958         u32 hwa, hwi;
 959         int ioc_gen;
 960
 961         /* hot path - if uptodate, use cached */
 962         ioc_gen = atomic_read(&ioc->hweight_gen);
 963         if (ioc_gen == iocg->hweight_gen)
 964                 goto out;
 965
 966         /*
 967          * Paired with wmb in commit_active_weights().  If we saw the
 968          * updated hweight_gen, all the weight updates from
 969          * __propagate_active_weight() are visible too.
 970          *
 971          * We can race with weight updates during calculation and get it
 972          * wrong.  However, hweight_gen would have changed and a future
 973          * reader will recalculate and we're guaranteed to discard the
 974          * wrong result soon.
 975          */
 976         smp_rmb();
 977
 978         hwa = hwi = HWEIGHT_WHOLE;
 979         for (lvl = 0; lvl <= iocg->level - 1; lvl++) {
 980                 struct ioc_gq *parent = iocg->ancestors[lvl];
 981                 struct ioc_gq *child = iocg->ancestors[lvl + 1];
 982                 u32 active_sum = READ_ONCE(parent->child_active_sum);
 983                 u32 inuse_sum = READ_ONCE(parent->child_inuse_sum);
 984                 u32 active = READ_ONCE(child->active);
 985                 u32 inuse = READ_ONCE(child->inuse);
 986
 987                 /* we can race with deactivations and either may read as zero */
 988                 if (!active_sum || !inuse_sum)
 989                         continue;
 990
 991                 active_sum = max(active, active_sum);
 992                 hwa = hwa * active / active_sum;        /* max 16bits * 10000 */
 993
 994                 inuse_sum = max(inuse, inuse_sum);
 995                 hwi = hwi * inuse / inuse_sum;          /* max 16bits * 10000 */
 996         }
 997
 998         iocg->hweight_active = max_t(u32, hwa, 1);
 999         iocg->hweight_inuse = max_t(u32, hwi, 1);
1000         iocg->hweight_gen = ioc_gen;
1001 out:
1002         if (hw_activep)
1003                 *hw_activep = iocg->hweight_active;
1004         if (hw_inusep)
1005                 *hw_inusep = iocg->hweight_inuse;
1006 }
1007
1008 static void weight_updated(struct ioc_gq *iocg)
1009 {
1010         struct ioc *ioc = iocg->ioc;
1011         struct blkcg_gq *blkg = iocg_to_blkg(iocg);
1012         struct ioc_cgrp *iocc = blkcg_to_iocc(blkg->blkcg);
1013         u32 weight;
1014
1015         lockdep_assert_held(&ioc->lock);
1016
1017         weight = iocg->cfg_weight ?: iocc->dfl_weight;
1018         if (weight != iocg->weight && iocg->active)
1019                 propagate_active_weight(iocg, weight,
1020                         DIV64_U64_ROUND_UP(iocg->inuse * weight, iocg->weight));
1021         iocg->weight = weight;
1022 }
1023
1024 static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)
1025 {
1026         struct ioc *ioc = iocg->ioc;
1027         u64 last_period, cur_period, max_period_delta;
1028         u64 vtime, vmargin, vmin;
1029         int i;
1030
1031         /*
1032          * If seem to be already active, just update the stamp to tell the
1033          * timer that we're still active.  We don't mind occassional races.
1034          */
1035         if (!list_empty(&iocg->active_list)) {
1036                 ioc_now(ioc, now);
1037                 cur_period = atomic64_read(&ioc->cur_period);
1038                 if (atomic64_read(&iocg->active_period) != cur_period)
1039                         atomic64_set(&iocg->active_period, cur_period);
1040                 return true;
1041         }
1042
1043         /* racy check on internal node IOs, treat as root level IOs */
1044         if (iocg->child_active_sum)
1045                 return false;
1046
1047         spin_lock_irq(&ioc->lock);
1048
1049         ioc_now(ioc, now);
1050
1051         /* update period */
1052         cur_period = atomic64_read(&ioc->cur_period);
1053         last_period = atomic64_read(&iocg->active_period);
1054         atomic64_set(&iocg->active_period, cur_period);
1055
1056         /* already activated or breaking leaf-only constraint? */
1057         if (!list_empty(&iocg->active_list))
1058                 goto succeed_unlock;
1059         for (i = iocg->level - 1; i > 0; i--)
1060                 if (!list_empty(&iocg->ancestors[i]->active_list))
1061                         goto fail_unlock;
1062
1063         if (iocg->child_active_sum)
1064                 goto fail_unlock;
1065
1066         /*
1067          * vtime may wrap when vrate is raised substantially due to
1068          * underestimated IO costs.  Look at the period and ignore its
1069          * vtime if the iocg has been idle for too long.  Also, cap the
1070          * budget it can start with to the margin.
1071          */
1072         max_period_delta = DIV64_U64_ROUND_UP(VTIME_VALID_DUR, ioc->period_us);
1073         vtime = atomic64_read(&iocg->vtime);
1074         vmargin = ioc->margin_us * now->vrate;
1075         vmin = now->vnow - vmargin;
1076
1077         if (last_period + max_period_delta < cur_period ||
1078             time_before64(vtime, vmin)) {
1079                 atomic64_add(vmin - vtime, &iocg->vtime);
1080                 atomic64_add(vmin - vtime, &iocg->done_vtime);
1081                 vtime = vmin;
1082         }
1083
1084         /*
1085          * Activate, propagate weight and start period timer if not
1086          * running.  Reset hweight_gen to avoid accidental match from
1087          * wrapping.
1088          */
1089         iocg->hweight_gen = atomic_read(&ioc->hweight_gen) - 1;
1090         list_add(&iocg->active_list, &ioc->active_iocgs);
1091         propagate_active_weight(iocg, iocg->weight,
1092                                 iocg->last_inuse ?: iocg->weight);
1093
1094         TRACE_IOCG_PATH(iocg_activate, iocg, now,
1095                         last_period, cur_period, vtime);
1096
1097         iocg->last_vtime = vtime;
1098
1099         if (ioc->running == IOC_IDLE) {
1100                 ioc->running = IOC_RUNNING;
1101                 ioc_start_period(ioc, now);
1102         }
1103
1104 succeed_unlock:
1105         spin_unlock_irq(&ioc->lock);
1106         return true;
1107
1108 fail_unlock:
1109         spin_unlock_irq(&ioc->lock);
1110         return false;
1111 }
1112
1113 static int iocg_wake_fn(struct wait_queue_entry *wq_entry, unsigned mode,
1114                         int flags, void *key)
1115 {
1116         struct iocg_wait *wait = container_of(wq_entry, struct iocg_wait, wait);
1117         struct iocg_wake_ctx *ctx = (struct iocg_wake_ctx *)key;
1118         u64 cost = abs_cost_to_cost(wait->abs_cost, ctx->hw_inuse);
1119
1120         ctx->vbudget -= cost;
1121
1122         if (ctx->vbudget < 0)
1123                 return -1;
1124
1125         iocg_commit_bio(ctx->iocg, wait->bio, cost);
1126
1127         /*
1128          * autoremove_wake_function() removes the wait entry only when it
1129          * actually changed the task state.  We want the wait always
1130          * removed.  Remove explicitly and use default_wake_function().
1131          */
1132         list_del_init(&wq_entry->entry);
1133         wait->committed = true;
1134
1135         default_wake_function(wq_entry, mode, flags, key);
1136         return 0;
1137 }
1138
1139 static void iocg_kick_waitq(struct ioc_gq *iocg, struct ioc_now *now)
1140 {
1141         struct ioc *ioc = iocg->ioc;
1142         struct iocg_wake_ctx ctx = { .iocg = iocg };
1143         u64 margin_ns = (u64)(ioc->period_us *
1144                               WAITQ_TIMER_MARGIN_PCT / 100) * NSEC_PER_USEC;
1145         u64 abs_vdebt, vdebt, vshortage, expires, oexpires;
1146         s64 vbudget;
1147         u32 hw_inuse;
1148
1149         lockdep_assert_held(&iocg->waitq.lock);
1150
1151         current_hweight(iocg, NULL, &hw_inuse);
1152         vbudget = now->vnow - atomic64_read(&iocg->vtime);
1153
1154         /* pay off debt */
1155         abs_vdebt = atomic64_read(&iocg->abs_vdebt);
1156         vdebt = abs_cost_to_cost(abs_vdebt, hw_inuse);
1157         if (vdebt && vbudget > 0) {
1158                 u64 delta = min_t(u64, vbudget, vdebt);
1159                 u64 abs_delta = min(cost_to_abs_cost(delta, hw_inuse),
1160                                     abs_vdebt);
1161
1162                 atomic64_add(delta, &iocg->vtime);
1163                 atomic64_add(delta, &iocg->done_vtime);
1164                 atomic64_sub(abs_delta, &iocg->abs_vdebt);
1165                 if (WARN_ON_ONCE(atomic64_read(&iocg->abs_vdebt) < 0))
1166                         atomic64_set(&iocg->abs_vdebt, 0);
1167         }
1168
1169         /*
1170          * Wake up the ones which are due and see how much vtime we'll need
1171          * for the next one.
1172          */
1173         ctx.hw_inuse = hw_inuse;
1174         ctx.vbudget = vbudget - vdebt;
1175         __wake_up_locked_key(&iocg->waitq, TASK_NORMAL, &ctx);
1176         if (!waitqueue_active(&iocg->waitq))
1177                 return;
1178         if (WARN_ON_ONCE(ctx.vbudget >= 0))
1179                 return;
1180
1181         /* determine next wakeup, add a quarter margin to guarantee chunking */
1182         vshortage = -ctx.vbudget;
1183         expires = now->now_ns +
1184                 DIV64_U64_ROUND_UP(vshortage, now->vrate) * NSEC_PER_USEC;
1185         expires += margin_ns / 4;
1186
1187         /* if already active and close enough, don't bother */
1188         oexpires = ktime_to_ns(hrtimer_get_softexpires(&iocg->waitq_timer));
1189         if (hrtimer_is_queued(&iocg->waitq_timer) &&
1190             abs(oexpires - expires) <= margin_ns / 4)
1191                 return;
1192
1193         hrtimer_start_range_ns(&iocg->waitq_timer, ns_to_ktime(expires),
1194                                margin_ns / 4, HRTIMER_MODE_ABS);
1195 }
1196
1197 static enum hrtimer_restart iocg_waitq_timer_fn(struct hrtimer *timer)
1198 {
1199         struct ioc_gq *iocg = container_of(timer, struct ioc_gq, waitq_timer);
1200         struct ioc_now now;
1201         unsigned long flags;
1202
1203         ioc_now(iocg->ioc, &now);
1204
1205         spin_lock_irqsave(&iocg->waitq.lock, flags);
1206         iocg_kick_waitq(iocg, &now);
1207         spin_unlock_irqrestore(&iocg->waitq.lock, flags);
1208
1209         return HRTIMER_NORESTART;
1210 }
1211
1212 static bool iocg_kick_delay(struct ioc_gq *iocg, struct ioc_now *now, u64 cost)
1213 {
1214         struct ioc *ioc = iocg->ioc;
1215         struct blkcg_gq *blkg = iocg_to_blkg(iocg);
1216         u64 vtime = atomic64_read(&iocg->vtime);
1217         u64 vmargin = ioc->margin_us * now->vrate;
1218         u64 margin_ns = ioc->margin_us * NSEC_PER_USEC;
1219         u64 expires, oexpires;
1220         u32 hw_inuse;
1221
1222         /* debt-adjust vtime */
1223         current_hweight(iocg, NULL, &hw_inuse);
1224         vtime += abs_cost_to_cost(atomic64_read(&iocg->abs_vdebt), hw_inuse);
1225
1226         /* clear or maintain depending on the overage */
1227         if (time_before_eq64(vtime, now->vnow)) {
1228                 blkcg_clear_delay(blkg);
1229                 return false;
1230         }
1231         if (!atomic_read(&blkg->use_delay) &&
1232             time_before_eq64(vtime, now->vnow + vmargin))
1233                 return false;
1234
1235         /* use delay */
1236         if (cost) {
1237                 u64 cost_ns = DIV64_U64_ROUND_UP(cost * NSEC_PER_USEC,
1238                                                  now->vrate);
1239                 blkcg_add_delay(blkg, now->now_ns, cost_ns);
1240         }
1241         blkcg_use_delay(blkg);
1242
1243         expires = now->now_ns + DIV64_U64_ROUND_UP(vtime - now->vnow,
1244                                                    now->vrate) * NSEC_PER_USEC;
1245
1246         /* if already active and close enough, don't bother */
1247         oexpires = ktime_to_ns(hrtimer_get_softexpires(&iocg->delay_timer));
1248         if (hrtimer_is_queued(&iocg->delay_timer) &&
1249             abs(oexpires - expires) <= margin_ns / 4)
1250                 return true;
1251
1252         hrtimer_start_range_ns(&iocg->delay_timer, ns_to_ktime(expires),
1253                                margin_ns / 4, HRTIMER_MODE_ABS);
1254         return true;
1255 }
1256
1257 static enum hrtimer_restart iocg_delay_timer_fn(struct hrtimer *timer)
1258 {
1259         struct ioc_gq *iocg = container_of(timer, struct ioc_gq, delay_timer);
1260         struct ioc_now now;
1261
1262         ioc_now(iocg->ioc, &now);
1263         iocg_kick_delay(iocg, &now, 0);
1264
1265         return HRTIMER_NORESTART;
1266 }
1267
1268 static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p)
1269 {
1270         u32 nr_met[2] = { };
1271         u32 nr_missed[2] = { };
1272         u64 rq_wait_ns = 0;
1273         int cpu, rw;
1274
1275         for_each_online_cpu(cpu) {
1276                 struct ioc_pcpu_stat *stat = per_cpu_ptr(ioc->pcpu_stat, cpu);
1277                 u64 this_rq_wait_ns;
1278
1279                 for (rw = READ; rw <= WRITE; rw++) {
1280                         u32 this_met = READ_ONCE(stat->missed[rw].nr_met);
1281                         u32 this_missed = READ_ONCE(stat->missed[rw].nr_missed);
1282
1283                         nr_met[rw] += this_met - stat->missed[rw].last_met;
1284                         nr_missed[rw] += this_missed - stat->missed[rw].last_missed;
1285                         stat->missed[rw].last_met = this_met;
1286                         stat->missed[rw].last_missed = this_missed;
1287                 }
1288
1289                 this_rq_wait_ns = READ_ONCE(stat->rq_wait_ns);
1290                 rq_wait_ns += this_rq_wait_ns - stat->last_rq_wait_ns;
1291                 stat->last_rq_wait_ns = this_rq_wait_ns;
1292         }
1293
1294         for (rw = READ; rw <= WRITE; rw++) {
1295                 if (nr_met[rw] + nr_missed[rw])
1296                         missed_ppm_ar[rw] =
1297                                 DIV64_U64_ROUND_UP((u64)nr_missed[rw] * MILLION,
1298                                                    nr_met[rw] + nr_missed[rw]);
1299                 else
1300                         missed_ppm_ar[rw] = 0;
1301         }
1302
1303         *rq_wait_pct_p = div64_u64(rq_wait_ns * 100,
1304                                    ioc->period_us * NSEC_PER_USEC);
1305 }
1306
1307 /* was iocg idle this period? */
1308 static bool iocg_is_idle(struct ioc_gq *iocg)
1309 {
1310         struct ioc *ioc = iocg->ioc;
1311
1312         /* did something get issued this period? */
1313         if (atomic64_read(&iocg->active_period) ==
1314             atomic64_read(&ioc->cur_period))
1315                 return false;
1316
1317         /* is something in flight? */
1318         if (atomic64_read(&iocg->done_vtime) != atomic64_read(&iocg->vtime))
1319                 return false;
1320
1321         return true;
1322 }
1323
1324 /* returns usage with margin added if surplus is large enough */
1325 static u32 surplus_adjusted_hweight_inuse(u32 usage, u32 hw_inuse)
1326 {
1327         /* add margin */
1328         usage = DIV_ROUND_UP(usage * SURPLUS_SCALE_PCT, 100);
1329         usage += SURPLUS_SCALE_ABS;
1330
1331         /* don't bother if the surplus is too small */
1332         if (usage + SURPLUS_MIN_ADJ_DELTA > hw_inuse)
1333                 return 0;
1334
1335         return usage;
1336 }
1337
1338 static void ioc_timer_fn(struct timer_list *timer)
1339 {
1340         struct ioc *ioc = container_of(timer, struct ioc, timer);
1341         struct ioc_gq *iocg, *tiocg;
1342         struct ioc_now now;
1343         int nr_surpluses = 0, nr_shortages = 0, nr_lagging = 0;
1344         u32 ppm_rthr = MILLION - ioc->params.qos[QOS_RPPM];
1345         u32 ppm_wthr = MILLION - ioc->params.qos[QOS_WPPM];
1346         u32 missed_ppm[2], rq_wait_pct;
1347         u64 period_vtime;
1348         int prev_busy_level, i;
1349
1350         /* how were the latencies during the period? */
1351         ioc_lat_stat(ioc, missed_ppm, &rq_wait_pct);
1352
1353         /* take care of active iocgs */
1354         spin_lock_irq(&ioc->lock);
1355
1356         ioc_now(ioc, &now);
1357
1358         period_vtime = now.vnow - ioc->period_at_vtime;
1359         if (WARN_ON_ONCE(!period_vtime)) {
1360                 spin_unlock_irq(&ioc->lock);
1361                 return;
1362         }
1363
1364         /*
1365          * Waiters determine the sleep durations based on the vrate they
1366          * saw at the time of sleep.  If vrate has increased, some waiters
1367          * could be sleeping for too long.  Wake up tardy waiters which
1368          * should have woken up in the last period and expire idle iocgs.
1369          */
1370         list_for_each_entry_safe(iocg, tiocg, &ioc->active_iocgs, active_list) {
1371                 if (!waitqueue_active(&iocg->waitq) &&
1372                     !atomic64_read(&iocg->abs_vdebt) && !iocg_is_idle(iocg))
1373                         continue;
1374
1375                 spin_lock(&iocg->waitq.lock);
1376
1377                 if (waitqueue_active(&iocg->waitq) ||
1378                     atomic64_read(&iocg->abs_vdebt)) {
1379                         /* might be oversleeping vtime / hweight changes, kick */
1380                         iocg_kick_waitq(iocg, &now);
1381                         iocg_kick_delay(iocg, &now, 0);
1382                 } else if (iocg_is_idle(iocg)) {
1383                         /* no waiter and idle, deactivate */
1384                         iocg->last_inuse = iocg->inuse;
1385                         __propagate_active_weight(iocg, 0, 0);
1386                         list_del_init(&iocg->active_list);
1387                 }
1388
1389                 spin_unlock(&iocg->waitq.lock);
1390         }
1391         commit_active_weights(ioc);
1392
1393         /* calc usages and see whether some weights need to be moved around */
1394         list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
1395                 u64 vdone, vtime, vusage, vmargin, vmin;
1396                 u32 hw_active, hw_inuse, usage;
1397
1398                 /*
1399                  * Collect unused and wind vtime closer to vnow to prevent
1400                  * iocgs from accumulating a large amount of budget.
1401                  */
1402                 vdone = atomic64_read(&iocg->done_vtime);
1403                 vtime = atomic64_read(&iocg->vtime);
1404                 current_hweight(iocg, &hw_active, &hw_inuse);
1405
1406                 /*
1407                  * Latency QoS detection doesn't account for IOs which are
1408                  * in-flight for longer than a period.  Detect them by
1409                  * comparing vdone against period start.  If lagging behind
1410                  * IOs from past periods, don't increase vrate.
1411                  */
1412                 if ((ppm_rthr != MILLION || ppm_wthr != MILLION) &&
1413                     !atomic_read(&iocg_to_blkg(iocg)->use_delay) &&
1414                     time_after64(vtime, vdone) &&
1415                     time_after64(vtime, now.vnow -
1416                                  MAX_LAGGING_PERIODS * period_vtime) &&
1417                     time_before64(vdone, now.vnow - period_vtime))
1418                         nr_lagging++;
1419
1420                 if (waitqueue_active(&iocg->waitq))
1421                         vusage = now.vnow - iocg->last_vtime;
1422                 else if (time_before64(iocg->last_vtime, vtime))
1423                         vusage = vtime - iocg->last_vtime;
1424                 else
1425                         vusage = 0;
1426
1427                 iocg->last_vtime += vusage;
1428                 /*
1429                  * Factor in in-flight vtime into vusage to avoid
1430                  * high-latency completions appearing as idle.  This should
1431                  * be done after the above ->last_time adjustment.
1432                  */
1433                 vusage = max(vusage, vtime - vdone);
1434
1435                 /* calculate hweight based usage ratio and record */
1436                 if (vusage) {
1437                         usage = DIV64_U64_ROUND_UP(vusage * hw_inuse,
1438                                                    period_vtime);
1439                         iocg->usage_idx = (iocg->usage_idx + 1) % NR_USAGE_SLOTS;
1440                         iocg->usages[iocg->usage_idx] = usage;
1441                 } else {
1442                         usage = 0;
1443                 }
1444
1445                 /* see whether there's surplus vtime */
1446                 vmargin = ioc->margin_us * now.vrate;
1447                 vmin = now.vnow - vmargin;
1448
1449                 iocg->has_surplus = false;
1450
1451                 if (!waitqueue_active(&iocg->waitq) &&
1452                     time_before64(vtime, vmin)) {
1453                         u64 delta = vmin - vtime;
1454
1455                         /* throw away surplus vtime */
1456                         atomic64_add(delta, &iocg->vtime);
1457                         atomic64_add(delta, &iocg->done_vtime);
1458                         iocg->last_vtime += delta;
1459                         /* if usage is sufficiently low, maybe it can donate */
1460                         if (surplus_adjusted_hweight_inuse(usage, hw_inuse)) {
1461                                 iocg->has_surplus = true;
1462                                 nr_surpluses++;
1463                         }
1464                 } else if (hw_inuse < hw_active) {
1465                         u32 new_hwi, new_inuse;
1466
1467                         /* was donating but might need to take back some */
1468                         if (waitqueue_active(&iocg->waitq)) {
1469                                 new_hwi = hw_active;
1470                         } else {
1471                                 new_hwi = max(hw_inuse,
1472                                               usage * SURPLUS_SCALE_PCT / 100 +
1473                                               SURPLUS_SCALE_ABS);
1474                         }
1475
1476                         new_inuse = div64_u64((u64)iocg->inuse * new_hwi,
1477                                               hw_inuse);
1478                         new_inuse = clamp_t(u32, new_inuse, 1, iocg->active);
1479
1480                         if (new_inuse > iocg->inuse) {
1481                                 TRACE_IOCG_PATH(inuse_takeback, iocg, &now,
1482                                                 iocg->inuse, new_inuse,
1483                                                 hw_inuse, new_hwi);
1484                                 __propagate_active_weight(iocg, iocg->weight,
1485                                                           new_inuse);
1486                         }
1487                 } else {
1488                         /* genuninely out of vtime */
1489                         nr_shortages++;
1490                 }
1491         }
1492
1493         if (!nr_shortages || !nr_surpluses)
1494                 goto skip_surplus_transfers;
1495
1496         /* there are both shortages and surpluses, transfer surpluses */
1497         list_for_each_entry(iocg, &ioc->active_iocgs, active_list) {
1498                 u32 usage, hw_active, hw_inuse, new_hwi, new_inuse;
1499                 int nr_valid = 0;
1500
1501                 if (!iocg->has_surplus)
1502                         continue;
1503
1504                 /* base the decision on max historical usage */
1505                 for (i = 0, usage = 0; i < NR_USAGE_SLOTS; i++) {
1506                         if (iocg->usages[i]) {
1507                                 usage = max(usage, iocg->usages[i]);
1508                                 nr_valid++;
1509                         }
1510                 }
1511                 if (nr_valid < MIN_VALID_USAGES)
1512                         continue;
1513
1514                 current_hweight(iocg, &hw_active, &hw_inuse);
1515                 new_hwi = surplus_adjusted_hweight_inuse(usage, hw_inuse);
1516                 if (!new_hwi)
1517                         continue;
1518
1519                 new_inuse = DIV64_U64_ROUND_UP((u64)iocg->inuse * new_hwi,
1520                                                hw_inuse);
1521                 if (new_inuse < iocg->inuse) {
1522                         TRACE_IOCG_PATH(inuse_giveaway, iocg, &now,
1523                                         iocg->inuse, new_inuse,
1524                                         hw_inuse, new_hwi);
1525                         __propagate_active_weight(iocg, iocg->weight, new_inuse);
1526                 }
1527         }
1528 skip_surplus_transfers:
1529         commit_active_weights(ioc);
1530
1531         /*
1532          * If q is getting clogged or we're missing too much, we're issuing
1533          * too much IO and should lower vtime rate.  If we're not missing
1534          * and experiencing shortages but not surpluses, we're too stingy
1535          * and should increase vtime rate.
1536          */
1537         prev_busy_level = ioc->busy_level;
1538         if (rq_wait_pct > RQ_WAIT_BUSY_PCT ||
1539             missed_ppm[READ] > ppm_rthr ||
1540             missed_ppm[WRITE] > ppm_wthr) {
1541                 ioc->busy_level = max(ioc->busy_level, 0);
1542                 ioc->busy_level++;
1543         } else if (rq_wait_pct <= RQ_WAIT_BUSY_PCT * UNBUSY_THR_PCT / 100 &&
1544                    missed_ppm[READ] <= ppm_rthr * UNBUSY_THR_PCT / 100 &&
1545                    missed_ppm[WRITE] <= ppm_wthr * UNBUSY_THR_PCT / 100) {
1546                 /* take action iff there is contention */
1547                 if (nr_shortages && !nr_lagging) {
1548                         ioc->busy_level = min(ioc->busy_level, 0);
1549                         /* redistribute surpluses first */
1550                         if (!nr_surpluses)
1551                                 ioc->busy_level--;
1552                 }
1553         } else {
1554                 ioc->busy_level = 0;
1555         }
1556
1557         ioc->busy_level = clamp(ioc->busy_level, -1000, 1000);
1558
1559         if (ioc->busy_level > 0 || (ioc->busy_level < 0 && !nr_lagging)) {
1560                 u64 vrate = atomic64_read(&ioc->vtime_rate);
1561                 u64 vrate_min = ioc->vrate_min, vrate_max = ioc->vrate_max;
1562
1563                 /* rq_wait signal is always reliable, ignore user vrate_min */
1564                 if (rq_wait_pct > RQ_WAIT_BUSY_PCT)
1565                         vrate_min = VRATE_MIN;
1566
1567                 /*
1568                  * If vrate is out of bounds, apply clamp gradually as the
1569                  * bounds can change abruptly.  Otherwise, apply busy_level
1570                  * based adjustment.
1571                  */
1572                 if (vrate < vrate_min) {
1573                         vrate = div64_u64(vrate * (100 + VRATE_CLAMP_ADJ_PCT),
1574                                           100);
1575                         vrate = min(vrate, vrate_min);
1576                 } else if (vrate > vrate_max) {
1577                         vrate = div64_u64(vrate * (100 - VRATE_CLAMP_ADJ_PCT),
1578                                           100);
1579                         vrate = max(vrate, vrate_max);
1580                 } else {
1581                         int idx = min_t(int, abs(ioc->busy_level),
1582                                         ARRAY_SIZE(vrate_adj_pct) - 1);
1583                         u32 adj_pct = vrate_adj_pct[idx];
1584
1585                         if (ioc->busy_level > 0)
1586                                 adj_pct = 100 - adj_pct;
1587                         else
1588                                 adj_pct = 100 + adj_pct;
1589
1590                         vrate = clamp(DIV64_U64_ROUND_UP(vrate * adj_pct, 100),
1591                                       vrate_min, vrate_max);
1592                 }
1593
1594                 trace_iocost_ioc_vrate_adj(ioc, vrate, &missed_ppm, rq_wait_pct,
1595                                            nr_lagging, nr_shortages,
1596                                            nr_surpluses);
1597
1598                 atomic64_set(&ioc->vtime_rate, vrate);
1599                 ioc->inuse_margin_vtime = DIV64_U64_ROUND_UP(
1600                         ioc->period_us * vrate * INUSE_MARGIN_PCT, 100);
1601         } else if (ioc->busy_level != prev_busy_level || nr_lagging) {
1602                 trace_iocost_ioc_vrate_adj(ioc, atomic64_read(&ioc->vtime_rate),
1603                                            &missed_ppm, rq_wait_pct, nr_lagging,
1604                                            nr_shortages, nr_surpluses);
1605         }
1606
1607         ioc_refresh_params(ioc, false);
1608
1609         /*
1610          * This period is done.  Move onto the next one.  If nothing's
1611          * going on with the device, stop the timer.
1612          */
1613         atomic64_inc(&ioc->cur_period);
1614
1615         if (ioc->running != IOC_STOP) {
1616                 if (!list_empty(&ioc->active_iocgs)) {
1617                         ioc_start_period(ioc, &now);
1618                 } else {
1619                         ioc->busy_level = 0;
1620                         ioc->running = IOC_IDLE;
1621                 }
1622         }
1623
1624         spin_unlock_irq(&ioc->lock);
1625 }
1626
1627 static void calc_vtime_cost_builtin(struct bio *bio, struct ioc_gq *iocg,
1628                                     bool is_merge, u64 *costp)
1629 {
1630         struct ioc *ioc = iocg->ioc;
1631         u64 coef_seqio, coef_randio, coef_page;
1632         u64 pages = max_t(u64, bio_sectors(bio) >> IOC_SECT_TO_PAGE_SHIFT, 1);
1633         u64 seek_pages = 0;
1634         u64 cost = 0;
1635
1636         switch (bio_op(bio)) {
1637         case REQ_OP_READ:
1638                 coef_seqio      = ioc->params.lcoefs[LCOEF_RSEQIO];
1639                 coef_randio     = ioc->params.lcoefs[LCOEF_RRANDIO];
1640                 coef_page       = ioc->params.lcoefs[LCOEF_RPAGE];
1641                 break;
1642         case REQ_OP_WRITE:
1643                 coef_seqio      = ioc->params.lcoefs[LCOEF_WSEQIO];
1644                 coef_randio     = ioc->params.lcoefs[LCOEF_WRANDIO];
1645                 coef_page       = ioc->params.lcoefs[LCOEF_WPAGE];
1646                 break;
1647         default:
1648                 goto out;
1649         }
1650
1651         if (iocg->cursor) {
1652                 seek_pages = abs(bio->bi_iter.bi_sector - iocg->cursor);
1653                 seek_pages >>= IOC_SECT_TO_PAGE_SHIFT;
1654         }
1655
1656         if (!is_merge) {
1657                 if (seek_pages > LCOEF_RANDIO_PAGES) {
1658                         cost += coef_randio;
1659                 } else {
1660                         cost += coef_seqio;
1661                 }
1662         }
1663         cost += pages * coef_page;
1664 out:
1665         *costp = cost;
1666 }
1667
1668 static u64 calc_vtime_cost(struct bio *bio, struct ioc_gq *iocg, bool is_merge)
1669 {
1670         u64 cost;
1671
1672         calc_vtime_cost_builtin(bio, iocg, is_merge, &cost);
1673         return cost;
1674 }
1675
1676 static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio)
1677 {
1678         struct blkcg_gq *blkg = bio->bi_blkg;
1679         struct ioc *ioc = rqos_to_ioc(rqos);
1680         struct ioc_gq *iocg = blkg_to_iocg(blkg);
1681         struct ioc_now now;
1682         struct iocg_wait wait;
1683         u32 hw_active, hw_inuse;
1684         u64 abs_cost, cost, vtime;
1685
1686         /* bypass IOs if disabled or for root cgroup */
1687         if (!ioc->enabled || !iocg->level)
1688                 return;
1689
1690         /* always activate so that even 0 cost IOs get protected to some level */
1691         if (!iocg_activate(iocg, &now))
1692                 return;
1693
1694         /* calculate the absolute vtime cost */
1695         abs_cost = calc_vtime_cost(bio, iocg, false);
1696         if (!abs_cost)
1697                 return;
1698
1699         iocg->cursor = bio_end_sector(bio);
1700
1701         vtime = atomic64_read(&iocg->vtime);
1702         current_hweight(iocg, &hw_active, &hw_inuse);
1703
1704         if (hw_inuse < hw_active &&
1705             time_after_eq64(vtime + ioc->inuse_margin_vtime, now.vnow)) {
1706                 TRACE_IOCG_PATH(inuse_reset, iocg, &now,
1707                                 iocg->inuse, iocg->weight, hw_inuse, hw_active);
1708                 spin_lock_irq(&ioc->lock);
1709                 propagate_active_weight(iocg, iocg->weight, iocg->weight);
1710                 spin_unlock_irq(&ioc->lock);
1711                 current_hweight(iocg, &hw_active, &hw_inuse);
1712         }
1713
1714         cost = abs_cost_to_cost(abs_cost, hw_inuse);
1715
1716         /*
1717          * If no one's waiting and within budget, issue right away.  The
1718          * tests are racy but the races aren't systemic - we only miss once
1719          * in a while which is fine.
1720          */
1721         if (!waitqueue_active(&iocg->waitq) &&
1722             !atomic64_read(&iocg->abs_vdebt) &&
1723             time_before_eq64(vtime + cost, now.vnow)) {
1724                 iocg_commit_bio(iocg, bio, cost);
1725                 return;
1726         }
1727
1728         /*
1729          * We're over budget.  If @bio has to be issued regardless,
1730          * remember the abs_cost instead of advancing vtime.
1731          * iocg_kick_waitq() will pay off the debt before waking more IOs.
1732          * This way, the debt is continuously paid off each period with the
1733          * actual budget available to the cgroup.  If we just wound vtime,
1734          * we would incorrectly use the current hw_inuse for the entire
1735          * amount which, for example, can lead to the cgroup staying
1736          * blocked for a long time even with substantially raised hw_inuse.
1737          */
1738         if (bio_issue_as_root_blkg(bio) || fatal_signal_pending(current)) {
1739                 atomic64_add(abs_cost, &iocg->abs_vdebt);
1740                 if (iocg_kick_delay(iocg, &now, cost))
1741                         blkcg_schedule_throttle(rqos->q,
1742                                         (bio->bi_opf & REQ_SWAP) == REQ_SWAP);
1743                 return;
1744         }
1745
1746         /*
1747          * Append self to the waitq and schedule the wakeup timer if we're
1748          * the first waiter.  The timer duration is calculated based on the
1749          * current vrate.  vtime and hweight changes can make it too short
1750          * or too long.  Each wait entry records the absolute cost it's
1751          * waiting for to allow re-evaluation using a custom wait entry.
1752          *
1753          * If too short, the timer simply reschedules itself.  If too long,
1754          * the period timer will notice and trigger wakeups.
1755          *
1756          * All waiters are on iocg->waitq and the wait states are
1757          * synchronized using waitq.lock.
1758          */
1759         spin_lock_irq(&iocg->waitq.lock);
1760
1761         /*
1762          * We activated above but w/o any synchronization.  Deactivation is
1763          * synchronized with waitq.lock and we won't get deactivated as
1764          * long as we're waiting, so we're good if we're activated here.
1765          * In the unlikely case that we are deactivated, just issue the IO.
1766          */
1767         if (unlikely(list_empty(&iocg->active_list))) {
1768                 spin_unlock_irq(&iocg->waitq.lock);
1769                 iocg_commit_bio(iocg, bio, cost);
1770                 return;
1771         }
1772
1773         init_waitqueue_func_entry(&wait.wait, iocg_wake_fn);
1774         wait.wait.private = current;
1775         wait.bio = bio;
1776         wait.abs_cost = abs_cost;
1777         wait.committed = false; /* will be set true by waker */
1778
1779         __add_wait_queue_entry_tail(&iocg->waitq, &wait.wait);
1780         iocg_kick_waitq(iocg, &now);
1781
1782         spin_unlock_irq(&iocg->waitq.lock);
1783
1784         while (true) {
1785                 set_current_state(TASK_UNINTERRUPTIBLE);
1786                 if (wait.committed)
1787                         break;
1788                 io_schedule();
1789         }
1790
1791         /* waker already committed us, proceed */
1792         finish_wait(&iocg->waitq, &wait.wait);
1793 }
1794
1795 static void ioc_rqos_merge(struct rq_qos *rqos, struct request *rq,
1796                            struct bio *bio)
1797 {
1798         struct ioc_gq *iocg = blkg_to_iocg(bio->bi_blkg);
1799         struct ioc *ioc = iocg->ioc;
1800         sector_t bio_end = bio_end_sector(bio);
1801         struct ioc_now now;
1802         u32 hw_inuse;
1803         u64 abs_cost, cost;
1804
1805         /* bypass if disabled or for root cgroup */
1806         if (!ioc->enabled || !iocg->level)
1807                 return;
1808
1809         abs_cost = calc_vtime_cost(bio, iocg, true);
1810         if (!abs_cost)
1811                 return;
1812
1813         ioc_now(ioc, &now);
1814         current_hweight(iocg, NULL, &hw_inuse);
1815         cost = abs_cost_to_cost(abs_cost, hw_inuse);
1816
1817         /* update cursor if backmerging into the request at the cursor */
1818         if (blk_rq_pos(rq) < bio_end &&
1819             blk_rq_pos(rq) + blk_rq_sectors(rq) == iocg->cursor)
1820                 iocg->cursor = bio_end;
1821
1822         /*
1823          * Charge if there's enough vtime budget and the existing request
1824          * has cost assigned.  Otherwise, account it as debt.  See debt
1825          * handling in ioc_rqos_throttle() for details.
1826          */
1827         if (rq->bio && rq->bio->bi_iocost_cost &&
1828             time_before_eq64(atomic64_read(&iocg->vtime) + cost, now.vnow))
1829                 iocg_commit_bio(iocg, bio, cost);
1830         else
1831                 atomic64_add(abs_cost, &iocg->abs_vdebt);
1832 }
1833
1834 static void ioc_rqos_done_bio(struct rq_qos *rqos, struct bio *bio)
1835 {
1836         struct ioc_gq *iocg = blkg_to_iocg(bio->bi_blkg);
1837
1838         if (iocg && bio->bi_iocost_cost)
1839                 atomic64_add(bio->bi_iocost_cost, &iocg->done_vtime);
1840 }
1841
1842 static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq)
1843 {
1844         struct ioc *ioc = rqos_to_ioc(rqos);
1845         u64 on_q_ns, rq_wait_ns;
1846         int pidx, rw;
1847
1848         if (!ioc->enabled || !rq->alloc_time_ns || !rq->start_time_ns)
1849                 return;
1850
1851         switch (req_op(rq) & REQ_OP_MASK) {
1852         case REQ_OP_READ:
1853                 pidx = QOS_RLAT;
1854                 rw = READ;
1855                 break;
1856         case REQ_OP_WRITE:
1857                 pidx = QOS_WLAT;
1858                 rw = WRITE;
1859                 break;
1860         default:
1861                 return;
1862         }
1863
1864         on_q_ns = ktime_get_ns() - rq->alloc_time_ns;
1865         rq_wait_ns = rq->start_time_ns - rq->alloc_time_ns;
1866
1867         if (on_q_ns <= ioc->params.qos[pidx] * NSEC_PER_USEC)
1868                 this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_met);
1869         else
1870                 this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_missed);
1871
1872         this_cpu_add(ioc->pcpu_stat->rq_wait_ns, rq_wait_ns);
1873 }
1874
1875 static void ioc_rqos_queue_depth_changed(struct rq_qos *rqos)
1876 {
1877         struct ioc *ioc = rqos_to_ioc(rqos);
1878
1879         spin_lock_irq(&ioc->lock);
1880         ioc_refresh_params(ioc, false);
1881         spin_unlock_irq(&ioc->lock);
1882 }
1883
1884 static void ioc_rqos_exit(struct rq_qos *rqos)
1885 {
1886         struct ioc *ioc = rqos_to_ioc(rqos);
1887
1888         blkcg_deactivate_policy(rqos->q, &blkcg_policy_iocost);
1889
1890         spin_lock_irq(&ioc->lock);
1891         ioc->running = IOC_STOP;
1892         spin_unlock_irq(&ioc->lock);
1893
1894         del_timer_sync(&ioc->timer);
1895         free_percpu(ioc->pcpu_stat);
1896         kfree(ioc);
1897 }
1898
1899 static struct rq_qos_ops ioc_rqos_ops = {
1900         .throttle = ioc_rqos_throttle,
1901         .merge = ioc_rqos_merge,
1902         .done_bio = ioc_rqos_done_bio,
1903         .done = ioc_rqos_done,
1904         .queue_depth_changed = ioc_rqos_queue_depth_changed,
1905         .exit = ioc_rqos_exit,
1906 };
1907
1908 static int blk_iocost_init(struct request_queue *q)
1909 {
1910         struct ioc *ioc;
1911         struct rq_qos *rqos;
1912         int ret;
1913
1914         ioc = kzalloc(sizeof(*ioc), GFP_KERNEL);
1915         if (!ioc)
1916                 return -ENOMEM;
1917
1918         ioc->pcpu_stat = alloc_percpu(struct ioc_pcpu_stat);
1919         if (!ioc->pcpu_stat) {
1920                 kfree(ioc);
1921                 return -ENOMEM;
1922         }
1923
1924         rqos = &ioc->rqos;
1925         rqos->id = RQ_QOS_COST;
1926         rqos->ops = &ioc_rqos_ops;
1927         rqos->q = q;
1928
1929         spin_lock_init(&ioc->lock);
1930         timer_setup(&ioc->timer, ioc_timer_fn, 0);
1931         INIT_LIST_HEAD(&ioc->active_iocgs);
1932
1933         ioc->running = IOC_IDLE;
1934         atomic64_set(&ioc->vtime_rate, VTIME_PER_USEC);
1935         seqcount_init(&ioc->period_seqcount);
1936         ioc->period_at = ktime_to_us(ktime_get());
1937         atomic64_set(&ioc->cur_period, 0);
1938         atomic_set(&ioc->hweight_gen, 0);
1939
1940         spin_lock_irq(&ioc->lock);
1941         ioc->autop_idx = AUTOP_INVALID;
1942         ioc_refresh_params(ioc, true);
1943         spin_unlock_irq(&ioc->lock);
1944
1945         rq_qos_add(q, rqos);
1946         ret = blkcg_activate_policy(q, &blkcg_policy_iocost);
1947         if (ret) {
1948                 rq_qos_del(q, rqos);
1949                 free_percpu(ioc->pcpu_stat);
1950                 kfree(ioc);
1951                 return ret;
1952         }
1953         return 0;
1954 }
1955
1956 static struct blkcg_policy_data *ioc_cpd_alloc(gfp_t gfp)
1957 {
1958         struct ioc_cgrp *iocc;
1959
1960         iocc = kzalloc(sizeof(struct ioc_cgrp), gfp);
1961         if (!iocc)
1962                 return NULL;
1963
1964         iocc->dfl_weight = CGROUP_WEIGHT_DFL;
1965         return &iocc->cpd;
1966 }
1967
1968 static void ioc_cpd_free(struct blkcg_policy_data *cpd)
1969 {
1970         kfree(container_of(cpd, struct ioc_cgrp, cpd));
1971 }
1972
1973 static struct blkg_policy_data *ioc_pd_alloc(gfp_t gfp, struct request_queue *q,
1974                                              struct blkcg *blkcg)
1975 {
1976         int levels = blkcg->css.cgroup->level + 1;
1977         struct ioc_gq *iocg;
1978
1979         iocg = kzalloc_node(sizeof(*iocg) + levels * sizeof(iocg->ancestors[0]),
1980                             gfp, q->node);
1981         if (!iocg)
1982                 return NULL;
1983
1984         return &iocg->pd;
1985 }
1986
1987 static void ioc_pd_init(struct blkg_policy_data *pd)
1988 {
1989         struct ioc_gq *iocg = pd_to_iocg(pd);
1990         struct blkcg_gq *blkg = pd_to_blkg(&iocg->pd);
1991         struct ioc *ioc = q_to_ioc(blkg->q);
1992         struct ioc_now now;
1993         struct blkcg_gq *tblkg;
1994         unsigned long flags;
1995
1996         ioc_now(ioc, &now);
1997
1998         iocg->ioc = ioc;
1999         atomic64_set(&iocg->vtime, now.vnow);
2000         atomic64_set(&iocg->done_vtime, now.vnow);
2001         atomic64_set(&iocg->abs_vdebt, 0);
2002         atomic64_set(&iocg->active_period, atomic64_read(&ioc->cur_period));
2003         INIT_LIST_HEAD(&iocg->active_list);
2004         iocg->hweight_active = HWEIGHT_WHOLE;
2005         iocg->hweight_inuse = HWEIGHT_WHOLE;
2006
2007         init_waitqueue_head(&iocg->waitq);
2008         hrtimer_init(&iocg->waitq_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2009         iocg->waitq_timer.function = iocg_waitq_timer_fn;
2010         hrtimer_init(&iocg->delay_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2011         iocg->delay_timer.function = iocg_delay_timer_fn;
2012
2013         iocg->level = blkg->blkcg->css.cgroup->level;
2014
2015         for (tblkg = blkg; tblkg; tblkg = tblkg->parent) {
2016                 struct ioc_gq *tiocg = blkg_to_iocg(tblkg);
2017                 iocg->ancestors[tiocg->level] = tiocg;
2018         }
2019
2020         spin_lock_irqsave(&ioc->lock, flags);
2021         weight_updated(iocg);
2022         spin_unlock_irqrestore(&ioc->lock, flags);
2023 }
2024
2025 static void ioc_pd_free(struct blkg_policy_data *pd)
2026 {
2027         struct ioc_gq *iocg = pd_to_iocg(pd);
2028         struct ioc *ioc = iocg->ioc;
2029
2030         if (ioc) {
2031                 spin_lock(&ioc->lock);
2032                 if (!list_empty(&iocg->active_list)) {
2033                         propagate_active_weight(iocg, 0, 0);
2034                         list_del_init(&iocg->active_list);
2035                 }
2036                 spin_unlock(&ioc->lock);
2037
2038                 hrtimer_cancel(&iocg->waitq_timer);
2039                 hrtimer_cancel(&iocg->delay_timer);
2040         }
2041         kfree(iocg);
2042 }
2043
2044 static u64 ioc_weight_prfill(struct seq_file *sf, struct blkg_policy_data *pd,
2045                              int off)
2046 {
2047         const char *dname = blkg_dev_name(pd->blkg);
2048         struct ioc_gq *iocg = pd_to_iocg(pd);
2049
2050         if (dname && iocg->cfg_weight)
2051                 seq_printf(sf, "%s %u\n", dname, iocg->cfg_weight);
2052         return 0;
2053 }
2054
2055
2056 static int ioc_weight_show(struct seq_file *sf, void *v)
2057 {
2058         struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
2059         struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);
2060
2061         seq_printf(sf, "default %u\n", iocc->dfl_weight);
2062         blkcg_print_blkgs(sf, blkcg, ioc_weight_prfill,
2063                           &blkcg_policy_iocost, seq_cft(sf)->private, false);
2064         return 0;
2065 }
2066
2067 static ssize_t ioc_weight_write(struct kernfs_open_file *of, char *buf,
2068                                 size_t nbytes, loff_t off)
2069 {
2070         struct blkcg *blkcg = css_to_blkcg(of_css(of));
2071         struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);
2072         struct blkg_conf_ctx ctx;
2073         struct ioc_gq *iocg;
2074         u32 v;
2075         int ret;
2076
2077         if (!strchr(buf, ':')) {
2078                 struct blkcg_gq *blkg;
2079
2080                 if (!sscanf(buf, "default %u", &v) && !sscanf(buf, "%u", &v))
2081                         return -EINVAL;
2082
2083                 if (v < CGROUP_WEIGHT_MIN || v > CGROUP_WEIGHT_MAX)
2084                         return -EINVAL;
2085
2086                 spin_lock(&blkcg->lock);
2087                 iocc->dfl_weight = v;
2088                 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
2089                         struct ioc_gq *iocg = blkg_to_iocg(blkg);
2090
2091                         if (iocg) {
2092                                 spin_lock_irq(&iocg->ioc->lock);
2093                                 weight_updated(iocg);
2094                                 spin_unlock_irq(&iocg->ioc->lock);
2095                         }
2096                 }
2097                 spin_unlock(&blkcg->lock);
2098
2099                 return nbytes;
2100         }
2101
2102         ret = blkg_conf_prep(blkcg, &blkcg_policy_iocost, buf, &ctx);
2103         if (ret)
2104                 return ret;
2105
2106         iocg = blkg_to_iocg(ctx.blkg);
2107
2108         if (!strncmp(ctx.body, "default", 7)) {
2109                 v = 0;
2110         } else {
2111                 if (!sscanf(ctx.body, "%u", &v))
2112                         goto einval;
2113                 if (v < CGROUP_WEIGHT_MIN || v > CGROUP_WEIGHT_MAX)
2114                         goto einval;
2115         }
2116
2117         spin_lock(&iocg->ioc->lock);
2118         iocg->cfg_weight = v;
2119         weight_updated(iocg);
2120         spin_unlock(&iocg->ioc->lock);
2121
2122         blkg_conf_finish(&ctx);
2123         return nbytes;
2124
2125 einval:
2126         blkg_conf_finish(&ctx);
2127         return -EINVAL;
2128 }
2129
2130 static u64 ioc_qos_prfill(struct seq_file *sf, struct blkg_policy_data *pd,
2131                           int off)
2132 {
2133         const char *dname = blkg_dev_name(pd->blkg);
2134         struct ioc *ioc = pd_to_iocg(pd)->ioc;
2135
2136         if (!dname)
2137                 return 0;
2138
2139         seq_printf(sf, "%s enable=%d ctrl=%s rpct=%u.%02u rlat=%u wpct=%u.%02u wlat=%u min=%u.%02u max=%u.%02u\n",
2140                    dname, ioc->enabled, ioc->user_qos_params ? "user" : "auto",
2141                    ioc->params.qos[QOS_RPPM] / 10000,
2142                    ioc->params.qos[QOS_RPPM] % 10000 / 100,
2143                    ioc->params.qos[QOS_RLAT],
2144                    ioc->params.qos[QOS_WPPM] / 10000,
2145                    ioc->params.qos[QOS_WPPM] % 10000 / 100,
2146                    ioc->params.qos[QOS_WLAT],
2147                    ioc->params.qos[QOS_MIN] / 10000,
2148                    ioc->params.qos[QOS_MIN] % 10000 / 100,
2149                    ioc->params.qos[QOS_MAX] / 10000,
2150                    ioc->params.qos[QOS_MAX] % 10000 / 100);
2151         return 0;
2152 }
2153
2154 static int ioc_qos_show(struct seq_file *sf, void *v)
2155 {
2156         struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
2157
2158         blkcg_print_blkgs(sf, blkcg, ioc_qos_prfill,
2159                           &blkcg_policy_iocost, seq_cft(sf)->private, false);
2160         return 0;
2161 }
2162
2163 static const match_table_t qos_ctrl_tokens = {
2164         { QOS_ENABLE,           "enable=%u"     },
2165         { QOS_CTRL,             "ctrl=%s"       },
2166         { NR_QOS_CTRL_PARAMS,   NULL            },
2167 };
2168
2169 static const match_table_t qos_tokens = {
2170         { QOS_RPPM,             "rpct=%s"       },
2171         { QOS_RLAT,             "rlat=%u"       },
2172         { QOS_WPPM,             "wpct=%s"       },
2173         { QOS_WLAT,             "wlat=%u"       },
2174         { QOS_MIN,              "min=%s"        },
2175         { QOS_MAX,              "max=%s"        },
2176         { NR_QOS_PARAMS,        NULL            },
2177 };
2178
2179 static ssize_t ioc_qos_write(struct kernfs_open_file *of, char *input,
2180                              size_t nbytes, loff_t off)
2181 {
2182         struct gendisk *disk;
2183         struct ioc *ioc;
2184         u32 qos[NR_QOS_PARAMS];
2185         bool enable, user;
2186         char *p;
2187         int ret;
2188
2189         disk = blkcg_conf_get_disk(&input);
2190         if (IS_ERR(disk))
2191                 return PTR_ERR(disk);
2192
2193         ioc = q_to_ioc(disk->queue);
2194         if (!ioc) {
2195                 ret = blk_iocost_init(disk->queue);
2196                 if (ret)
2197                         goto err;
2198                 ioc = q_to_ioc(disk->queue);
2199         }
2200
2201         spin_lock_irq(&ioc->lock);
2202         memcpy(qos, ioc->params.qos, sizeof(qos));
2203         enable = ioc->enabled;
2204         user = ioc->user_qos_params;
2205         spin_unlock_irq(&ioc->lock);
2206
2207         while ((p = strsep(&input, " \t\n"))) {
2208                 substring_t args[MAX_OPT_ARGS];
2209                 char buf[32];
2210                 int tok;
2211                 s64 v;
2212
2213                 if (!*p)
2214                         continue;
2215
2216                 switch (match_token(p, qos_ctrl_tokens, args)) {
2217                 case QOS_ENABLE:
2218                         match_u64(&args[0], &v);
2219                         enable = v;
2220                         continue;
2221                 case QOS_CTRL:
2222                         match_strlcpy(buf, &args[0], sizeof(buf));
2223                         if (!strcmp(buf, "auto"))
2224                                 user = false;
2225                         else if (!strcmp(buf, "user"))
2226                                 user = true;
2227                         else
2228                                 goto einval;
2229                         continue;
2230                 }
2231
2232                 tok = match_token(p, qos_tokens, args);
2233                 switch (tok) {
2234                 case QOS_RPPM:
2235                 case QOS_WPPM:
2236                         if (match_strlcpy(buf, &args[0], sizeof(buf)) >=
2237                             sizeof(buf))
2238                                 goto einval;
2239                         if (cgroup_parse_float(buf, 2, &v))
2240                                 goto einval;
2241                         if (v < 0 || v > 10000)
2242                                 goto einval;
2243                         qos[tok] = v * 100;
2244                         break;
2245                 case QOS_RLAT:
2246                 case QOS_WLAT:
2247                         if (match_u64(&args[0], &v))
2248                                 goto einval;
2249                         qos[tok] = v;
2250                         break;
2251                 case QOS_MIN:
2252                 case QOS_MAX:
2253                         if (match_strlcpy(buf, &args[0], sizeof(buf)) >=
2254                             sizeof(buf))
2255                                 goto einval;
2256                         if (cgroup_parse_float(buf, 2, &v))
2257                                 goto einval;
2258                         if (v < 0)
2259                                 goto einval;
2260                         qos[tok] = clamp_t(s64, v * 100,
2261                                            VRATE_MIN_PPM, VRATE_MAX_PPM);
2262                         break;
2263                 default:
2264                         goto einval;
2265                 }
2266                 user = true;
2267         }
2268
2269         if (qos[QOS_MIN] > qos[QOS_MAX])
2270                 goto einval;
2271
2272         spin_lock_irq(&ioc->lock);
2273
2274         if (enable) {
2275                 blk_queue_flag_set(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
2276                 ioc->enabled = true;
2277         } else {
2278                 blk_queue_flag_clear(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
2279                 ioc->enabled = false;
2280         }
2281
2282         if (user) {
2283                 memcpy(ioc->params.qos, qos, sizeof(qos));
2284                 ioc->user_qos_params = true;
2285         } else {
2286                 ioc->user_qos_params = false;
2287         }
2288
2289         ioc_refresh_params(ioc, true);
2290         spin_unlock_irq(&ioc->lock);
2291
2292         put_disk_and_module(disk);
2293         return nbytes;
2294 einval:
2295         ret = -EINVAL;
2296 err:
2297         put_disk_and_module(disk);
2298         return ret;
2299 }
2300
2301 static u64 ioc_cost_model_prfill(struct seq_file *sf,
2302                                  struct blkg_policy_data *pd, int off)
2303 {
2304         const char *dname = blkg_dev_name(pd->blkg);
2305         struct ioc *ioc = pd_to_iocg(pd)->ioc;
2306         u64 *u = ioc->params.i_lcoefs;
2307
2308         if (!dname)
2309                 return 0;
2310
2311         seq_printf(sf, "%s ctrl=%s model=linear "
2312                    "rbps=%llu rseqiops=%llu rrandiops=%llu "
2313                    "wbps=%llu wseqiops=%llu wrandiops=%llu\n",
2314                    dname, ioc->user_cost_model ? "user" : "auto",
2315                    u[I_LCOEF_RBPS], u[I_LCOEF_RSEQIOPS], u[I_LCOEF_RRANDIOPS],
2316                    u[I_LCOEF_WBPS], u[I_LCOEF_WSEQIOPS], u[I_LCOEF_WRANDIOPS]);
2317         return 0;
2318 }
2319
2320 static int ioc_cost_model_show(struct seq_file *sf, void *v)
2321 {
2322         struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
2323
2324         blkcg_print_blkgs(sf, blkcg, ioc_cost_model_prfill,
2325                           &blkcg_policy_iocost, seq_cft(sf)->private, false);
2326         return 0;
2327 }
2328
2329 static const match_table_t cost_ctrl_tokens = {
2330         { COST_CTRL,            "ctrl=%s"       },
2331         { COST_MODEL,           "model=%s"      },
2332         { NR_COST_CTRL_PARAMS,  NULL            },
2333 };
2334
2335 static const match_table_t i_lcoef_tokens = {
2336         { I_LCOEF_RBPS,         "rbps=%u"       },
2337         { I_LCOEF_RSEQIOPS,     "rseqiops=%u"   },
2338         { I_LCOEF_RRANDIOPS,    "rrandiops=%u"  },
2339         { I_LCOEF_WBPS,         "wbps=%u"       },
2340         { I_LCOEF_WSEQIOPS,     "wseqiops=%u"   },
2341         { I_LCOEF_WRANDIOPS,    "wrandiops=%u"  },
2342         { NR_I_LCOEFS,          NULL            },
2343 };
2344
2345 static ssize_t ioc_cost_model_write(struct kernfs_open_file *of, char *input,
2346                                     size_t nbytes, loff_t off)
2347 {
2348         struct gendisk *disk;
2349         struct ioc *ioc;
2350         u64 u[NR_I_LCOEFS];
2351         bool user;
2352         char *p;
2353         int ret;
2354
2355         disk = blkcg_conf_get_disk(&input);
2356         if (IS_ERR(disk))
2357                 return PTR_ERR(disk);
2358
2359         ioc = q_to_ioc(disk->queue);
2360         if (!ioc) {
2361                 ret = blk_iocost_init(disk->queue);
2362                 if (ret)
2363                         goto err;
2364                 ioc = q_to_ioc(disk->queue);
2365         }
2366
2367         spin_lock_irq(&ioc->lock);
2368         memcpy(u, ioc->params.i_lcoefs, sizeof(u));
2369         user = ioc->user_cost_model;
2370         spin_unlock_irq(&ioc->lock);
2371
2372         while ((p = strsep(&input, " \t\n"))) {
2373                 substring_t args[MAX_OPT_ARGS];
2374                 char buf[32];
2375                 int tok;
2376                 u64 v;
2377
2378                 if (!*p)
2379                         continue;
2380
2381                 switch (match_token(p, cost_ctrl_tokens, args)) {
2382                 case COST_CTRL:
2383                         match_strlcpy(buf, &args[0], sizeof(buf));
2384                         if (!strcmp(buf, "auto"))
2385                                 user = false;
2386                         else if (!strcmp(buf, "user"))
2387                                 user = true;
2388                         else
2389                                 goto einval;
2390                         continue;
2391                 case COST_MODEL:
2392                         match_strlcpy(buf, &args[0], sizeof(buf));
2393                         if (strcmp(buf, "linear"))
2394                                 goto einval;
2395                         continue;
2396                 }
2397
2398                 tok = match_token(p, i_lcoef_tokens, args);
2399                 if (tok == NR_I_LCOEFS)
2400                         goto einval;
2401                 if (match_u64(&args[0], &v))
2402                         goto einval;
2403                 u[tok] = v;
2404                 user = true;
2405         }
2406
2407         spin_lock_irq(&ioc->lock);
2408         if (user) {
2409                 memcpy(ioc->params.i_lcoefs, u, sizeof(u));
2410                 ioc->user_cost_model = true;
2411         } else {
2412                 ioc->user_cost_model = false;
2413         }
2414         ioc_refresh_params(ioc, true);
2415         spin_unlock_irq(&ioc->lock);
2416
2417         put_disk_and_module(disk);
2418         return nbytes;
2419
2420 einval:
2421         ret = -EINVAL;
2422 err:
2423         put_disk_and_module(disk);
2424         return ret;
2425 }
2426
2427 static struct cftype ioc_files[] = {
2428         {
2429                 .name = "weight",
2430                 .flags = CFTYPE_NOT_ON_ROOT,
2431                 .seq_show = ioc_weight_show,
2432                 .write = ioc_weight_write,
2433         },
2434         {
2435                 .name = "cost.qos",
2436                 .flags = CFTYPE_ONLY_ON_ROOT,
2437                 .seq_show = ioc_qos_show,
2438                 .write = ioc_qos_write,
2439         },
2440         {
2441                 .name = "cost.model",
2442                 .flags = CFTYPE_ONLY_ON_ROOT,
2443                 .seq_show = ioc_cost_model_show,
2444                 .write = ioc_cost_model_write,
2445         },
2446         {}
2447 };
2448
2449 static struct blkcg_policy blkcg_policy_iocost = {
2450         .dfl_cftypes    = ioc_files,
2451         .cpd_alloc_fn   = ioc_cpd_alloc,
2452         .cpd_free_fn    = ioc_cpd_free,
2453         .pd_alloc_fn    = ioc_pd_alloc,
2454         .pd_init_fn     = ioc_pd_init,
2455         .pd_free_fn     = ioc_pd_free,
2456 };
2457
2458 static int __init ioc_init(void)
2459 {
2460         return blkcg_policy_register(&blkcg_policy_iocost);
2461 }
2462
2463 static void __exit ioc_exit(void)
2464 {
2465         return blkcg_policy_unregister(&blkcg_policy_iocost);
2466 }
2467
2468 module_init(ioc_init);
2469 module_exit(ioc_exit);