dm kcopyd: introduce configurable throttling

This patch allows the administrator to reduce the rate at which kcopyd
issues I/O.

Each module that uses kcopyd acquires a throttle parameter that can be
set in /sys/module/*/parameters.

We maintain a history of kcopyd usage by each module in the variables
io_period and total_period in struct dm_kcopyd_throttle. The actual
kcopyd activity is calculated as a percentage of time equal to
"(100 * io_period / total_period)".  This is compared with the user-defined
throttle percentage threshold and if it is exceeded, we sleep.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Alasdair G Kergon <agk@redhat.com>

diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c
index 68c02673263ba87b9749cda3e119838e75edf22a..d581fe5d2faf1df83f1fce5725926c8619d7e7aa 100644 (file)
--- a/drivers/md/dm-kcopyd.c
+++ b/drivers/md/dm-kcopyd.c
@@ -22,6 +22,7 @@
 #include <linux/vmalloc.h>
 #include <linux/workqueue.h>
 #include <linux/mutex.h>
 #include <linux/vmalloc.h>
 #include <linux/workqueue.h>
 #include <linux/mutex.h>

+#include <linux/delay.h>

 #include <linux/device-mapper.h>
 #include <linux/dm-kcopyd.h>
 
 #include <linux/device-mapper.h>
 #include <linux/dm-kcopyd.h>
 


@@ -51,6 +52,8 @@ struct dm_kcopyd_client {
        struct workqueue_struct *kcopyd_wq;
        struct work_struct kcopyd_work;
 
        struct workqueue_struct *kcopyd_wq;
        struct work_struct kcopyd_work;
 

+       struct dm_kcopyd_throttle *throttle;
+

 /*
  * We maintain three lists of jobs:
  *
 /*
  * We maintain three lists of jobs:
  *


@@ -68,6 +71,117 @@ struct dm_kcopyd_client {
 
 static struct page_list zero_page_list;
 
 
 static struct page_list zero_page_list;
 

+static DEFINE_SPINLOCK(throttle_spinlock);
+
+/*
+ * IO/IDLE accounting slowly decays after (1 << ACCOUNT_INTERVAL_SHIFT) period.
+ * When total_period >= (1 << ACCOUNT_INTERVAL_SHIFT) the counters are divided
+ * by 2.
+ */
+#define ACCOUNT_INTERVAL_SHIFT         SHIFT_HZ
+
+/*
+ * Sleep this number of milliseconds.
+ *
+ * The value was decided experimentally.
+ * Smaller values seem to cause an increased copy rate above the limit.
+ * The reason for this is unknown but possibly due to jiffies rounding errors
+ * or read/write cache inside the disk.
+ */
+#define SLEEP_MSEC                     100
+
+/*
+ * Maximum number of sleep events. There is a theoretical livelock if more
+ * kcopyd clients do work simultaneously which this limit avoids.
+ */
+#define MAX_SLEEPS                     10
+
+static void io_job_start(struct dm_kcopyd_throttle *t)
+{
+       unsigned throttle, now, difference;
+       int slept = 0, skew;
+
+       if (unlikely(!t))
+               return;
+
+try_again:
+       spin_lock_irq(&throttle_spinlock);
+
+       throttle = ACCESS_ONCE(t->throttle);
+
+       if (likely(throttle >= 100))
+               goto skip_limit;
+
+       now = jiffies;
+       difference = now - t->last_jiffies;
+       t->last_jiffies = now;
+       if (t->num_io_jobs)
+               t->io_period += difference;
+       t->total_period += difference;
+
+       /*
+        * Maintain sane values if we got a temporary overflow.
+        */
+       if (unlikely(t->io_period > t->total_period))
+               t->io_period = t->total_period;
+
+       if (unlikely(t->total_period >= (1 << ACCOUNT_INTERVAL_SHIFT))) {
+               int shift = fls(t->total_period >> ACCOUNT_INTERVAL_SHIFT);
+               t->total_period >>= shift;
+               t->io_period >>= shift;
+       }
+
+       skew = t->io_period - throttle * t->total_period / 100;
+
+       if (unlikely(skew > 0) && slept < MAX_SLEEPS) {
+               slept++;
+               spin_unlock_irq(&throttle_spinlock);
+               msleep(SLEEP_MSEC);
+               goto try_again;
+       }
+
+skip_limit:
+       t->num_io_jobs++;
+
+       spin_unlock_irq(&throttle_spinlock);
+}
+
+static void io_job_finish(struct dm_kcopyd_throttle *t)
+{
+       unsigned long flags;
+
+       if (unlikely(!t))
+               return;
+
+       spin_lock_irqsave(&throttle_spinlock, flags);
+
+       t->num_io_jobs--;
+
+       if (likely(ACCESS_ONCE(t->throttle) >= 100))
+               goto skip_limit;
+
+       if (!t->num_io_jobs) {
+               unsigned now, difference;
+
+               now = jiffies;
+               difference = now - t->last_jiffies;
+               t->last_jiffies = now;
+
+               t->io_period += difference;
+               t->total_period += difference;
+
+               /*
+                * Maintain sane values if we got a temporary overflow.
+                */
+               if (unlikely(t->io_period > t->total_period))
+                       t->io_period = t->total_period;
+       }
+
+skip_limit:
+       spin_unlock_irqrestore(&throttle_spinlock, flags);
+}
+
+

 static void wake(struct dm_kcopyd_client *kc)
 {
        queue_work(kc->kcopyd_wq, &kc->kcopyd_work);
 static void wake(struct dm_kcopyd_client *kc)
 {
        queue_work(kc->kcopyd_wq, &kc->kcopyd_work);


@@ -348,6 +462,8 @@ static void complete_io(unsigned long error, void *context)
        struct kcopyd_job *job = (struct kcopyd_job *) context;
        struct dm_kcopyd_client *kc = job->kc;
 
        struct kcopyd_job *job = (struct kcopyd_job *) context;
        struct dm_kcopyd_client *kc = job->kc;
 

+       io_job_finish(kc->throttle);
+

        if (error) {
                if (job->rw & WRITE)
                        job->write_err |= error;
        if (error) {
                if (job->rw & WRITE)
                        job->write_err |= error;


@@ -389,6 +505,8 @@ static int run_io_job(struct kcopyd_job *job)
                .client = job->kc->io_client,
        };
 
                .client = job->kc->io_client,
        };
 

+       io_job_start(job->kc->throttle);
+

        if (job->rw == READ)
                r = dm_io(&io_req, 1, &job->source, NULL);
        else
        if (job->rw == READ)
                r = dm_io(&io_req, 1, &job->source, NULL);
        else


@@ -695,7 +813,7 @@ int kcopyd_cancel(struct kcopyd_job *job, int block)
 /*-----------------------------------------------------------------
  * Client setup
  *---------------------------------------------------------------*/
 /*-----------------------------------------------------------------
  * Client setup
  *---------------------------------------------------------------*/

-struct dm_kcopyd_client *dm_kcopyd_client_create(void)
+struct dm_kcopyd_client *dm_kcopyd_client_create(struct dm_kcopyd_throttle *throttle)

 {
        int r = -ENOMEM;
        struct dm_kcopyd_client *kc;
 {
        int r = -ENOMEM;
        struct dm_kcopyd_client *kc;


@@ -708,6 +826,7 @@ struct dm_kcopyd_client *dm_kcopyd_client_create(void)
        INIT_LIST_HEAD(&kc->complete_jobs);
        INIT_LIST_HEAD(&kc->io_jobs);
        INIT_LIST_HEAD(&kc->pages_jobs);
        INIT_LIST_HEAD(&kc->complete_jobs);
        INIT_LIST_HEAD(&kc->io_jobs);
        INIT_LIST_HEAD(&kc->pages_jobs);

+       kc->throttle = throttle;

 
        kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache);
        if (!kc->job_pool)
 
        kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache);
        if (!kc->job_pool)

diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index e2ea97723e109c7e7ec6122b5dd04ad02b3c8b38..d053098c6a917f1ac55fec457ab7c7f590ecb1eb 100644 (file)
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -82,6 +82,9 @@ struct mirror_set {
        struct mirror mirror[0];
 };
 
        struct mirror mirror[0];
 };
 

+DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(raid1_resync_throttle,
+               "A percentage of time allocated for raid resynchronization");
+

 static void wakeup_mirrord(void *context)
 {
        struct mirror_set *ms = context;
 static void wakeup_mirrord(void *context)
 {
        struct mirror_set *ms = context;


@@ -1111,7 +1114,7 @@ static int mirror_ctr(struct dm_target *ti, unsigned int argc, char **argv)
                goto err_destroy_wq;
        }
 
                goto err_destroy_wq;
        }
 

-       ms->kcopyd_client = dm_kcopyd_client_create();
+       ms->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle);

        if (IS_ERR(ms->kcopyd_client)) {
                r = PTR_ERR(ms->kcopyd_client);
                goto err_destroy_wq;
        if (IS_ERR(ms->kcopyd_client)) {
                r = PTR_ERR(ms->kcopyd_client);
                goto err_destroy_wq;

diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index 58c2b58813777074bc35d3c6ff79cd1cebc7db00..c0e07026a8d136f084958800192c12236ffd37e2 100644 (file)
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -124,6 +124,9 @@ struct dm_snapshot {
 #define RUNNING_MERGE          0
 #define SHUTDOWN_MERGE         1
 
 #define RUNNING_MERGE          0
 #define SHUTDOWN_MERGE         1
 

+DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
+               "A percentage of time allocated for copy on write");
+

 struct dm_dev *dm_snap_origin(struct dm_snapshot *s)
 {
        return s->origin;
 struct dm_dev *dm_snap_origin(struct dm_snapshot *s)
 {
        return s->origin;


@@ -1108,7 +1111,7 @@ static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
                goto bad_hash_tables;
        }
 
                goto bad_hash_tables;
        }
 

-       s->kcopyd_client = dm_kcopyd_client_create();
+       s->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle);

        if (IS_ERR(s->kcopyd_client)) {
                r = PTR_ERR(s->kcopyd_client);
                ti->error = "Could not create kcopyd client";
        if (IS_ERR(s->kcopyd_client)) {
                r = PTR_ERR(s->kcopyd_client);
                ti->error = "Could not create kcopyd client";

diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index 303e11da7c2a5e5af1f3761efd99bdb1deea4569..35d9d0396cc23252e1b8aab56adb5948a24a2329 100644 (file)
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -26,6 +26,9 @@
 #define PRISON_CELLS 1024
 #define COMMIT_PERIOD HZ
 
 #define PRISON_CELLS 1024
 #define COMMIT_PERIOD HZ
 

+DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
+               "A percentage of time allocated for copy on write");
+

 /*
  * The block size of the device holding pool data must be
  * between 64KB and 1GB.
 /*
  * The block size of the device holding pool data must be
  * between 64KB and 1GB.


@@ -1642,7 +1645,7 @@ static struct pool *pool_create(struct mapped_device *pool_md,
                goto bad_prison;
        }
 
                goto bad_prison;
        }
 

-       pool->copier = dm_kcopyd_client_create();
+       pool->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);

        if (IS_ERR(pool->copier)) {
                r = PTR_ERR(pool->copier);
                *error = "Error creating pool's kcopyd client";
        if (IS_ERR(pool->copier)) {
                r = PTR_ERR(pool->copier);
                *error = "Error creating pool's kcopyd client";

diff --git a/include/linux/dm-kcopyd.h b/include/linux/dm-kcopyd.h
index 47d9d376e4e7a9e0cecb609065b3d735825cfdb9..f486d636b82eebef5396c0af33d423831bc82b62 100644 (file)
--- a/include/linux/dm-kcopyd.h
+++ b/include/linux/dm-kcopyd.h
@@ -21,11 +21,34 @@
 
 #define DM_KCOPYD_IGNORE_ERROR 1
 
 
 #define DM_KCOPYD_IGNORE_ERROR 1
 

+struct dm_kcopyd_throttle {
+       unsigned throttle;
+       unsigned num_io_jobs;
+       unsigned io_period;
+       unsigned total_period;
+       unsigned last_jiffies;
+};
+
+/*
+ * kcopyd clients that want to support throttling must pass an initialised
+ * dm_kcopyd_throttle struct into dm_kcopyd_client_create().
+ * Two or more clients may share the same instance of this struct between
+ * them if they wish to be throttled as a group.
+ *
+ * This macro also creates a corresponding module parameter to configure
+ * the amount of throttling.
+ */
+#define DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(name, description) \
+static struct dm_kcopyd_throttle dm_kcopyd_throttle = { 100, 0, 0, 0, 0 }; \
+module_param_named(name, dm_kcopyd_throttle.throttle, uint, 0644); \
+MODULE_PARM_DESC(name, description)
+

 /*
  * To use kcopyd you must first create a dm_kcopyd_client object.
 /*
  * To use kcopyd you must first create a dm_kcopyd_client object.

+ * throttle can be NULL if you don't want any throttling.

  */
 struct dm_kcopyd_client;
  */
 struct dm_kcopyd_client;

-struct dm_kcopyd_client *dm_kcopyd_client_create(void);
+struct dm_kcopyd_client *dm_kcopyd_client_create(struct dm_kcopyd_throttle *throttle);

 void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc);
 
 /*
 void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc);
 
 /*