dm crypt: move cipher data out of per_cpu struct

Move static dm-crypt cipher data out of per-cpu structure.

Cipher information is static, so it does not have to be in a per-cpu
structure.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Alasdair G Kergon <agk@redhat.com>

diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index d7f0e01..711e0ac 100644 (file)
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -109,9 +109,6 @@ enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID };
  */
 struct crypt_cpu {
        struct ablkcipher_request *req;
  */
 struct crypt_cpu {
        struct ablkcipher_request *req;

-       /* ESSIV: struct crypto_cipher *essiv_tfm */
-       void *iv_private;
-       struct crypto_ablkcipher *tfms[0];

 };
 
 /*
 };
 
 /*


@@ -151,6 +148,10 @@ struct crypt_config {
         * per_cpu_ptr() only.
         */
        struct crypt_cpu __percpu *cpu;
         * per_cpu_ptr() only.
         */
        struct crypt_cpu __percpu *cpu;

+
+       /* ESSIV: struct crypto_cipher *essiv_tfm */
+       void *iv_private;
+       struct crypto_ablkcipher **tfms;

        unsigned tfms_count;
 
        /*
        unsigned tfms_count;
 
        /*


@@ -193,7 +194,7 @@ static struct crypt_cpu *this_crypt_config(struct crypt_config *cc)
  */
 static struct crypto_ablkcipher *any_tfm(struct crypt_config *cc)
 {
  */
 static struct crypto_ablkcipher *any_tfm(struct crypt_config *cc)
 {

-       return __this_cpu_ptr(cc->cpu)->tfms[0];
+       return cc->tfms[0];

 }
 
 /*
 }
 
 /*


@@ -258,7 +259,7 @@ static int crypt_iv_essiv_init(struct crypt_config *cc)
        struct hash_desc desc;
        struct scatterlist sg;
        struct crypto_cipher *essiv_tfm;
        struct hash_desc desc;
        struct scatterlist sg;
        struct crypto_cipher *essiv_tfm;

-       int err, cpu;
+       int err;

 
        sg_init_one(&sg, cc->key, cc->key_size);
        desc.tfm = essiv->hash_tfm;
 
        sg_init_one(&sg, cc->key, cc->key_size);
        desc.tfm = essiv->hash_tfm;


@@ -268,14 +269,12 @@ static int crypt_iv_essiv_init(struct crypt_config *cc)
        if (err)
                return err;
 
        if (err)
                return err;
 

-       for_each_possible_cpu(cpu) {
-               essiv_tfm = per_cpu_ptr(cc->cpu, cpu)->iv_private,
+       essiv_tfm = cc->iv_private;

 
 

-               err = crypto_cipher_setkey(essiv_tfm, essiv->salt,
-                                   crypto_hash_digestsize(essiv->hash_tfm));
-               if (err)
-                       return err;
-       }
+       err = crypto_cipher_setkey(essiv_tfm, essiv->salt,
+                           crypto_hash_digestsize(essiv->hash_tfm));
+       if (err)
+               return err;

 
        return 0;
 }
 
        return 0;
 }


@@ -286,16 +285,14 @@ static int crypt_iv_essiv_wipe(struct crypt_config *cc)
        struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
        unsigned salt_size = crypto_hash_digestsize(essiv->hash_tfm);
        struct crypto_cipher *essiv_tfm;
        struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
        unsigned salt_size = crypto_hash_digestsize(essiv->hash_tfm);
        struct crypto_cipher *essiv_tfm;

-       int cpu, r, err = 0;
+       int r, err = 0;

 
        memset(essiv->salt, 0, salt_size);
 
 
        memset(essiv->salt, 0, salt_size);
 

-       for_each_possible_cpu(cpu) {
-               essiv_tfm = per_cpu_ptr(cc->cpu, cpu)->iv_private;
-               r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size);
-               if (r)
-                       err = r;
-       }
+       essiv_tfm = cc->iv_private;
+       r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size);
+       if (r)
+               err = r;

 
        return err;
 }
 
        return err;
 }


@@ -335,8 +332,6 @@ static struct crypto_cipher *setup_essiv_cpu(struct crypt_config *cc,
 
 static void crypt_iv_essiv_dtr(struct crypt_config *cc)
 {
 
 static void crypt_iv_essiv_dtr(struct crypt_config *cc)
 {

-       int cpu;
-       struct crypt_cpu *cpu_cc;

        struct crypto_cipher *essiv_tfm;
        struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
 
        struct crypto_cipher *essiv_tfm;
        struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
 


@@ -346,15 +341,12 @@ static void crypt_iv_essiv_dtr(struct crypt_config *cc)
        kzfree(essiv->salt);
        essiv->salt = NULL;
 
        kzfree(essiv->salt);
        essiv->salt = NULL;
 

-       for_each_possible_cpu(cpu) {
-               cpu_cc = per_cpu_ptr(cc->cpu, cpu);
-               essiv_tfm = cpu_cc->iv_private;
+       essiv_tfm = cc->iv_private;

 
 

-               if (essiv_tfm)
-                       crypto_free_cipher(essiv_tfm);
+       if (essiv_tfm)
+               crypto_free_cipher(essiv_tfm);

 
 

-               cpu_cc->iv_private = NULL;
-       }
+       cc->iv_private = NULL;

 }
 
 static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
 }
 
 static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,


@@ -363,7 +355,7 @@ static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
        struct crypto_cipher *essiv_tfm = NULL;
        struct crypto_hash *hash_tfm = NULL;
        u8 *salt = NULL;
        struct crypto_cipher *essiv_tfm = NULL;
        struct crypto_hash *hash_tfm = NULL;
        u8 *salt = NULL;

-       int err, cpu;
+       int err;

 
        if (!opts) {
                ti->error = "Digest algorithm missing for ESSIV mode";
 
        if (!opts) {
                ti->error = "Digest algorithm missing for ESSIV mode";


@@ -388,15 +380,13 @@ static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
        cc->iv_gen_private.essiv.salt = salt;
        cc->iv_gen_private.essiv.hash_tfm = hash_tfm;
 
        cc->iv_gen_private.essiv.salt = salt;
        cc->iv_gen_private.essiv.hash_tfm = hash_tfm;
 

-       for_each_possible_cpu(cpu) {
-               essiv_tfm = setup_essiv_cpu(cc, ti, salt,
-                                       crypto_hash_digestsize(hash_tfm));
-               if (IS_ERR(essiv_tfm)) {
-                       crypt_iv_essiv_dtr(cc);
-                       return PTR_ERR(essiv_tfm);
-               }
-               per_cpu_ptr(cc->cpu, cpu)->iv_private = essiv_tfm;
+       essiv_tfm = setup_essiv_cpu(cc, ti, salt,
+                               crypto_hash_digestsize(hash_tfm));
+       if (IS_ERR(essiv_tfm)) {
+               crypt_iv_essiv_dtr(cc);
+               return PTR_ERR(essiv_tfm);

        }
        }

+       cc->iv_private = essiv_tfm;

 
        return 0;
 
 
        return 0;
 


@@ -410,7 +400,7 @@ bad:
 static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
                              struct dm_crypt_request *dmreq)
 {
 static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
                              struct dm_crypt_request *dmreq)
 {

-       struct crypto_cipher *essiv_tfm = this_crypt_config(cc)->iv_private;
+       struct crypto_cipher *essiv_tfm = cc->iv_private;

 
        memset(iv, 0, cc->iv_size);
        *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
 
        memset(iv, 0, cc->iv_size);
        *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);


@@ -754,7 +744,7 @@ static void crypt_alloc_req(struct crypt_config *cc,
        if (!this_cc->req)
                this_cc->req = mempool_alloc(cc->req_pool, GFP_NOIO);
 
        if (!this_cc->req)
                this_cc->req = mempool_alloc(cc->req_pool, GFP_NOIO);
 

-       ablkcipher_request_set_tfm(this_cc->req, this_cc->tfms[key_index]);
+       ablkcipher_request_set_tfm(this_cc->req, cc->tfms[key_index]);

        ablkcipher_request_set_callback(this_cc->req,
            CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
            kcryptd_async_done, dmreq_of_req(cc, this_cc->req));
        ablkcipher_request_set_callback(this_cc->req,
            CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
            kcryptd_async_done, dmreq_of_req(cc, this_cc->req));


@@ -1273,29 +1263,38 @@ static void crypt_encode_key(char *hex, u8 *key, unsigned int size)
        }
 }
 
        }
 }
 

-static void crypt_free_tfms(struct crypt_config *cc, int cpu)
+static void crypt_free_tfms(struct crypt_config *cc)

 {
 {

-       struct crypt_cpu *cpu_cc = per_cpu_ptr(cc->cpu, cpu);

        unsigned i;
 
        unsigned i;
 

+       if (!cc->tfms)
+               return;
+

        for (i = 0; i < cc->tfms_count; i++)
        for (i = 0; i < cc->tfms_count; i++)

-               if (cpu_cc->tfms[i] && !IS_ERR(cpu_cc->tfms[i])) {
-                       crypto_free_ablkcipher(cpu_cc->tfms[i]);
-                       cpu_cc->tfms[i] = NULL;
+               if (cc->tfms[i] && !IS_ERR(cc->tfms[i])) {
+                       crypto_free_ablkcipher(cc->tfms[i]);
+                       cc->tfms[i] = NULL;

                }
                }

+
+       kfree(cc->tfms);
+       cc->tfms = NULL;

 }
 
 }
 

-static int crypt_alloc_tfms(struct crypt_config *cc, int cpu, char *ciphermode)
+static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)

 {
 {

-       struct crypt_cpu *cpu_cc = per_cpu_ptr(cc->cpu, cpu);

        unsigned i;
        int err;
 
        unsigned i;
        int err;
 

+       cc->tfms = kmalloc(cc->tfms_count * sizeof(struct crypto_ablkcipher *),
+                          GFP_KERNEL);
+       if (!cc->tfms)
+               return -ENOMEM;
+

        for (i = 0; i < cc->tfms_count; i++) {
        for (i = 0; i < cc->tfms_count; i++) {

-               cpu_cc->tfms[i] = crypto_alloc_ablkcipher(ciphermode, 0, 0);
-               if (IS_ERR(cpu_cc->tfms[i])) {
-                       err = PTR_ERR(cpu_cc->tfms[i]);
-                       crypt_free_tfms(cc, cpu);
+               cc->tfms[i] = crypto_alloc_ablkcipher(ciphermode, 0, 0);
+               if (IS_ERR(cc->tfms[i])) {
+                       err = PTR_ERR(cc->tfms[i]);
+                       crypt_free_tfms(cc);

                        return err;
                }
        }
                        return err;
                }
        }


@@ -1306,15 +1305,14 @@ static int crypt_alloc_tfms(struct crypt_config *cc, int cpu, char *ciphermode)
 static int crypt_setkey_allcpus(struct crypt_config *cc)
 {
        unsigned subkey_size = cc->key_size >> ilog2(cc->tfms_count);
 static int crypt_setkey_allcpus(struct crypt_config *cc)
 {
        unsigned subkey_size = cc->key_size >> ilog2(cc->tfms_count);

-       int cpu, err = 0, i, r;
-
-       for_each_possible_cpu(cpu) {
-               for (i = 0; i < cc->tfms_count; i++) {
-                       r = crypto_ablkcipher_setkey(per_cpu_ptr(cc->cpu, cpu)->tfms[i],
-                                                    cc->key + (i * subkey_size), subkey_size);
-                       if (r)
-                               err = r;
-               }
+       int err = 0, i, r;
+
+       for (i = 0; i < cc->tfms_count; i++) {
+               r = crypto_ablkcipher_setkey(cc->tfms[i],
+                                            cc->key + (i * subkey_size),
+                                            subkey_size);
+               if (r)
+                       err = r;

        }
 
        return err;
        }
 
        return err;


@@ -1376,9 +1374,10 @@ static void crypt_dtr(struct dm_target *ti)
                        cpu_cc = per_cpu_ptr(cc->cpu, cpu);
                        if (cpu_cc->req)
                                mempool_free(cpu_cc->req, cc->req_pool);
                        cpu_cc = per_cpu_ptr(cc->cpu, cpu);
                        if (cpu_cc->req)
                                mempool_free(cpu_cc->req, cc->req_pool);

-                       crypt_free_tfms(cc, cpu);

                }
 
                }
 

+       crypt_free_tfms(cc);
+

        if (cc->bs)
                bioset_free(cc->bs);
 
        if (cc->bs)
                bioset_free(cc->bs);
 


@@ -1411,7 +1410,7 @@ static int crypt_ctr_cipher(struct dm_target *ti,
        struct crypt_config *cc = ti->private;
        char *tmp, *cipher, *chainmode, *ivmode, *ivopts, *keycount;
        char *cipher_api = NULL;
        struct crypt_config *cc = ti->private;
        char *tmp, *cipher, *chainmode, *ivmode, *ivopts, *keycount;
        char *cipher_api = NULL;

-       int cpu, ret = -EINVAL;
+       int ret = -EINVAL;

        char dummy;
 
        /* Convert to crypto api definition? */
        char dummy;
 
        /* Convert to crypto api definition? */


@@ -1452,8 +1451,7 @@ static int crypt_ctr_cipher(struct dm_target *ti,
        if (tmp)
                DMWARN("Ignoring unexpected additional cipher options");
 
        if (tmp)
                DMWARN("Ignoring unexpected additional cipher options");
 

-       cc->cpu = __alloc_percpu(sizeof(*(cc->cpu)) +
-                                cc->tfms_count * sizeof(*(cc->cpu->tfms)),
+       cc->cpu = __alloc_percpu(sizeof(*(cc->cpu)),

                                 __alignof__(struct crypt_cpu));
        if (!cc->cpu) {
                ti->error = "Cannot allocate per cpu state";
                                 __alignof__(struct crypt_cpu));
        if (!cc->cpu) {
                ti->error = "Cannot allocate per cpu state";


@@ -1486,12 +1484,10 @@ static int crypt_ctr_cipher(struct dm_target *ti,
        }
 
        /* Allocate cipher */
        }
 
        /* Allocate cipher */

-       for_each_possible_cpu(cpu) {
-               ret = crypt_alloc_tfms(cc, cpu, cipher_api);
-               if (ret < 0) {
-                       ti->error = "Error allocating crypto tfm";
-                       goto bad;
-               }
+       ret = crypt_alloc_tfms(cc, cipher_api);
+       if (ret < 0) {
+               ti->error = "Error allocating crypto tfm";
+               goto bad;

        }
 
        /* Initialize and set key */
        }
 
        /* Initialize and set key */