4 * Glue code for the SHA256 Secure Hash Algorithm assembler
5 * implementation using supplemental SSE3 / AVX / AVX2 instructions.
7 * This file is based on sha256_generic.c
9 * Copyright (C) 2013 Intel Corporation.
12 * Tim Chen <tim.c.chen@linux.intel.com>
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU General Public License as published by the Free
16 * Software Foundation; either version 2 of the License, or (at your option)
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
23 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
24 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
32 #include <crypto/internal/hash.h>
33 #include <crypto/internal/simd.h>
34 #include <linux/init.h>
35 #include <linux/module.h>
37 #include <linux/types.h>
38 #include <crypto/sha2.h>
39 #include <crypto/sha256_base.h>
40 #include <linux/string.h>
41 #include <asm/cpu_device_id.h>
44 asmlinkage void sha256_transform_ssse3(struct sha256_state *state,
45 const u8 *data, int blocks);
47 static const struct x86_cpu_id module_cpu_ids[] = {
48 X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL),
49 X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL),
50 X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL),
53 MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids);
55 static int _sha256_update(struct shash_desc *desc, const u8 *data,
56 unsigned int len, sha256_block_fn *sha256_xform)
58 struct sha256_state *sctx = shash_desc_ctx(desc);
60 if (!crypto_simd_usable() ||
61 (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
62 return crypto_sha256_update(desc, data, len);
65 * Make sure struct sha256_state begins directly with the SHA256
66 * 256-bit internal state, as this is what the asm functions expect.
68 BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
71 sha256_base_do_update(desc, data, len, sha256_xform);
77 static int sha256_finup(struct shash_desc *desc, const u8 *data,
78 unsigned int len, u8 *out, sha256_block_fn *sha256_xform)
80 if (!crypto_simd_usable())
81 return crypto_sha256_finup(desc, data, len, out);
85 sha256_base_do_update(desc, data, len, sha256_xform);
86 sha256_base_do_finalize(desc, sha256_xform);
89 return sha256_base_finish(desc, out);
92 static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
95 return _sha256_update(desc, data, len, sha256_transform_ssse3);
98 static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
99 unsigned int len, u8 *out)
101 return sha256_finup(desc, data, len, out, sha256_transform_ssse3);
104 /* Add padding and return the message digest. */
105 static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
107 return sha256_ssse3_finup(desc, NULL, 0, out);
110 static struct shash_alg sha256_ssse3_algs[] = { {
111 .digestsize = SHA256_DIGEST_SIZE,
112 .init = sha256_base_init,
113 .update = sha256_ssse3_update,
114 .final = sha256_ssse3_final,
115 .finup = sha256_ssse3_finup,
116 .descsize = sizeof(struct sha256_state),
118 .cra_name = "sha256",
119 .cra_driver_name = "sha256-ssse3",
121 .cra_blocksize = SHA256_BLOCK_SIZE,
122 .cra_module = THIS_MODULE,
125 .digestsize = SHA224_DIGEST_SIZE,
126 .init = sha224_base_init,
127 .update = sha256_ssse3_update,
128 .final = sha256_ssse3_final,
129 .finup = sha256_ssse3_finup,
130 .descsize = sizeof(struct sha256_state),
132 .cra_name = "sha224",
133 .cra_driver_name = "sha224-ssse3",
135 .cra_blocksize = SHA224_BLOCK_SIZE,
136 .cra_module = THIS_MODULE,
140 static int register_sha256_ssse3(void)
142 if (boot_cpu_has(X86_FEATURE_SSSE3))
143 return crypto_register_shashes(sha256_ssse3_algs,
144 ARRAY_SIZE(sha256_ssse3_algs));
148 static void unregister_sha256_ssse3(void)
150 if (boot_cpu_has(X86_FEATURE_SSSE3))
151 crypto_unregister_shashes(sha256_ssse3_algs,
152 ARRAY_SIZE(sha256_ssse3_algs));
155 asmlinkage void sha256_transform_avx(struct sha256_state *state,
156 const u8 *data, int blocks);
158 static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
161 return _sha256_update(desc, data, len, sha256_transform_avx);
164 static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
165 unsigned int len, u8 *out)
167 return sha256_finup(desc, data, len, out, sha256_transform_avx);
170 static int sha256_avx_final(struct shash_desc *desc, u8 *out)
172 return sha256_avx_finup(desc, NULL, 0, out);
175 static struct shash_alg sha256_avx_algs[] = { {
176 .digestsize = SHA256_DIGEST_SIZE,
177 .init = sha256_base_init,
178 .update = sha256_avx_update,
179 .final = sha256_avx_final,
180 .finup = sha256_avx_finup,
181 .descsize = sizeof(struct sha256_state),
183 .cra_name = "sha256",
184 .cra_driver_name = "sha256-avx",
186 .cra_blocksize = SHA256_BLOCK_SIZE,
187 .cra_module = THIS_MODULE,
190 .digestsize = SHA224_DIGEST_SIZE,
191 .init = sha224_base_init,
192 .update = sha256_avx_update,
193 .final = sha256_avx_final,
194 .finup = sha256_avx_finup,
195 .descsize = sizeof(struct sha256_state),
197 .cra_name = "sha224",
198 .cra_driver_name = "sha224-avx",
200 .cra_blocksize = SHA224_BLOCK_SIZE,
201 .cra_module = THIS_MODULE,
205 static bool avx_usable(void)
207 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
208 if (boot_cpu_has(X86_FEATURE_AVX))
209 pr_info("AVX detected but unusable.\n");
216 static int register_sha256_avx(void)
219 return crypto_register_shashes(sha256_avx_algs,
220 ARRAY_SIZE(sha256_avx_algs));
224 static void unregister_sha256_avx(void)
227 crypto_unregister_shashes(sha256_avx_algs,
228 ARRAY_SIZE(sha256_avx_algs));
231 asmlinkage void sha256_transform_rorx(struct sha256_state *state,
232 const u8 *data, int blocks);
234 static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
237 return _sha256_update(desc, data, len, sha256_transform_rorx);
240 static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
241 unsigned int len, u8 *out)
243 return sha256_finup(desc, data, len, out, sha256_transform_rorx);
246 static int sha256_avx2_final(struct shash_desc *desc, u8 *out)
248 return sha256_avx2_finup(desc, NULL, 0, out);
251 static struct shash_alg sha256_avx2_algs[] = { {
252 .digestsize = SHA256_DIGEST_SIZE,
253 .init = sha256_base_init,
254 .update = sha256_avx2_update,
255 .final = sha256_avx2_final,
256 .finup = sha256_avx2_finup,
257 .descsize = sizeof(struct sha256_state),
259 .cra_name = "sha256",
260 .cra_driver_name = "sha256-avx2",
262 .cra_blocksize = SHA256_BLOCK_SIZE,
263 .cra_module = THIS_MODULE,
266 .digestsize = SHA224_DIGEST_SIZE,
267 .init = sha224_base_init,
268 .update = sha256_avx2_update,
269 .final = sha256_avx2_final,
270 .finup = sha256_avx2_finup,
271 .descsize = sizeof(struct sha256_state),
273 .cra_name = "sha224",
274 .cra_driver_name = "sha224-avx2",
276 .cra_blocksize = SHA224_BLOCK_SIZE,
277 .cra_module = THIS_MODULE,
281 static bool avx2_usable(void)
283 if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
284 boot_cpu_has(X86_FEATURE_BMI2))
290 static int register_sha256_avx2(void)
293 return crypto_register_shashes(sha256_avx2_algs,
294 ARRAY_SIZE(sha256_avx2_algs));
298 static void unregister_sha256_avx2(void)
301 crypto_unregister_shashes(sha256_avx2_algs,
302 ARRAY_SIZE(sha256_avx2_algs));
305 #ifdef CONFIG_AS_SHA256_NI
306 asmlinkage void sha256_ni_transform(struct sha256_state *digest,
307 const u8 *data, int rounds);
309 static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
312 return _sha256_update(desc, data, len, sha256_ni_transform);
315 static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
316 unsigned int len, u8 *out)
318 return sha256_finup(desc, data, len, out, sha256_ni_transform);
321 static int sha256_ni_final(struct shash_desc *desc, u8 *out)
323 return sha256_ni_finup(desc, NULL, 0, out);
326 static struct shash_alg sha256_ni_algs[] = { {
327 .digestsize = SHA256_DIGEST_SIZE,
328 .init = sha256_base_init,
329 .update = sha256_ni_update,
330 .final = sha256_ni_final,
331 .finup = sha256_ni_finup,
332 .descsize = sizeof(struct sha256_state),
334 .cra_name = "sha256",
335 .cra_driver_name = "sha256-ni",
337 .cra_blocksize = SHA256_BLOCK_SIZE,
338 .cra_module = THIS_MODULE,
341 .digestsize = SHA224_DIGEST_SIZE,
342 .init = sha224_base_init,
343 .update = sha256_ni_update,
344 .final = sha256_ni_final,
345 .finup = sha256_ni_finup,
346 .descsize = sizeof(struct sha256_state),
348 .cra_name = "sha224",
349 .cra_driver_name = "sha224-ni",
351 .cra_blocksize = SHA224_BLOCK_SIZE,
352 .cra_module = THIS_MODULE,
356 static int register_sha256_ni(void)
358 if (boot_cpu_has(X86_FEATURE_SHA_NI))
359 return crypto_register_shashes(sha256_ni_algs,
360 ARRAY_SIZE(sha256_ni_algs));
364 static void unregister_sha256_ni(void)
366 if (boot_cpu_has(X86_FEATURE_SHA_NI))
367 crypto_unregister_shashes(sha256_ni_algs,
368 ARRAY_SIZE(sha256_ni_algs));
372 static inline int register_sha256_ni(void) { return 0; }
373 static inline void unregister_sha256_ni(void) { }
376 static int __init sha256_ssse3_mod_init(void)
378 if (!x86_match_cpu(module_cpu_ids))
381 if (register_sha256_ssse3())
384 if (register_sha256_avx()) {
385 unregister_sha256_ssse3();
389 if (register_sha256_avx2()) {
390 unregister_sha256_avx();
391 unregister_sha256_ssse3();
395 if (register_sha256_ni()) {
396 unregister_sha256_avx2();
397 unregister_sha256_avx();
398 unregister_sha256_ssse3();
407 static void __exit sha256_ssse3_mod_fini(void)
409 unregister_sha256_ni();
410 unregister_sha256_avx2();
411 unregister_sha256_avx();
412 unregister_sha256_ssse3();
415 module_init(sha256_ssse3_mod_init);
416 module_exit(sha256_ssse3_mod_fini);
418 MODULE_LICENSE("GPL");
419 MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
421 MODULE_ALIAS_CRYPTO("sha256");
422 MODULE_ALIAS_CRYPTO("sha256-ssse3");
423 MODULE_ALIAS_CRYPTO("sha256-avx");
424 MODULE_ALIAS_CRYPTO("sha256-avx2");
425 MODULE_ALIAS_CRYPTO("sha224");
426 MODULE_ALIAS_CRYPTO("sha224-ssse3");
427 MODULE_ALIAS_CRYPTO("sha224-avx");
428 MODULE_ALIAS_CRYPTO("sha224-avx2");
429 #ifdef CONFIG_AS_SHA256_NI
430 MODULE_ALIAS_CRYPTO("sha256-ni");
431 MODULE_ALIAS_CRYPTO("sha224-ni");