projects / openwrt / staging / mans0n.git / blob
? search:


01037a6ee6a940200c46009b5d3294b04d9b5643
[openwrt/staging/mans0n.git] /
1 From 2ceb2e26de65cce974875e0487dde20bc5f1826c Mon Sep 17 00:00:00 2001
2 From: Ard Biesheuvel <ardb@kernel.org>
3 Date: Fri, 8 Nov 2019 13:22:23 +0100
4 Subject: [PATCH 017/124] crypto: x86/poly1305 - expose existing driver as
5 poly1305 library
6
7 commit f0e89bcfbb894e5844cd1bbf6b3cf7c63cb0f5ac upstream.
8
9 Implement the arch init/update/final Poly1305 library routines in the
10 accelerated SIMD driver for x86 so they are accessible to users of
11 the Poly1305 library interface as well.
12
13 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
14 Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
15 Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
16 ---
17 arch/x86/crypto/poly1305_glue.c | 57 ++++++++++++++++++++++++---------
18 crypto/Kconfig | 1 +
19 lib/crypto/Kconfig | 1 +
20 3 files changed, 43 insertions(+), 16 deletions(-)
21
22 --- a/arch/x86/crypto/poly1305_glue.c
23 +++ b/arch/x86/crypto/poly1305_glue.c
24 @@ -10,6 +10,7 @@
25 #include <crypto/internal/poly1305.h>
26 #include <crypto/internal/simd.h>
27 #include <linux/crypto.h>
28 +#include <linux/jump_label.h>
29 #include <linux/kernel.h>
30 #include <linux/module.h>
31 #include <asm/simd.h>
32 @@ -21,7 +22,8 @@ asmlinkage void poly1305_2block_sse2(u32
33 asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r,
34 unsigned int blocks, const u32 *u);
35
36 -static bool poly1305_use_avx2 __ro_after_init;
37 +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_simd);
38 +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2);
39
40 static void poly1305_simd_mult(u32 *a, const u32 *b)
41 {
42 @@ -64,7 +66,7 @@ static unsigned int poly1305_simd_blocks
43 }
44
45 if (IS_ENABLED(CONFIG_AS_AVX2) &&
46 - poly1305_use_avx2 &&
47 + static_branch_likely(&poly1305_use_avx2) &&
48 srclen >= POLY1305_BLOCK_SIZE * 4) {
49 if (unlikely(dctx->rset < 4)) {
50 if (dctx->rset < 2) {
51 @@ -103,10 +105,15 @@ static unsigned int poly1305_simd_blocks
52 return srclen;
53 }
54
55 -static int poly1305_simd_update(struct shash_desc *desc,
56 - const u8 *src, unsigned int srclen)
57 +void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key)
58 +{
59 + poly1305_init_generic(desc, key);
60 +}
61 +EXPORT_SYMBOL(poly1305_init_arch);
62 +
63 +void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src,
64 + unsigned int srclen)
65 {
66 - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
67 unsigned int bytes;
68
69 if (unlikely(dctx->buflen)) {
70 @@ -117,7 +124,8 @@ static int poly1305_simd_update(struct s
71 dctx->buflen += bytes;
72
73 if (dctx->buflen == POLY1305_BLOCK_SIZE) {
74 - if (likely(crypto_simd_usable())) {
75 + if (static_branch_likely(&poly1305_use_simd) &&
76 + likely(crypto_simd_usable())) {
77 kernel_fpu_begin();
78 poly1305_simd_blocks(dctx, dctx->buf,
79 POLY1305_BLOCK_SIZE);
80 @@ -131,7 +139,8 @@ static int poly1305_simd_update(struct s
81 }
82
83 if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
84 - if (likely(crypto_simd_usable())) {
85 + if (static_branch_likely(&poly1305_use_simd) &&
86 + likely(crypto_simd_usable())) {
87 kernel_fpu_begin();
88 bytes = poly1305_simd_blocks(dctx, src, srclen);
89 kernel_fpu_end();
90 @@ -147,6 +156,13 @@ static int poly1305_simd_update(struct s
91 memcpy(dctx->buf, src, srclen);
92 }
93 }
94 +EXPORT_SYMBOL(poly1305_update_arch);
95 +
96 +void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest)
97 +{
98 + poly1305_final_generic(desc, digest);
99 +}
100 +EXPORT_SYMBOL(poly1305_final_arch);
101
102 static int crypto_poly1305_init(struct shash_desc *desc)
103 {
104 @@ -171,6 +187,15 @@ static int crypto_poly1305_final(struct
105 return 0;
106 }
107
108 +static int poly1305_simd_update(struct shash_desc *desc,
109 + const u8 *src, unsigned int srclen)
110 +{
111 + struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
112 +
113 + poly1305_update_arch(dctx, src, srclen);
114 + return 0;
115 +}
116 +
117 static struct shash_alg alg = {
118 .digestsize = POLY1305_DIGEST_SIZE,
119 .init = crypto_poly1305_init,
120 @@ -189,15 +214,15 @@ static struct shash_alg alg = {
121 static int __init poly1305_simd_mod_init(void)
122 {
123 if (!boot_cpu_has(X86_FEATURE_XMM2))
124 - return -ENODEV;
125 + return 0;
126
127 - poly1305_use_avx2 = IS_ENABLED(CONFIG_AS_AVX2) &&
128 - boot_cpu_has(X86_FEATURE_AVX) &&
129 - boot_cpu_has(X86_FEATURE_AVX2) &&
130 - cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
131 - alg.descsize = sizeof(struct poly1305_desc_ctx) + 5 * sizeof(u32);
132 - if (poly1305_use_avx2)
133 - alg.descsize += 10 * sizeof(u32);
134 + static_branch_enable(&poly1305_use_simd);
135 +
136 + if (IS_ENABLED(CONFIG_AS_AVX2) &&
137 + boot_cpu_has(X86_FEATURE_AVX) &&
138 + boot_cpu_has(X86_FEATURE_AVX2) &&
139 + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
140 + static_branch_enable(&poly1305_use_avx2);
141
142 return crypto_register_shash(&alg);
143 }
144 --- a/crypto/Kconfig
145 +++ b/crypto/Kconfig
146 @@ -698,6 +698,7 @@ config CRYPTO_POLY1305_X86_64
147 tristate "Poly1305 authenticator algorithm (x86_64/SSE2/AVX2)"
148 depends on X86 && 64BIT
149 select CRYPTO_LIB_POLY1305_GENERIC
150 + select CRYPTO_ARCH_HAVE_LIB_POLY1305
151 help
152 Poly1305 authenticator algorithm, RFC7539.
153
154 --- a/lib/crypto/Kconfig
155 +++ b/lib/crypto/Kconfig
156 @@ -39,6 +39,7 @@ config CRYPTO_LIB_DES
157
158 config CRYPTO_LIB_POLY1305_RSIZE
159 int
160 + default 4 if X86_64
161 default 1
162
163 config CRYPTO_ARCH_HAVE_LIB_POLY1305
Staging tree of Sungbo Eo