crypto: x86 - convert to use crypto_simd_usable()

Replace all calls to irq_fpu_usable() in the x86 crypto code with
crypto_simd_usable(), in order to allow testing the no-SIMD code paths.

Signed-off-by: Eric Biggers <ebiggers@google.com>
Reviewed-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 1466d59384ac5f9f7f55c3bc59b874a38f596fd8..21c246799aa58cd7b537dc976b89a17afabf027d 100644 (file)
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -30,8 +30,8 @@
 #include <crypto/gcm.h>
 #include <crypto/xts.h>
 #include <asm/cpu_device_id.h>
-#include <asm/fpu/api.h>
 #include <asm/crypto/aes.h>
+#include <asm/simd.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/internal/aead.h>
 #include <crypto/internal/simd.h>
@@ -332,7 +332,7 @@ static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
                return -EINVAL;
        }
 
-       if (!irq_fpu_usable())
+       if (!crypto_simd_usable())
                err = crypto_aes_expand_key(ctx, in_key, key_len);
        else {
                kernel_fpu_begin();
@@ -353,7 +353,7 @@ static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 {
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 
-       if (!irq_fpu_usable())
+       if (!crypto_simd_usable())
                crypto_aes_encrypt_x86(ctx, dst, src);
        else {
                kernel_fpu_begin();
@@ -366,7 +366,7 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 {
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 
-       if (!irq_fpu_usable())
+       if (!crypto_simd_usable())
                crypto_aes_decrypt_x86(ctx, dst, src);
        else {
                kernel_fpu_begin();

diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c
index 45c1c41431766dc397a9f8591f6785d0ca013cb5..4967ad620775b109048ba1d9a19bc448a549d012 100644 (file)
--- a/arch/x86/crypto/chacha_glue.c
+++ b/arch/x86/crypto/chacha_glue.c
@@ -12,10 +12,10 @@
 
 #include <crypto/algapi.h>
 #include <crypto/chacha.h>
+#include <crypto/internal/simd.h>
 #include <crypto/internal/skcipher.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <asm/fpu/api.h>
 #include <asm/simd.h>
 
 #define CHACHA_STATE_ALIGN 16
@@ -170,7 +170,7 @@ static int chacha_simd(struct skcipher_request *req)
        struct skcipher_walk walk;
        int err;
 
-       if (req->cryptlen <= CHACHA_BLOCK_SIZE || !irq_fpu_usable())
+       if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
                return crypto_chacha_crypt(req);
 
        err = skcipher_walk_virt(&walk, req, true);
@@ -193,7 +193,7 @@ static int xchacha_simd(struct skcipher_request *req)
        u8 real_iv[16];
        int err;
 
-       if (req->cryptlen <= CHACHA_BLOCK_SIZE || !irq_fpu_usable())
+       if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
                return crypto_xchacha_crypt(req);
 
        err = skcipher_walk_virt(&walk, req, true);

diff --git a/arch/x86/crypto/crc32-pclmul_glue.c b/arch/x86/crypto/crc32-pclmul_glue.c
index c8d9cdacbf1046bf35ba95da8ebd0730ce369bbd..cb4ab6645106e444bfe4f5225c3082774f5d1f27 100644 (file)
--- a/arch/x86/crypto/crc32-pclmul_glue.c
+++ b/arch/x86/crypto/crc32-pclmul_glue.c
@@ -32,10 +32,11 @@
 #include <linux/kernel.h>
 #include <linux/crc32.h>
 #include <crypto/internal/hash.h>
+#include <crypto/internal/simd.h>
 
 #include <asm/cpufeatures.h>
 #include <asm/cpu_device_id.h>
-#include <asm/fpu/api.h>
+#include <asm/simd.h>
 
 #define CHKSUM_BLOCK_SIZE      1
 #define CHKSUM_DIGEST_SIZE     4
@@ -54,7 +55,7 @@ static u32 __attribute__((pure))
        unsigned int iremainder;
        unsigned int prealign;
 
-       if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !irq_fpu_usable())
+       if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !crypto_simd_usable())
                return crc32_le(crc, p, len);
 
        if ((long)p & SCALE_F_MASK) {

diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c
index 5773e11610723f74fc4f99d44a53d6ef8402d050..a58fe217c856da1f7dd35b087be48be8446ca688 100644 (file)
--- a/arch/x86/crypto/crc32c-intel_glue.c
+++ b/arch/x86/crypto/crc32c-intel_glue.c
@@ -29,10 +29,11 @@
 #include <linux/string.h>
 #include <linux/kernel.h>
 #include <crypto/internal/hash.h>
+#include <crypto/internal/simd.h>
 
 #include <asm/cpufeatures.h>
 #include <asm/cpu_device_id.h>
-#include <asm/fpu/internal.h>
+#include <asm/simd.h>
 
 #define CHKSUM_BLOCK_SIZE      1
 #define CHKSUM_DIGEST_SIZE     4
@@ -177,7 +178,7 @@ static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data,
         * use faster PCL version if datasize is large enough to
         * overcome kernel fpu state save/restore overhead
         */
-       if (len >= CRC32C_PCL_BREAKEVEN && irq_fpu_usable()) {
+       if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) {
                kernel_fpu_begin();
                *crcp = crc_pcl(data, len, *crcp);
                kernel_fpu_end();
@@ -189,7 +190,7 @@ static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data,
 static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
                                u8 *out)
 {
-       if (len >= CRC32C_PCL_BREAKEVEN && irq_fpu_usable()) {
+       if (len >= CRC32C_PCL_BREAKEVEN && crypto_simd_usable()) {
                kernel_fpu_begin();
                *(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp));
                kernel_fpu_end();

diff --git a/arch/x86/crypto/crct10dif-pclmul_glue.c b/arch/x86/crypto/crct10dif-pclmul_glue.c
index 0e785c0b235422aee1232c02a448baab2d74a545..64f17d2d8b67ab24f9801cd86e363afe2729dccb 100644 (file)
--- a/arch/x86/crypto/crct10dif-pclmul_glue.c
+++ b/arch/x86/crypto/crct10dif-pclmul_glue.c
@@ -26,12 +26,13 @@
 #include <linux/module.h>
 #include <linux/crc-t10dif.h>
 #include <crypto/internal/hash.h>
+#include <crypto/internal/simd.h>
 #include <linux/init.h>
 #include <linux/string.h>
 #include <linux/kernel.h>
-#include <asm/fpu/api.h>
 #include <asm/cpufeatures.h>
 #include <asm/cpu_device_id.h>
+#include <asm/simd.h>
 
 asmlinkage u16 crc_t10dif_pcl(u16 init_crc, const u8 *buf, size_t len);
 
@@ -53,7 +54,7 @@ static int chksum_update(struct shash_desc *desc, const u8 *data,
 {
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
 
-       if (length >= 16 && irq_fpu_usable()) {
+       if (length >= 16 && crypto_simd_usable()) {
                kernel_fpu_begin();
                ctx->crc = crc_t10dif_pcl(ctx->crc, data, length);
                kernel_fpu_end();
@@ -73,7 +74,7 @@ static int chksum_final(struct shash_desc *desc, u8 *out)
 static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len,
                        u8 *out)
 {
-       if (len >= 16 && irq_fpu_usable()) {
+       if (len >= 16 && crypto_simd_usable()) {
                kernel_fpu_begin();
                *(__u16 *)out = crc_t10dif_pcl(*crcp, data, len);
                kernel_fpu_end();

diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c
index 3582ae885ee11c04ea3e49f9c5dc77d0815d1202..4099a0ae17dd11da7f9c433b75ed83ca28af56cf 100644 (file)
--- a/arch/x86/crypto/ghash-clmulni-intel_glue.c
+++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c
@@ -19,8 +19,9 @@
 #include <crypto/cryptd.h>
 #include <crypto/gf128mul.h>
 #include <crypto/internal/hash.h>
-#include <asm/fpu/api.h>
+#include <crypto/internal/simd.h>
 #include <asm/cpu_device_id.h>
+#include <asm/simd.h>
 
 #define GHASH_BLOCK_SIZE       16
 #define GHASH_DIGEST_SIZE      16
@@ -182,7 +183,7 @@ static int ghash_async_update(struct ahash_request *req)
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 
-       if (!irq_fpu_usable() ||
+       if (!crypto_simd_usable() ||
            (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
                memcpy(cryptd_req, req, sizeof(*req));
                ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
@@ -200,7 +201,7 @@ static int ghash_async_final(struct ahash_request *req)
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 
-       if (!irq_fpu_usable() ||
+       if (!crypto_simd_usable() ||
            (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
                memcpy(cryptd_req, req, sizeof(*req));
                ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
@@ -241,7 +242,7 @@ static int ghash_async_digest(struct ahash_request *req)
        struct ahash_request *cryptd_req = ahash_request_ctx(req);
        struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
 
-       if (!irq_fpu_usable() ||
+       if (!crypto_simd_usable() ||
            (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
                memcpy(cryptd_req, req, sizeof(*req));
                ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);

diff --git a/arch/x86/crypto/nhpoly1305-avx2-glue.c b/arch/x86/crypto/nhpoly1305-avx2-glue.c
index 20d815ea4b6a98fbd3ccd30f8f50f052ae9f58fc..f7567cbd35b6958dbac7e54f4a4406b98bf0fa79 100644 (file)
--- a/arch/x86/crypto/nhpoly1305-avx2-glue.c
+++ b/arch/x86/crypto/nhpoly1305-avx2-glue.c
@@ -7,9 +7,10 @@
  */
 
 #include <crypto/internal/hash.h>
+#include <crypto/internal/simd.h>
 #include <crypto/nhpoly1305.h>
 #include <linux/module.h>
-#include <asm/fpu/api.h>
+#include <asm/simd.h>
 
 asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
                        u8 hash[NH_HASH_BYTES]);
@@ -24,7 +25,7 @@ static void _nh_avx2(const u32 *key, const u8 *message, size_t message_len,
 static int nhpoly1305_avx2_update(struct shash_desc *desc,
                                  const u8 *src, unsigned int srclen)
 {
-       if (srclen < 64 || !irq_fpu_usable())
+       if (srclen < 64 || !crypto_simd_usable())
                return crypto_nhpoly1305_update(desc, src, srclen);
 
        do {

diff --git a/arch/x86/crypto/nhpoly1305-sse2-glue.c b/arch/x86/crypto/nhpoly1305-sse2-glue.c
index ed68d164ce1402ee971737c96575a2d0ea4dc42f..a661ede3b5cfac684f38249bf26ad3ca24e00b72 100644 (file)
--- a/arch/x86/crypto/nhpoly1305-sse2-glue.c
+++ b/arch/x86/crypto/nhpoly1305-sse2-glue.c
@@ -7,9 +7,10 @@
  */
 
 #include <crypto/internal/hash.h>
+#include <crypto/internal/simd.h>
 #include <crypto/nhpoly1305.h>
 #include <linux/module.h>
-#include <asm/fpu/api.h>
+#include <asm/simd.h>
 
 asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
                        u8 hash[NH_HASH_BYTES]);
@@ -24,7 +25,7 @@ static void _nh_sse2(const u32 *key, const u8 *message, size_t message_len,
 static int nhpoly1305_sse2_update(struct shash_desc *desc,
                                  const u8 *src, unsigned int srclen)
 {
-       if (srclen < 64 || !irq_fpu_usable())
+       if (srclen < 64 || !crypto_simd_usable())
                return crypto_nhpoly1305_update(desc, src, srclen);
 
        do {

diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c
index 88cc01506c84a64bc4731e8974ed76487be0f168..6eb65b237b3cbcdaa8fe49316e24d736618dd452 100644 (file)
--- a/arch/x86/crypto/poly1305_glue.c
+++ b/arch/x86/crypto/poly1305_glue.c
@@ -11,11 +11,11 @@
 
 #include <crypto/algapi.h>
 #include <crypto/internal/hash.h>
+#include <crypto/internal/simd.h>
 #include <crypto/poly1305.h>
 #include <linux/crypto.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <asm/fpu/api.h>
 #include <asm/simd.h>
 
 struct poly1305_simd_desc_ctx {
@@ -126,7 +126,7 @@ static int poly1305_simd_update(struct shash_desc *desc,
        unsigned int bytes;
 
        /* kernel_fpu_begin/end is costly, use fallback for small updates */
-       if (srclen <= 288 || !may_use_simd())
+       if (srclen <= 288 || !crypto_simd_usable())
                return crypto_poly1305_update(desc, src, srclen);
 
        kernel_fpu_begin();

diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c
index 7391c7de72c7cabdbab895726de8ec4db7e2f7e9..42f177afc33a45475109258c798826e80c827ac1 100644 (file)
--- a/arch/x86/crypto/sha1_ssse3_glue.c
+++ b/arch/x86/crypto/sha1_ssse3_glue.c
@@ -22,6 +22,7 @@
 #define pr_fmt(fmt)    KBUILD_MODNAME ": " fmt
 
 #include <crypto/internal/hash.h>
+#include <crypto/internal/simd.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/mm.h>
@@ -29,7 +30,7 @@
 #include <linux/types.h>
 #include <crypto/sha.h>
 #include <crypto/sha1_base.h>
-#include <asm/fpu/api.h>
+#include <asm/simd.h>
 
 typedef void (sha1_transform_fn)(u32 *digest, const char *data,
                                unsigned int rounds);
@@ -39,7 +40,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data,
 {
        struct sha1_state *sctx = shash_desc_ctx(desc);
 
-       if (!irq_fpu_usable() ||
+       if (!crypto_simd_usable() ||
            (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
                return crypto_sha1_update(desc, data, len);
 
@@ -57,7 +58,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data,
 static int sha1_finup(struct shash_desc *desc, const u8 *data,
                      unsigned int len, u8 *out, sha1_transform_fn *sha1_xform)
 {
-       if (!irq_fpu_usable())
+       if (!crypto_simd_usable())
                return crypto_sha1_finup(desc, data, len, out);
 
        kernel_fpu_begin();

diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c
index 773a873d2b28a31cdcaa3b1d085cb8ddf94664d6..73867da3cbeeded6ae57149f2e7011ceb2445822 100644 (file)
--- a/arch/x86/crypto/sha256_ssse3_glue.c
+++ b/arch/x86/crypto/sha256_ssse3_glue.c
@@ -30,6 +30,7 @@
 #define pr_fmt(fmt)    KBUILD_MODNAME ": " fmt
 
 #include <crypto/internal/hash.h>
+#include <crypto/internal/simd.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/mm.h>
@@ -37,8 +38,8 @@
 #include <linux/types.h>
 #include <crypto/sha.h>
 #include <crypto/sha256_base.h>
-#include <asm/fpu/api.h>
 #include <linux/string.h>
+#include <asm/simd.h>
 
 asmlinkage void sha256_transform_ssse3(u32 *digest, const char *data,
                                       u64 rounds);
@@ -49,7 +50,7 @@ static int sha256_update(struct shash_desc *desc, const u8 *data,
 {
        struct sha256_state *sctx = shash_desc_ctx(desc);
 
-       if (!irq_fpu_usable() ||
+       if (!crypto_simd_usable() ||
            (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
                return crypto_sha256_update(desc, data, len);
 
@@ -67,7 +68,7 @@ static int sha256_update(struct shash_desc *desc, const u8 *data,
 static int sha256_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out, sha256_transform_fn *sha256_xform)
 {
-       if (!irq_fpu_usable())
+       if (!crypto_simd_usable())
                return crypto_sha256_finup(desc, data, len, out);
 
        kernel_fpu_begin();

diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c
index f1b811b60ba6f852968274cb6ecbf8ea976406b4..458356a3f124f37daf290a021f8f7abd00ab27e4 100644 (file)
--- a/arch/x86/crypto/sha512_ssse3_glue.c
+++ b/arch/x86/crypto/sha512_ssse3_glue.c
@@ -28,16 +28,16 @@
 #define pr_fmt(fmt)    KBUILD_MODNAME ": " fmt
 
 #include <crypto/internal/hash.h>
+#include <crypto/internal/simd.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/mm.h>
 #include <linux/cryptohash.h>
+#include <linux/string.h>
 #include <linux/types.h>
 #include <crypto/sha.h>
 #include <crypto/sha512_base.h>
-#include <asm/fpu/api.h>
-
-#include <linux/string.h>
+#include <asm/simd.h>
 
 asmlinkage void sha512_transform_ssse3(u64 *digest, const char *data,
                                       u64 rounds);
@@ -49,7 +49,7 @@ static int sha512_update(struct shash_desc *desc, const u8 *data,
 {
        struct sha512_state *sctx = shash_desc_ctx(desc);
 
-       if (!irq_fpu_usable() ||
+       if (!crypto_simd_usable() ||
            (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE)
                return crypto_sha512_update(desc, data, len);
 
@@ -67,7 +67,7 @@ static int sha512_update(struct shash_desc *desc, const u8 *data,
 static int sha512_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out, sha512_transform_fn *sha512_xform)
 {
-       if (!irq_fpu_usable())
+       if (!crypto_simd_usable())
                return crypto_sha512_finup(desc, data, len, out);
 
        kernel_fpu_begin();