crypto: caam/qi - add GCM support

Add support for AES working in Galois Counter Mode.
The following algorithms are added:
gcm(aes)
rfc4106(gcm(aes))
rfc4543(gcm(aes))

There is a limitation related to IV size, similar to the one present in
SW implementation (crypto/gcm.c):
The only IV size allowed is 12 bytes. It will be padded by HW to the right
with 0x0000_0001 (up to 16 bytes - AES block size), according to the
GCM specification.

Signed-off-by: Horia Geantă <horia.geanta@nxp.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index f4db39495d228998f26a89cb46e0d2ac3837cc11..c2b5762d56a0a1c28cc4635aac752cee005b18a2 100644 (file)
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -284,6 +284,309 @@ badkey:
        return -EINVAL;
 }
 
+static int gcm_set_sh_desc(struct crypto_aead *aead)
+{
+       struct caam_ctx *ctx = crypto_aead_ctx(aead);
+       unsigned int ivsize = crypto_aead_ivsize(aead);
+       int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
+                       ctx->cdata.keylen;
+
+       if (!ctx->cdata.keylen || !ctx->authsize)
+               return 0;
+
+       /*
+        * Job Descriptor and Shared Descriptor
+        * must fit into the 64-word Descriptor h/w Buffer
+        */
+       if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
+               ctx->cdata.key_inline = true;
+               ctx->cdata.key_virt = ctx->key;
+       } else {
+               ctx->cdata.key_inline = false;
+               ctx->cdata.key_dma = ctx->key_dma;
+       }
+
+       cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
+                             ctx->authsize, true);
+
+       /*
+        * Job Descriptor and Shared Descriptor
+        * must fit into the 64-word Descriptor h/w Buffer
+        */
+       if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
+               ctx->cdata.key_inline = true;
+               ctx->cdata.key_virt = ctx->key;
+       } else {
+               ctx->cdata.key_inline = false;
+               ctx->cdata.key_dma = ctx->key_dma;
+       }
+
+       cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
+                             ctx->authsize, true);
+
+       return 0;
+}
+
+static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
+{
+       struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+
+       ctx->authsize = authsize;
+       gcm_set_sh_desc(authenc);
+
+       return 0;
+}
+
+static int gcm_setkey(struct crypto_aead *aead,
+                     const u8 *key, unsigned int keylen)
+{
+       struct caam_ctx *ctx = crypto_aead_ctx(aead);
+       struct device *jrdev = ctx->jrdev;
+       int ret;
+
+#ifdef DEBUG
+       print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
+                      DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+#endif
+
+       memcpy(ctx->key, key, keylen);
+       dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
+       ctx->cdata.keylen = keylen;
+
+       ret = gcm_set_sh_desc(aead);
+       if (ret)
+               return ret;
+
+       /* Now update the driver contexts with the new shared descriptor */
+       if (ctx->drv_ctx[ENCRYPT]) {
+               ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
+                                         ctx->sh_desc_enc);
+               if (ret) {
+                       dev_err(jrdev, "driver enc context update failed\n");
+                       return ret;
+               }
+       }
+
+       if (ctx->drv_ctx[DECRYPT]) {
+               ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
+                                         ctx->sh_desc_dec);
+               if (ret) {
+                       dev_err(jrdev, "driver dec context update failed\n");
+                       return ret;
+               }
+       }
+
+       return 0;
+}
+
+static int rfc4106_set_sh_desc(struct crypto_aead *aead)
+{
+       struct caam_ctx *ctx = crypto_aead_ctx(aead);
+       unsigned int ivsize = crypto_aead_ivsize(aead);
+       int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
+                       ctx->cdata.keylen;
+
+       if (!ctx->cdata.keylen || !ctx->authsize)
+               return 0;
+
+       ctx->cdata.key_virt = ctx->key;
+
+       /*
+        * Job Descriptor and Shared Descriptor
+        * must fit into the 64-word Descriptor h/w Buffer
+        */
+       if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
+               ctx->cdata.key_inline = true;
+       } else {
+               ctx->cdata.key_inline = false;
+               ctx->cdata.key_dma = ctx->key_dma;
+       }
+
+       cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
+                                 ctx->authsize, true);
+
+       /*
+        * Job Descriptor and Shared Descriptor
+        * must fit into the 64-word Descriptor h/w Buffer
+        */
+       if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
+               ctx->cdata.key_inline = true;
+       } else {
+               ctx->cdata.key_inline = false;
+               ctx->cdata.key_dma = ctx->key_dma;
+       }
+
+       cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
+                                 ctx->authsize, true);
+
+       return 0;
+}
+
+static int rfc4106_setauthsize(struct crypto_aead *authenc,
+                              unsigned int authsize)
+{
+       struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+
+       ctx->authsize = authsize;
+       rfc4106_set_sh_desc(authenc);
+
+       return 0;
+}
+
+static int rfc4106_setkey(struct crypto_aead *aead,
+                         const u8 *key, unsigned int keylen)
+{
+       struct caam_ctx *ctx = crypto_aead_ctx(aead);
+       struct device *jrdev = ctx->jrdev;
+       int ret;
+
+       if (keylen < 4)
+               return -EINVAL;
+
+#ifdef DEBUG
+       print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
+                      DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+#endif
+
+       memcpy(ctx->key, key, keylen);
+       /*
+        * The last four bytes of the key material are used as the salt value
+        * in the nonce. Update the AES key length.
+        */
+       ctx->cdata.keylen = keylen - 4;
+       dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
+                                  ctx->dir);
+
+       ret = rfc4106_set_sh_desc(aead);
+       if (ret)
+               return ret;
+
+       /* Now update the driver contexts with the new shared descriptor */
+       if (ctx->drv_ctx[ENCRYPT]) {
+               ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
+                                         ctx->sh_desc_enc);
+               if (ret) {
+                       dev_err(jrdev, "driver enc context update failed\n");
+                       return ret;
+               }
+       }
+
+       if (ctx->drv_ctx[DECRYPT]) {
+               ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
+                                         ctx->sh_desc_dec);
+               if (ret) {
+                       dev_err(jrdev, "driver dec context update failed\n");
+                       return ret;
+               }
+       }
+
+       return 0;
+}
+
+static int rfc4543_set_sh_desc(struct crypto_aead *aead)
+{
+       struct caam_ctx *ctx = crypto_aead_ctx(aead);
+       unsigned int ivsize = crypto_aead_ivsize(aead);
+       int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
+                       ctx->cdata.keylen;
+
+       if (!ctx->cdata.keylen || !ctx->authsize)
+               return 0;
+
+       ctx->cdata.key_virt = ctx->key;
+
+       /*
+        * Job Descriptor and Shared Descriptor
+        * must fit into the 64-word Descriptor h/w Buffer
+        */
+       if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
+               ctx->cdata.key_inline = true;
+       } else {
+               ctx->cdata.key_inline = false;
+               ctx->cdata.key_dma = ctx->key_dma;
+       }
+
+       cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
+                                 ctx->authsize, true);
+
+       /*
+        * Job Descriptor and Shared Descriptor
+        * must fit into the 64-word Descriptor h/w Buffer
+        */
+       if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
+               ctx->cdata.key_inline = true;
+       } else {
+               ctx->cdata.key_inline = false;
+               ctx->cdata.key_dma = ctx->key_dma;
+       }
+
+       cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
+                                 ctx->authsize, true);
+
+       return 0;
+}
+
+static int rfc4543_setauthsize(struct crypto_aead *authenc,
+                              unsigned int authsize)
+{
+       struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+
+       ctx->authsize = authsize;
+       rfc4543_set_sh_desc(authenc);
+
+       return 0;
+}
+
+static int rfc4543_setkey(struct crypto_aead *aead,
+                         const u8 *key, unsigned int keylen)
+{
+       struct caam_ctx *ctx = crypto_aead_ctx(aead);
+       struct device *jrdev = ctx->jrdev;
+       int ret;
+
+       if (keylen < 4)
+               return -EINVAL;
+
+#ifdef DEBUG
+       print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
+                      DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+#endif
+
+       memcpy(ctx->key, key, keylen);
+       /*
+        * The last four bytes of the key material are used as the salt value
+        * in the nonce. Update the AES key length.
+        */
+       ctx->cdata.keylen = keylen - 4;
+       dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
+                                  ctx->dir);
+
+       ret = rfc4543_set_sh_desc(aead);
+       if (ret)
+               return ret;
+
+       /* Now update the driver contexts with the new shared descriptor */
+       if (ctx->drv_ctx[ENCRYPT]) {
+               ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
+                                         ctx->sh_desc_enc);
+               if (ret) {
+                       dev_err(jrdev, "driver enc context update failed\n");
+                       return ret;
+               }
+       }
+
+       if (ctx->drv_ctx[DECRYPT]) {
+               ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
+                                         ctx->sh_desc_dec);
+               if (ret) {
+                       dev_err(jrdev, "driver dec context update failed\n");
+                       return ret;
+               }
+       }
+
+       return 0;
+}
+
 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
                             const u8 *key, unsigned int keylen)
 {
@@ -817,6 +1120,22 @@ static int aead_decrypt(struct aead_request *req)
        return aead_crypt(req, false);
 }
 
+static int ipsec_gcm_encrypt(struct aead_request *req)
+{
+       if (req->assoclen < 8)
+               return -EINVAL;
+
+       return aead_crypt(req, true);
+}
+
+static int ipsec_gcm_decrypt(struct aead_request *req)
+{
+       if (req->assoclen < 8)
+               return -EINVAL;
+
+       return aead_crypt(req, false);
+}
+
 static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status)
 {
        struct ablkcipher_edesc *edesc;
@@ -1337,6 +1656,61 @@ static struct caam_alg_template driver_algs[] = {
 };
 
 static struct caam_aead_alg driver_aeads[] = {
+       {
+               .aead = {
+                       .base = {
+                               .cra_name = "rfc4106(gcm(aes))",
+                               .cra_driver_name = "rfc4106-gcm-aes-caam-qi",
+                               .cra_blocksize = 1,
+                       },
+                       .setkey = rfc4106_setkey,
+                       .setauthsize = rfc4106_setauthsize,
+                       .encrypt = ipsec_gcm_encrypt,
+                       .decrypt = ipsec_gcm_decrypt,
+                       .ivsize = 8,
+                       .maxauthsize = AES_BLOCK_SIZE,
+               },
+               .caam = {
+                       .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
+               },
+       },
+       {
+               .aead = {
+                       .base = {
+                               .cra_name = "rfc4543(gcm(aes))",
+                               .cra_driver_name = "rfc4543-gcm-aes-caam-qi",
+                               .cra_blocksize = 1,
+                       },
+                       .setkey = rfc4543_setkey,
+                       .setauthsize = rfc4543_setauthsize,
+                       .encrypt = ipsec_gcm_encrypt,
+                       .decrypt = ipsec_gcm_decrypt,
+                       .ivsize = 8,
+                       .maxauthsize = AES_BLOCK_SIZE,
+               },
+               .caam = {
+                       .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
+               },
+       },
+       /* Galois Counter Mode */
+       {
+               .aead = {
+                       .base = {
+                               .cra_name = "gcm(aes)",
+                               .cra_driver_name = "gcm-aes-caam-qi",
+                               .cra_blocksize = 1,
+                       },
+                       .setkey = gcm_setkey,
+                       .setauthsize = gcm_setauthsize,
+                       .encrypt = aead_encrypt,
+                       .decrypt = aead_decrypt,
+                       .ivsize = 12,
+                       .maxauthsize = AES_BLOCK_SIZE,
+               },
+               .caam = {
+                       .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
+               }
+       },
        /* single-pass ipsec_esp descriptor */
        {
                .aead = {