crypto: nx - fix concurrency issue

The NX driver uses the transformation context to store several fields
containing data related to the state of the operations in progress.
Since a single tfm can be used by different kernel threads at the same
time, we need to protect the data stored into the context.

This patch makes use of spin locks to protect the data where a race
condition can happen.

Reviewed-by: Fionnuala Gunter <fin@linux.vnet.ibm.com>
Reviewed-by: Joy Latten <jmlatten@linux.vnet.ibm.com>
Signed-off-by: Marcelo Cerri <mhcerri@linux.vnet.ibm.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/drivers/crypto/nx/nx-aes-cbc.c b/drivers/crypto/nx/nx-aes-cbc.c
index a2f99a910e4ab821ad9580f417f1569a18f4179d..7c0237dae02db1cde29272f0d9695f59f2028acf 100644 (file)
--- a/drivers/crypto/nx/nx-aes-cbc.c
+++ b/drivers/crypto/nx/nx-aes-cbc.c
@@ -70,10 +70,15 @@ static int cbc_aes_nx_crypt(struct blkcipher_desc *desc,
 {
        struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
        struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+       unsigned long irq_flags;
        int rc;
 
-       if (nbytes > nx_ctx->ap->databytelen)
-               return -EINVAL;
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+       if (nbytes > nx_ctx->ap->databytelen) {
+               rc = -EINVAL;
+               goto out;
+       }
 
        if (enc)
                NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
@@ -100,6 +105,7 @@ static int cbc_aes_nx_crypt(struct blkcipher_desc *desc,
        atomic64_add(csbcpb->csb.processed_byte_count,
                     &(nx_ctx->stats->aes_bytes));
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 

diff --git a/drivers/crypto/nx/nx-aes-ccm.c b/drivers/crypto/nx/nx-aes-ccm.c
index ef5eae6d1400382a8830bca980752b123e12bf52..39d42245bc79a3d647ecff7556cb8831654be96f 100644 (file)
--- a/drivers/crypto/nx/nx-aes-ccm.c
+++ b/drivers/crypto/nx/nx-aes-ccm.c
@@ -271,10 +271,15 @@ static int ccm_nx_decrypt(struct aead_request   *req,
        unsigned int nbytes = req->cryptlen;
        unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
        struct nx_ccm_priv *priv = &nx_ctx->priv.ccm;
+       unsigned long irq_flags;
        int rc = -1;
 
-       if (nbytes > nx_ctx->ap->databytelen)
-               return -EINVAL;
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+       if (nbytes > nx_ctx->ap->databytelen) {
+               rc = -EINVAL;
+               goto out;
+       }
 
        nbytes -= authsize;
 
@@ -308,6 +313,7 @@ static int ccm_nx_decrypt(struct aead_request   *req,
        rc = memcmp(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag,
                    authsize) ? -EBADMSG : 0;
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 
@@ -318,10 +324,15 @@ static int ccm_nx_encrypt(struct aead_request   *req,
        struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
        unsigned int nbytes = req->cryptlen;
        unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
+       unsigned long irq_flags;
        int rc = -1;
 
-       if (nbytes > nx_ctx->ap->databytelen)
-               return -EINVAL;
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+       if (nbytes > nx_ctx->ap->databytelen) {
+               rc = -EINVAL;
+               goto out;
+       }
 
        rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes,
                          csbcpb->cpb.aes_ccm.in_pat_or_b0);
@@ -350,6 +361,7 @@ static int ccm_nx_encrypt(struct aead_request   *req,
                                 req->dst, nbytes, authsize,
                                 SCATTERWALK_TO_SG);
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 

diff --git a/drivers/crypto/nx/nx-aes-ctr.c b/drivers/crypto/nx/nx-aes-ctr.c
index b6286f14680ba82fcb7a32136dddea9b2a24ce29..762611b883cb9d1ff536138ebcef73e36662abe1 100644 (file)
--- a/drivers/crypto/nx/nx-aes-ctr.c
+++ b/drivers/crypto/nx/nx-aes-ctr.c
@@ -88,10 +88,15 @@ static int ctr_aes_nx_crypt(struct blkcipher_desc *desc,
 {
        struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
        struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+       unsigned long irq_flags;
        int rc;
 
-       if (nbytes > nx_ctx->ap->databytelen)
-               return -EINVAL;
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+       if (nbytes > nx_ctx->ap->databytelen) {
+               rc = -EINVAL;
+               goto out;
+       }
 
        rc = nx_build_sg_lists(nx_ctx, desc, dst, src, nbytes,
                               csbcpb->cpb.aes_ctr.iv);
@@ -112,6 +117,7 @@ static int ctr_aes_nx_crypt(struct blkcipher_desc *desc,
        atomic64_add(csbcpb->csb.processed_byte_count,
                     &(nx_ctx->stats->aes_bytes));
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 

diff --git a/drivers/crypto/nx/nx-aes-ecb.c b/drivers/crypto/nx/nx-aes-ecb.c
index 7bbc9a81da219e5c27e021c9ef6c880d194c90d3..77dbe084ba4168b1ca42fbedfe3ada9b9c9a19c1 100644 (file)
--- a/drivers/crypto/nx/nx-aes-ecb.c
+++ b/drivers/crypto/nx/nx-aes-ecb.c
@@ -70,10 +70,15 @@ static int ecb_aes_nx_crypt(struct blkcipher_desc *desc,
 {
        struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
        struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+       unsigned long irq_flags;
        int rc;
 
-       if (nbytes > nx_ctx->ap->databytelen)
-               return -EINVAL;
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+       if (nbytes > nx_ctx->ap->databytelen) {
+               rc = -EINVAL;
+               goto out;
+       }
 
        if (enc)
                NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
@@ -98,6 +103,7 @@ static int ecb_aes_nx_crypt(struct blkcipher_desc *desc,
        atomic64_add(csbcpb->csb.processed_byte_count,
                     &(nx_ctx->stats->aes_bytes));
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 

diff --git a/drivers/crypto/nx/nx-aes-gcm.c b/drivers/crypto/nx/nx-aes-gcm.c
index 6cca6c392b00f34fa65ad4663b1725bd7b3842fe..df90d03afc104f0e95311e6afe37982395af4403 100644 (file)
--- a/drivers/crypto/nx/nx-aes-gcm.c
+++ b/drivers/crypto/nx/nx-aes-gcm.c
@@ -166,8 +166,11 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
        struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
        struct blkcipher_desc desc;
        unsigned int nbytes = req->cryptlen;
+       unsigned long irq_flags;
        int rc = -EINVAL;
 
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
        if (nbytes > nx_ctx->ap->databytelen)
                goto out;
 
@@ -255,6 +258,7 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
                     -EBADMSG : 0;
        }
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 

diff --git a/drivers/crypto/nx/nx-aes-xcbc.c b/drivers/crypto/nx/nx-aes-xcbc.c
index 93923e4628c05b8af34bf19e90eb3cb692a4a0a0..658da0fd3e1f18f241143fe1e4d3c12a4f0b8af0 100644 (file)
--- a/drivers/crypto/nx/nx-aes-xcbc.c
+++ b/drivers/crypto/nx/nx-aes-xcbc.c
@@ -89,8 +89,11 @@ static int nx_xcbc_update(struct shash_desc *desc,
        struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
        struct nx_sg *in_sg;
        u32 to_process, leftover;
+       unsigned long irq_flags;
        int rc = 0;
 
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
        if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
                /* we've hit the nx chip previously and we're updating again,
                 * so copy over the partial digest */
@@ -158,6 +161,7 @@ static int nx_xcbc_update(struct shash_desc *desc,
        /* everything after the first update is continuation */
        NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 
@@ -167,8 +171,11 @@ static int nx_xcbc_final(struct shash_desc *desc, u8 *out)
        struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
        struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
        struct nx_sg *in_sg, *out_sg;
+       unsigned long irq_flags;
        int rc = 0;
 
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
        if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
                /* we've hit the nx chip previously, now we're finalizing,
                 * so copy over the partial digest */
@@ -211,6 +218,7 @@ static int nx_xcbc_final(struct shash_desc *desc, u8 *out)
 
        memcpy(out, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 

diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c
index 254b01abef64167406343c8cfb054f12949bb17c..6547a7104bf64f96b4e0fe6c9a58e7114b3884e9 100644 (file)
--- a/drivers/crypto/nx/nx-sha256.c
+++ b/drivers/crypto/nx/nx-sha256.c
@@ -57,8 +57,11 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
        struct nx_sg *in_sg;
        u64 to_process, leftover, total;
        u32 max_sg_len;
+       unsigned long irq_flags;
        int rc = 0;
 
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
        /* 2 cases for total data len:
         *  1: < SHA256_BLOCK_SIZE: copy into state, return 0
         *  2: >= SHA256_BLOCK_SIZE: process X blocks, copy in leftover
@@ -136,6 +139,7 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
                memcpy(sctx->buf, data, leftover);
        sctx->count = leftover;
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 
@@ -146,8 +150,11 @@ static int nx_sha256_final(struct shash_desc *desc, u8 *out)
        struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
        struct nx_sg *in_sg, *out_sg;
        u32 max_sg_len;
+       unsigned long irq_flags;
        int rc;
 
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
        max_sg_len = min_t(u32, nx_driver.of.max_sg_len, nx_ctx->ap->sglen);
 
        if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
@@ -186,6 +193,7 @@ static int nx_sha256_final(struct shash_desc *desc, u8 *out)
                     &(nx_ctx->stats->sha256_bytes));
        memcpy(out, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 
@@ -195,6 +203,9 @@ static int nx_sha256_export(struct shash_desc *desc, void *out)
        struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
        struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
        struct sha256_state *octx = out;
+       unsigned long irq_flags;
+
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
        octx->count = sctx->count +
                      (csbcpb->cpb.sha256.message_bit_length / 8);
@@ -217,6 +228,7 @@ static int nx_sha256_export(struct shash_desc *desc, void *out)
                octx->state[7] = SHA256_H7;
        }
 
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return 0;
 }
 
@@ -226,6 +238,9 @@ static int nx_sha256_import(struct shash_desc *desc, const void *in)
        struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
        struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
        const struct sha256_state *ictx = in;
+       unsigned long irq_flags;
+
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
        memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
 
@@ -240,6 +255,7 @@ static int nx_sha256_import(struct shash_desc *desc, const void *in)
                NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
        }
 
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return 0;
 }
 

diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c
index 2d6d91359833c473d2e84fb57bef08190dbb0a66..236e6afeab1067701cd75e07bfdcf00fe41baef0 100644 (file)
--- a/drivers/crypto/nx/nx-sha512.c
+++ b/drivers/crypto/nx/nx-sha512.c
@@ -57,8 +57,11 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
        struct nx_sg *in_sg;
        u64 to_process, leftover, total, spbc_bits;
        u32 max_sg_len;
+       unsigned long irq_flags;
        int rc = 0;
 
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
        /* 2 cases for total data len:
         *  1: < SHA512_BLOCK_SIZE: copy into state, return 0
         *  2: >= SHA512_BLOCK_SIZE: process X blocks, copy in leftover
@@ -138,6 +141,7 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
                memcpy(sctx->buf, data, leftover);
        sctx->count[0] = leftover;
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 
@@ -149,8 +153,11 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out)
        struct nx_sg *in_sg, *out_sg;
        u32 max_sg_len;
        u64 count0;
+       unsigned long irq_flags;
        int rc;
 
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
        max_sg_len = min_t(u32, nx_driver.of.max_sg_len, nx_ctx->ap->sglen);
 
        if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
@@ -193,6 +200,7 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out)
 
        memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
 out:
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return rc;
 }
 
@@ -202,6 +210,9 @@ static int nx_sha512_export(struct shash_desc *desc, void *out)
        struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
        struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
        struct sha512_state *octx = out;
+       unsigned long irq_flags;
+
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
        /* move message_bit_length (128 bits) into count and convert its value
         * to bytes */
@@ -233,6 +244,7 @@ static int nx_sha512_export(struct shash_desc *desc, void *out)
                octx->state[7] = SHA512_H7;
        }
 
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return 0;
 }
 
@@ -242,6 +254,9 @@ static int nx_sha512_import(struct shash_desc *desc, const void *in)
        struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
        struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
        const struct sha512_state *ictx = in;
+       unsigned long irq_flags;
+
+       spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
        memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
        sctx->count[0] = ictx->count[0] & 0x3f;
@@ -259,6 +274,7 @@ static int nx_sha512_import(struct shash_desc *desc, const void *in)
                NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
        }
 
+       spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
        return 0;
 }
 

diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c
index ad07dc62b95a4990f245e6daa4909a6bc8fbde19..bdf4990f9758a59b6b3f73e1857ca8c91169f2cd 100644 (file)
--- a/drivers/crypto/nx/nx.c
+++ b/drivers/crypto/nx/nx.c
@@ -61,8 +61,7 @@ int nx_hcall_sync(struct nx_crypto_ctx *nx_ctx,
 
        do {
                rc = vio_h_cop_sync(viodev, op);
-       } while ((rc == -EBUSY && !may_sleep && retries--) ||
-                (rc == -EBUSY && may_sleep && cond_resched()));
+       } while (rc == -EBUSY && !may_sleep && retries--);
 
        if (rc) {
                dev_dbg(&viodev->dev, "vio_h_cop_sync failed: rc: %d "
@@ -251,6 +250,7 @@ int nx_build_sg_lists(struct nx_crypto_ctx  *nx_ctx,
  */
 void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function)
 {
+       spin_lock_init(&nx_ctx->lock);
        memset(nx_ctx->kmem, 0, nx_ctx->kmem_len);
        nx_ctx->csbcpb->csb.valid |= NX_CSB_VALID_BIT;
 

diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h
index 3232b182dd28da33d2784cc4cb8aa577dd028fe6..14bb97f1c339ce40ea47f3795ff0ea4594361aea 100644 (file)
--- a/drivers/crypto/nx/nx.h
+++ b/drivers/crypto/nx/nx.h
@@ -117,6 +117,7 @@ struct nx_ctr_priv {
 };
 
 struct nx_crypto_ctx {
+       spinlock_t lock;          /* synchronize access to the context */
        void *kmem;               /* unaligned, kmalloc'd buffer */
        size_t kmem_len;          /* length of kmem */
        struct nx_csbcpb *csbcpb; /* aligned page given to phyp @ hcall time */