Revert "Revert "crypto: tegra-aes: performance improvements""

This reverts commit 8bc4f710e981d53a9bd161c6c054241231e09149.

Signed-off-by: Varun Wadekar <vwadekar@nvidia.com>
Change-Id: I0497afd3ec54e7b835de37e9941b2418e3dad4e3
Reviewed-on: http://git-master/r/22963
Reviewed-by: Bharat Nihalani <bnihalani@nvidia.com>

diff --git a/drivers/crypto/tegra-aes.c b/drivers/crypto/tegra-aes.c
index 6d2f1c83421e4e2f046ce70efdf419bd341c1bf7..875ed152d96993396c8e65445b892bb0853692ed 100644 (file)
--- a/drivers/crypto/tegra-aes.c
+++ b/drivers/crypto/tegra-aes.c
@@ -229,7 +229,6 @@ static int aes_hw_init(struct tegra_aes_dev *dd)
                return ret;
        }
 
-       aes_writel(dd, 0x33, INT_ENB);
        return ret;
 }
 
@@ -243,9 +242,12 @@ static int aes_start_crypt(struct tegra_aes_dev *dd, u32 in_addr, u32 out_addr,
        int nblocks, int mode, bool upd_iv)
 {
        u32 cmdq[AES_HW_MAX_ICQ_LENGTH];
-       int qlen = 0, i, eng_busy, icq_empty, dma_busy, ret = 0;
+       int qlen = 0, i, eng_busy, icq_empty, ret;
        u32 value;
 
+       /* error, dma xfer complete */
+       aes_writel(dd, 0x33, INT_ENB);
+
        cmdq[qlen++] = UCQOPCODE_DMASETUP << ICQBITSHIFT_OPCODE;
        cmdq[qlen++] = in_addr;
        cmdq[qlen++] = UCQOPCODE_BLKSTARTENGINE << ICQBITSHIFT_OPCODE |
@@ -261,16 +263,6 @@ static int aes_start_crypt(struct tegra_aes_dev *dd, u32 in_addr, u32 out_addr,
        aes_writel(dd, value, CMDQUE_CONTROL);
        dev_dbg(dd->dev, "cmd_q_ctrl=0x%x", value);
 
-       value = 0;
-       value |= CONFIG_ENDIAN_ENB_FIELD;
-       aes_writel(dd, value, CONFIG);
-       dev_dbg(dd->dev, "config=0x%x", value);
-
-       value = aes_readl(dd, SECURE_CONFIG_EXT);
-       value &= ~SECURE_OFFSET_CNT_FIELD;
-       aes_writel(dd, value, SECURE_CONFIG_EXT);
-       dev_dbg(dd->dev, "secure_cfg_xt=0x%x", value);
-
        if (mode & FLAGS_CBC) {
                value = ((0x1 << SECURE_INPUT_ALG_SEL_SHIFT) |
                        ((dd->ctx->keylen * 8) << SECURE_INPUT_KEY_LEN_SHIFT) |
@@ -314,10 +306,9 @@ static int aes_start_crypt(struct tegra_aes_dev *dd, u32 in_addr, u32 out_addr,
        for (i = 0; i < qlen - 1; i++) {
                do {
                        value = aes_readl(dd, INTR_STATUS);
-                       eng_busy = value & (0x1);
-                       icq_empty = value & (0x1<<3);
-                       dma_busy = value & (0x1<<23);
-               } while (eng_busy & (!icq_empty) & dma_busy);
+                       eng_busy = value & BIT(0);
+                       icq_empty = value & BIT(3);
+               } while (eng_busy & (!icq_empty));
                aes_writel(dd, cmdq[i], ICMDQUE_WR);
        }
 
@@ -378,11 +369,6 @@ static int aes_set_key(struct tegra_aes_dev *dd)
                use_ssk = true;
        }
 
-       /* disable key read from hw */
-       value = aes_readl(dd, SECURE_SEC_SEL0+(ctx->slot->slot_num*4));
-       value &= ~SECURE_SEL0_KEYREAD_ENB0_FIELD;
-       aes_writel(dd, value, SECURE_SEC_SEL0+(ctx->slot->slot_num*4));
-
        /* enable key schedule generation in hardware */
        value = aes_readl(dd, SECURE_CONFIG_EXT);
        value &= ~SECURE_KEY_SCH_DIS_FIELD;
@@ -410,9 +396,9 @@ static int aes_set_key(struct tegra_aes_dev *dd)
 
        do {
                value = aes_readl(dd, INTR_STATUS);
-               eng_busy = value & (0x1);
-               icq_empty = value & (0x1<<3);
-               dma_busy = value & (0x1<<23);
+               eng_busy = value & BIT(0);
+               icq_empty = value & BIT(3);
+               dma_busy = value & BIT(23);
        } while (eng_busy & (!icq_empty) & dma_busy);
 
        /* settable command to get key into internal registers */
@@ -425,8 +411,8 @@ static int aes_set_key(struct tegra_aes_dev *dd)
        aes_writel(dd, value, ICMDQUE_WR);
        do {
                value = aes_readl(dd, INTR_STATUS);
-               eng_busy = value & (0x1);
-               icq_empty = value & (0x1<<3);
+               eng_busy = value & BIT(0);
+               icq_empty = value & BIT(3);
        } while (eng_busy & (!icq_empty));
 
 out:
@@ -466,8 +452,14 @@ static int tegra_aes_handle_req(struct tegra_aes_dev *dd)
 
        dev_dbg(dd->dev, "%s: get new req\n", __func__);
 
-       /* take mutex to access the aes hw */
-       mutex_lock(&aes_lock);
+       if (!req->src || !req->dst)
+               return -EINVAL;
+
+       /* take the hardware semaphore */
+       if (tegra_arb_mutex_lock_timeout(dd->res_id, ARB_SEMA_TIMEOUT) < 0) {
+               dev_err(dd->dev, "aes hardware not available\n");
+               return -EBUSY;
+       }
 
        /* assign new request to device */
        dd->req = req;
@@ -480,11 +472,6 @@ static int tegra_aes_handle_req(struct tegra_aes_dev *dd)
        in_sg = dd->in_sg;
        out_sg = dd->out_sg;
 
-       if (!in_sg || !out_sg) {
-               mutex_unlock(&aes_lock);
-               return -EINVAL;
-       }
-
        total = dd->total;
        rctx = ablkcipher_request_ctx(req);
        ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
@@ -494,47 +481,30 @@ static int tegra_aes_handle_req(struct tegra_aes_dev *dd)
        dd->iv = (u8 *)req->info;
        dd->ivlen = AES_BLOCK_SIZE;
 
-       if ((dd->flags & FLAGS_CBC) && dd->iv)
-               dd->flags |= FLAGS_NEW_IV;
-       else
-               dd->flags &= ~FLAGS_NEW_IV;
-
+       /* assign new context to device */
        ctx->dd = dd;
-       if (dd->ctx != ctx) {
-               /* assign new context to device */
+       if (dd->ctx != ctx)
                dd->ctx = ctx;
-               ctx->flags |= FLAGS_NEW_KEY;
-       }
 
-       /* take the hardware semaphore */
-       if (tegra_arb_mutex_lock_timeout(dd->res_id, ARB_SEMA_TIMEOUT) < 0) {
-               dev_err(dd->dev, "aes hardware not available\n");
-               mutex_unlock(&aes_lock);
-               return -EBUSY;
+       if (dd->flags & FLAGS_NEW_KEY) {
+               aes_set_key(dd);
+               dd->flags &= ~FLAGS_NEW_KEY;
        }
 
-       ret = aes_hw_init(dd);
-       if (ret < 0) {
-               dev_err(dd->dev, "%s: hw init fail(%d)\n", __func__, ret);
-               goto fail;
-       }
+       if ((dd->flags & FLAGS_CBC) && dd->iv) {
+               /* set iv to the aes hw slot */
+               memcpy(dd->buf_in, dd->iv, dd->ivlen);
 
-       aes_set_key(dd);
-
-       /* set iv to the aes hw slot */
-       memset(dd->buf_in, 0 , AES_BLOCK_SIZE);
-       memcpy(dd->buf_in, dd->iv, dd->ivlen);
-
-       ret = aes_start_crypt(dd, (u32)dd->dma_buf_in,
-         (u32)dd->dma_buf_out, 1, FLAGS_CBC, false);
-       if (ret < 0) {
-               dev_err(dd->dev, "aes_start_crypt fail(%d)\n", ret);
-               goto out;
+               ret = aes_start_crypt(dd, (u32)dd->dma_buf_in,
+                 (u32)dd->dma_buf_out, 1, FLAGS_CBC, false);
+               if (ret < 0) {
+                       dev_err(dd->dev, "aes_start_crypt fail(%d)\n", ret);
+                       goto out;
+               }
        }
 
        while (total) {
-               dev_dbg(dd->dev, "remain: 0x%x\n", total);
-
+               dev_dbg(dd->dev, "remain: %d\n", total);
                ret = dma_map_sg(dd->dev, in_sg, 1, DMA_TO_DEVICE);
                if (!ret) {
                        dev_err(dd->dev, "dma_map_sg() error\n");
@@ -543,11 +513,11 @@ static int tegra_aes_handle_req(struct tegra_aes_dev *dd)
 
                ret = dma_map_sg(dd->dev, out_sg, 1, DMA_FROM_DEVICE);
                if (!ret) {
-                               dev_err(dd->dev, "dma_map_sg() error\n");
-                               dma_unmap_sg(dd->dev, dd->in_sg,
-                                       1, DMA_TO_DEVICE);
-                               goto out;
-                       }
+                       dev_err(dd->dev, "dma_map_sg() error\n");
+                       dma_unmap_sg(dd->dev, dd->in_sg,
+                               1, DMA_TO_DEVICE);
+                       goto out;
+               }
 
                addr_in = sg_dma_address(in_sg);
                addr_out = sg_dma_address(out_sg);
@@ -568,7 +538,7 @@ static int tegra_aes_handle_req(struct tegra_aes_dev *dd)
                }
                dd->flags &= ~FLAGS_FAST;
 
-               dev_dbg(dd->dev, "out: copied 0x%x\n", count);
+               dev_dbg(dd->dev, "out: copied %d\n", count);
                total -= count;
                in_sg = sg_next(in_sg);
                out_sg = sg_next(out_sg);
@@ -576,17 +546,11 @@ static int tegra_aes_handle_req(struct tegra_aes_dev *dd)
        }
 
 out:
-       aes_hw_deinit(dd);
-
-fail:
        /* release the hardware semaphore */
        tegra_arb_mutex_unlock(dd->res_id);
 
        dd->total = total;
 
-       /* release the mutex */
-       mutex_unlock(&aes_lock);
-
        if (dd->req->base.complete)
                dd->req->base.complete(&dd->req->base, ret);
 
@@ -621,20 +585,22 @@ static int tegra_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
        if (ctx->slot)
                aes_release_key_slot(dd);
 
-       key_slot = aes_find_key_slot(dd);
-       if (!key_slot) {
-               dev_err(dd->dev, "no empty slot\n");
-               return -ENOMEM;
-       }
+       if (key) {
+               key_slot = aes_find_key_slot(dd);
+               if (!key_slot) {
+                       dev_err(dd->dev, "no empty slot\n");
+                       return -ENOMEM;
+               }
 
-       ctx->slot = key_slot;
-       ctx->keylen = keylen;
-       ctx->flags |= FLAGS_NEW_KEY;
+               ctx->slot = key_slot;
+               ctx->keylen = keylen;
 
-       /* copy the key */
-       memset(dd->ivkey_base, 0, AES_HW_KEY_TABLE_LENGTH_BYTES);
-       memcpy(dd->ivkey_base, key, keylen);
+               /* copy the key */
+               memset(dd->ivkey_base, 0, AES_HW_KEY_TABLE_LENGTH_BYTES);
+               memcpy(dd->ivkey_base, key, keylen);
+       }
 
+       dd->flags |= FLAGS_NEW_KEY;
        dev_dbg(dd->dev, "done\n");
        return 0;
 }
@@ -644,11 +610,14 @@ static void aes_workqueue_handler(struct work_struct *work)
        struct tegra_aes_dev *dd = aes_dev;
        int ret;
 
-       set_bit(FLAGS_BUSY, &dd->flags);
+       aes_hw_init(dd);
 
+       /* empty the crypto queue and then return */
        do {
                ret = tegra_aes_handle_req(dd);
        } while (!ret);
+
+       aes_hw_deinit(dd);
 }
 
 static irqreturn_t aes_irq(int irq, void *dev_id)
@@ -657,8 +626,12 @@ static irqreturn_t aes_irq(int irq, void *dev_id)
        u32 value = aes_readl(dd, INTR_STATUS);
 
        dev_dbg(dd->dev, "irq_stat: 0x%x", value);
-       if (!((value & ENGINE_BUSY_FIELD) & !(value & ICQ_EMPTY_FIELD)))
+       if (!((value & ENGINE_BUSY_FIELD) & !(value & ICQ_EMPTY_FIELD))) {
+               /* avoid misfires */
+               value &= ~0x33;
+               aes_writel(dd, value, INT_ENB);
                complete(&dd->op_complete);
+       }
 
        return IRQ_HANDLED;
 }