HARDWARE RANDOM NUMBER GENERATOR CORE
M: Matt Mackall <mpm@selenic.com>
M: Herbert Xu <herbert@gondor.apana.org.au>
+L: linux-crypto@vger.kernel.org
S: Odd fixes
F: Documentation/hw_random.txt
F: drivers/char/hw_random/
obj-y += dma-octeon.o
obj-y += octeon-memcpy.o
obj-y += executive/
+obj-y += crypto/
obj-$(CONFIG_MTD) += flash_setup.o
obj-$(CONFIG_SMP) += smp.o
--- /dev/null
+#
+# OCTEON-specific crypto modules.
+#
+
+obj-y += octeon-crypto.o
+
+obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o
--- /dev/null
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2004-2012 Cavium Networks
+ */
+
+#include <asm/cop2.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+
+#include "octeon-crypto.h"
+
+/**
+ * Enable access to Octeon's COP2 crypto hardware for kernel use. Wrap any
+ * crypto operations in calls to octeon_crypto_enable/disable in order to make
+ * sure the state of COP2 isn't corrupted if userspace is also performing
+ * hardware crypto operations. Allocate the state parameter on the stack.
+ * Preemption must be disabled to prevent context switches.
+ *
+ * @state: Pointer to state structure to store current COP2 state in.
+ *
+ * Returns: Flags to be passed to octeon_crypto_disable()
+ */
+unsigned long octeon_crypto_enable(struct octeon_cop2_state *state)
+{
+ int status;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ status = read_c0_status();
+ write_c0_status(status | ST0_CU2);
+ if (KSTK_STATUS(current) & ST0_CU2) {
+ octeon_cop2_save(&(current->thread.cp2));
+ KSTK_STATUS(current) &= ~ST0_CU2;
+ status &= ~ST0_CU2;
+ } else if (status & ST0_CU2) {
+ octeon_cop2_save(state);
+ }
+ local_irq_restore(flags);
+ return status & ST0_CU2;
+}
+EXPORT_SYMBOL_GPL(octeon_crypto_enable);
+
+/**
+ * Disable access to Octeon's COP2 crypto hardware in the kernel. This must be
+ * called after an octeon_crypto_enable() before any context switch or return to
+ * userspace.
+ *
+ * @state: Pointer to COP2 state to restore
+ * @flags: Return value from octeon_crypto_enable()
+ */
+void octeon_crypto_disable(struct octeon_cop2_state *state,
+ unsigned long crypto_flags)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ if (crypto_flags & ST0_CU2)
+ octeon_cop2_restore(state);
+ else
+ write_c0_status(read_c0_status() & ~ST0_CU2);
+ local_irq_restore(flags);
+}
+EXPORT_SYMBOL_GPL(octeon_crypto_disable);
--- /dev/null
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2012-2013 Cavium Inc., All Rights Reserved.
+ *
+ * MD5 instruction definitions added by Aaro Koskinen <aaro.koskinen@iki.fi>.
+ *
+ */
+#ifndef __LINUX_OCTEON_CRYPTO_H
+#define __LINUX_OCTEON_CRYPTO_H
+
+#include <linux/sched.h>
+#include <asm/mipsregs.h>
+
+#define OCTEON_CR_OPCODE_PRIORITY 300
+
+extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *state);
+extern void octeon_crypto_disable(struct octeon_cop2_state *state,
+ unsigned long flags);
+
+/*
+ * Macros needed to implement MD5:
+ */
+
+/*
+ * The index can be 0-1.
+ */
+#define write_octeon_64bit_hash_dword(value, index) \
+do { \
+ __asm__ __volatile__ ( \
+ "dmtc2 %[rt],0x0048+" STR(index) \
+ : \
+ : [rt] "d" (value)); \
+} while (0)
+
+/*
+ * The index can be 0-1.
+ */
+#define read_octeon_64bit_hash_dword(index) \
+({ \
+ u64 __value; \
+ \
+ __asm__ __volatile__ ( \
+ "dmfc2 %[rt],0x0048+" STR(index) \
+ : [rt] "=d" (__value) \
+ : ); \
+ \
+ __value; \
+})
+
+/*
+ * The index can be 0-6.
+ */
+#define write_octeon_64bit_block_dword(value, index) \
+do { \
+ __asm__ __volatile__ ( \
+ "dmtc2 %[rt],0x0040+" STR(index) \
+ : \
+ : [rt] "d" (value)); \
+} while (0)
+
+/*
+ * The value is the final block dword (64-bit).
+ */
+#define octeon_md5_start(value) \
+do { \
+ __asm__ __volatile__ ( \
+ "dmtc2 %[rt],0x4047" \
+ : \
+ : [rt] "d" (value)); \
+} while (0)
+
+#endif /* __LINUX_OCTEON_CRYPTO_H */
--- /dev/null
+/*
+ * Cryptographic API.
+ *
+ * MD5 Message Digest Algorithm (RFC1321).
+ *
+ * Adapted for OCTEON by Aaro Koskinen <aaro.koskinen@iki.fi>.
+ *
+ * Based on crypto/md5.c, which is:
+ *
+ * Derived from cryptoapi implementation, originally based on the
+ * public domain implementation written by Colin Plumb in 1993.
+ *
+ * Copyright (c) Cryptoapi developers.
+ * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <crypto/md5.h>
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/string.h>
+#include <asm/byteorder.h>
+#include <linux/cryptohash.h>
+#include <asm/octeon/octeon.h>
+#include <crypto/internal/hash.h>
+
+#include "octeon-crypto.h"
+
+/*
+ * We pass everything as 64-bit. OCTEON can handle misaligned data.
+ */
+
+static void octeon_md5_store_hash(struct md5_state *ctx)
+{
+ u64 *hash = (u64 *)ctx->hash;
+
+ write_octeon_64bit_hash_dword(hash[0], 0);
+ write_octeon_64bit_hash_dword(hash[1], 1);
+}
+
+static void octeon_md5_read_hash(struct md5_state *ctx)
+{
+ u64 *hash = (u64 *)ctx->hash;
+
+ hash[0] = read_octeon_64bit_hash_dword(0);
+ hash[1] = read_octeon_64bit_hash_dword(1);
+}
+
+static void octeon_md5_transform(const void *_block)
+{
+ const u64 *block = _block;
+
+ write_octeon_64bit_block_dword(block[0], 0);
+ write_octeon_64bit_block_dword(block[1], 1);
+ write_octeon_64bit_block_dword(block[2], 2);
+ write_octeon_64bit_block_dword(block[3], 3);
+ write_octeon_64bit_block_dword(block[4], 4);
+ write_octeon_64bit_block_dword(block[5], 5);
+ write_octeon_64bit_block_dword(block[6], 6);
+ octeon_md5_start(block[7]);
+}
+
+static int octeon_md5_init(struct shash_desc *desc)
+{
+ struct md5_state *mctx = shash_desc_ctx(desc);
+
+ mctx->hash[0] = cpu_to_le32(0x67452301);
+ mctx->hash[1] = cpu_to_le32(0xefcdab89);
+ mctx->hash[2] = cpu_to_le32(0x98badcfe);
+ mctx->hash[3] = cpu_to_le32(0x10325476);
+ mctx->byte_count = 0;
+
+ return 0;
+}
+
+static int octeon_md5_update(struct shash_desc *desc, const u8 *data,
+ unsigned int len)
+{
+ struct md5_state *mctx = shash_desc_ctx(desc);
+ const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f);
+ struct octeon_cop2_state state;
+ unsigned long flags;
+
+ mctx->byte_count += len;
+
+ if (avail > len) {
+ memcpy((char *)mctx->block + (sizeof(mctx->block) - avail),
+ data, len);
+ return 0;
+ }
+
+ memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), data,
+ avail);
+
+ local_bh_disable();
+ preempt_disable();
+ flags = octeon_crypto_enable(&state);
+ octeon_md5_store_hash(mctx);
+
+ octeon_md5_transform(mctx->block);
+ data += avail;
+ len -= avail;
+
+ while (len >= sizeof(mctx->block)) {
+ octeon_md5_transform(data);
+ data += sizeof(mctx->block);
+ len -= sizeof(mctx->block);
+ }
+
+ octeon_md5_read_hash(mctx);
+ octeon_crypto_disable(&state, flags);
+ preempt_enable();
+ local_bh_enable();
+
+ memcpy(mctx->block, data, len);
+
+ return 0;
+}
+
+static int octeon_md5_final(struct shash_desc *desc, u8 *out)
+{
+ struct md5_state *mctx = shash_desc_ctx(desc);
+ const unsigned int offset = mctx->byte_count & 0x3f;
+ char *p = (char *)mctx->block + offset;
+ int padding = 56 - (offset + 1);
+ struct octeon_cop2_state state;
+ unsigned long flags;
+
+ *p++ = 0x80;
+
+ local_bh_disable();
+ preempt_disable();
+ flags = octeon_crypto_enable(&state);
+ octeon_md5_store_hash(mctx);
+
+ if (padding < 0) {
+ memset(p, 0x00, padding + sizeof(u64));
+ octeon_md5_transform(mctx->block);
+ p = (char *)mctx->block;
+ padding = 56;
+ }
+
+ memset(p, 0, padding);
+ mctx->block[14] = cpu_to_le32(mctx->byte_count << 3);
+ mctx->block[15] = cpu_to_le32(mctx->byte_count >> 29);
+ octeon_md5_transform(mctx->block);
+
+ octeon_md5_read_hash(mctx);
+ octeon_crypto_disable(&state, flags);
+ preempt_enable();
+ local_bh_enable();
+
+ memcpy(out, mctx->hash, sizeof(mctx->hash));
+ memset(mctx, 0, sizeof(*mctx));
+
+ return 0;
+}
+
+static int octeon_md5_export(struct shash_desc *desc, void *out)
+{
+ struct md5_state *ctx = shash_desc_ctx(desc);
+
+ memcpy(out, ctx, sizeof(*ctx));
+ return 0;
+}
+
+static int octeon_md5_import(struct shash_desc *desc, const void *in)
+{
+ struct md5_state *ctx = shash_desc_ctx(desc);
+
+ memcpy(ctx, in, sizeof(*ctx));
+ return 0;
+}
+
+static struct shash_alg alg = {
+ .digestsize = MD5_DIGEST_SIZE,
+ .init = octeon_md5_init,
+ .update = octeon_md5_update,
+ .final = octeon_md5_final,
+ .export = octeon_md5_export,
+ .import = octeon_md5_import,
+ .descsize = sizeof(struct md5_state),
+ .statesize = sizeof(struct md5_state),
+ .base = {
+ .cra_name = "md5",
+ .cra_driver_name= "octeon-md5",
+ .cra_priority = OCTEON_CR_OPCODE_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_SHASH,
+ .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ }
+};
+
+static int __init md5_mod_init(void)
+{
+ if (!octeon_has_crypto())
+ return -ENOTSUPP;
+ return crypto_register_shash(&alg);
+}
+
+static void __exit md5_mod_fini(void)
+{
+ crypto_unregister_shash(&alg);
+}
+
+module_init(md5_mod_init);
+module_exit(md5_mod_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("MD5 Message Digest Algorithm (OCTEON)");
+MODULE_AUTHOR("Aaro Koskinen <aaro.koskinen@iki.fi>");
#include <asm/octeon/octeon.h>
+enum octeon_feature_bits __octeon_feature_bits __read_mostly;
+EXPORT_SYMBOL_GPL(__octeon_feature_bits);
+
/**
* Read a byte of fuse data
* @byte_addr: address to read
else
suffix = "NSP";
+ if (!fus_dat2.s.nocrypto)
+ __octeon_feature_bits |= OCTEON_HAS_CRYPTO;
+
/*
* Assume pass number is encoded using <5:3><2:0>. Exceptions
* will be fixed later.
OCTEON_FEATURE_SAAD,
/* Does this Octeon support the ZIP offload engine? */
OCTEON_FEATURE_ZIP,
- /* Does this Octeon support crypto acceleration using COP2? */
- OCTEON_FEATURE_CRYPTO,
OCTEON_FEATURE_DORM_CRYPTO,
/* Does this Octeon support PCI express? */
OCTEON_FEATURE_PCIE,
OCTEON_MAX_FEATURE
};
+enum octeon_feature_bits {
+ OCTEON_HAS_CRYPTO = 0x0001, /* Crypto acceleration using COP2 */
+};
+extern enum octeon_feature_bits __octeon_feature_bits;
+
+/**
+ * octeon_has_crypto() - Check if this OCTEON has crypto acceleration support.
+ *
+ * Returns: Non-zero if the feature exists. Zero if the feature does not exist.
+ */
+static inline int octeon_has_crypto(void)
+{
+ return __octeon_feature_bits & OCTEON_HAS_CRYPTO;
+}
+
/**
* Determine if the current Octeon supports a specific feature. These
* checks have been optimized to be fairly quick, but they should still
extern const char *octeon_get_boot_argument(int arg);
extern void octeon_hal_setup_reserved32(void);
extern void octeon_user_io_init(void);
-struct octeon_cop2_state;
-extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *state);
-extern void octeon_crypto_disable(struct octeon_cop2_state *state,
- unsigned long flags);
-extern asmlinkage void octeon_cop2_restore(struct octeon_cop2_state *task);
extern void octeon_init_cvmcount(void);
extern void octeon_setup_delays(void);
module_exit(aes_sparc64_mod_fini);
MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated");
+MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, sparc64 aes opcode accelerated");
MODULE_ALIAS_CRYPTO("aes");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Camellia Cipher Algorithm, sparc64 camellia opcode accelerated");
-MODULE_ALIAS_CRYPTO("aes");
+MODULE_ALIAS_CRYPTO("camellia");
#include "crop_devid.c"
MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
MODULE_ALIAS_CRYPTO("des");
+MODULE_ALIAS_CRYPTO("des3_ede");
#include "crop_devid.c"
module_exit(md5_sparc64_mod_fini);
MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("MD5 Secure Hash Algorithm, sparc64 md5 opcode accelerated");
+MODULE_DESCRIPTION("MD5 Message Digest Algorithm, sparc64 md5 opcode accelerated");
MODULE_ALIAS_CRYPTO("md5");
#include <linux/linkage.h>
#include <asm/inst.h>
+/*
+ * The following macros are used to move an (un)aligned 16 byte value to/from
+ * an XMM register. This can done for either FP or integer values, for FP use
+ * movaps (move aligned packed single) or integer use movdqa (move double quad
+ * aligned). It doesn't make a performance difference which instruction is used
+ * since Nehalem (original Core i7) was released. However, the movaps is a byte
+ * shorter, so that is the one we'll use for now. (same for unaligned).
+ */
+#define MOVADQ movaps
+#define MOVUDQ movups
+
#ifdef __x86_64__
+
.data
.align 16
.Lgf128mul_x_ble_mask:
.octa 0x00000000000000010000000000000087
-
POLY: .octa 0xC2000000000000000000000000000001
TWOONE: .octa 0x00000001000000000000000000000001
#define arg8 STACK_OFFSET+16(%r14)
#define arg9 STACK_OFFSET+24(%r14)
#define arg10 STACK_OFFSET+32(%r14)
+#define keysize 2*15*16(%arg1)
#endif
.macro INITIAL_BLOCKS_DEC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
+ MOVADQ SHUF_MASK(%rip), %xmm14
mov arg7, %r10 # %r10 = AAD
mov arg8, %r12 # %r12 = aadLen
mov %r12, %r11
pxor %xmm\i, %xmm\i
+
_get_AAD_loop\num_initial_blocks\operation:
movd (%r10), \TMP1
pslldq $12, \TMP1
add $4, %r10
sub $4, %r12
jne _get_AAD_loop\num_initial_blocks\operation
+
cmp $16, %r11
je _get_AAD_loop2_done\num_initial_blocks\operation
+
mov $16, %r12
_get_AAD_loop2\num_initial_blocks\operation:
psrldq $4, %xmm\i
sub $4, %r12
cmp %r11, %r12
jne _get_AAD_loop2\num_initial_blocks\operation
+
_get_AAD_loop2_done\num_initial_blocks\operation:
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data
xor %r11, %r11 # initialise the data pointer offset as zero
mov %arg5, %rax # %rax = *Y0
movdqu (%rax), \XMM0 # XMM0 = Y0
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, \XMM0
.if (\i == 5) || (\i == 6) || (\i == 7)
+ MOVADQ ONE(%RIP),\TMP1
+ MOVADQ (%arg1),\TMP2
.irpc index, \i_seq
- paddd ONE(%rip), \XMM0 # INCR Y0
+ paddd \TMP1, \XMM0 # INCR Y0
movdqa \XMM0, %xmm\index
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap
-
-.endr
-.irpc index, \i_seq
- pxor 16*0(%arg1), %xmm\index
-.endr
-.irpc index, \i_seq
- movaps 0x10(%rdi), \TMP1
- AESENC \TMP1, %xmm\index # Round 1
-.endr
-.irpc index, \i_seq
- movaps 0x20(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
-.irpc index, \i_seq
- movaps 0x30(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
-.irpc index, \i_seq
- movaps 0x40(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
-.irpc index, \i_seq
- movaps 0x50(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
-.irpc index, \i_seq
- movaps 0x60(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
+ pxor \TMP2, %xmm\index
.endr
-.irpc index, \i_seq
- movaps 0x70(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
-.irpc index, \i_seq
- movaps 0x80(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
-.irpc index, \i_seq
- movaps 0x90(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
+ lea 0x10(%arg1),%r10
+ mov keysize,%eax
+ shr $2,%eax # 128->4, 192->6, 256->8
+ add $5,%eax # 128->9, 192->11, 256->13
+
+aes_loop_initial_dec\num_initial_blocks:
+ MOVADQ (%r10),\TMP1
+.irpc index, \i_seq
+ AESENC \TMP1, %xmm\index
.endr
+ add $16,%r10
+ sub $1,%eax
+ jnz aes_loop_initial_dec\num_initial_blocks
+
+ MOVADQ (%r10), \TMP1
.irpc index, \i_seq
- movaps 0xa0(%arg1), \TMP1
- AESENCLAST \TMP1, %xmm\index # Round 10
+ AESENCLAST \TMP1, %xmm\index # Last Round
.endr
.irpc index, \i_seq
movdqu (%arg3 , %r11, 1), \TMP1
add $16, %r11
movdqa \TMP1, %xmm\index
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, %xmm\index
-
- # prepare plaintext/ciphertext for GHASH computation
+ # prepare plaintext/ciphertext for GHASH computation
.endr
.endif
GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
* Precomputations for HashKey parallel with encryption of first 4 blocks.
* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
*/
- paddd ONE(%rip), \XMM0 # INCR Y0
- movdqa \XMM0, \XMM1
- movdqa SHUF_MASK(%rip), %xmm14
+ MOVADQ ONE(%rip), \TMP1
+ paddd \TMP1, \XMM0 # INCR Y0
+ MOVADQ \XMM0, \XMM1
PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
- paddd ONE(%rip), \XMM0 # INCR Y0
- movdqa \XMM0, \XMM2
- movdqa SHUF_MASK(%rip), %xmm14
+ paddd \TMP1, \XMM0 # INCR Y0
+ MOVADQ \XMM0, \XMM2
PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
- paddd ONE(%rip), \XMM0 # INCR Y0
- movdqa \XMM0, \XMM3
- movdqa SHUF_MASK(%rip), %xmm14
+ paddd \TMP1, \XMM0 # INCR Y0
+ MOVADQ \XMM0, \XMM3
PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
- paddd ONE(%rip), \XMM0 # INCR Y0
- movdqa \XMM0, \XMM4
- movdqa SHUF_MASK(%rip), %xmm14
+ paddd \TMP1, \XMM0 # INCR Y0
+ MOVADQ \XMM0, \XMM4
PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
- pxor 16*0(%arg1), \XMM1
- pxor 16*0(%arg1), \XMM2
- pxor 16*0(%arg1), \XMM3
- pxor 16*0(%arg1), \XMM4
+ MOVADQ 0(%arg1),\TMP1
+ pxor \TMP1, \XMM1
+ pxor \TMP1, \XMM2
+ pxor \TMP1, \XMM3
+ pxor \TMP1, \XMM4
movdqa \TMP3, \TMP5
pshufd $78, \TMP3, \TMP1
pxor \TMP3, \TMP1
pshufd $78, \TMP5, \TMP1
pxor \TMP5, \TMP1
movdqa \TMP1, HashKey_4_k(%rsp)
- movaps 0xa0(%arg1), \TMP2
+ lea 0xa0(%arg1),%r10
+ mov keysize,%eax
+ shr $2,%eax # 128->4, 192->6, 256->8
+ sub $4,%eax # 128->0, 192->2, 256->4
+ jz aes_loop_pre_dec_done\num_initial_blocks
+
+aes_loop_pre_dec\num_initial_blocks:
+ MOVADQ (%r10),\TMP2
+.irpc index, 1234
+ AESENC \TMP2, %xmm\index
+.endr
+ add $16,%r10
+ sub $1,%eax
+ jnz aes_loop_pre_dec\num_initial_blocks
+
+aes_loop_pre_dec_done\num_initial_blocks:
+ MOVADQ (%r10), \TMP2
AESENCLAST \TMP2, \XMM1
AESENCLAST \TMP2, \XMM2
AESENCLAST \TMP2, \XMM3
movdqu \XMM4, 16*3(%arg2 , %r11 , 1)
movdqa \TMP1, \XMM4
add $64, %r11
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
pxor \XMMDst, \XMM1
# combine GHASHed value with the corresponding ciphertext
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
_initial_blocks_done\num_initial_blocks\operation:
.macro INITIAL_BLOCKS_ENC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
+ MOVADQ SHUF_MASK(%rip), %xmm14
mov arg7, %r10 # %r10 = AAD
mov arg8, %r12 # %r12 = aadLen
mov %r12, %r11
cmp %r11, %r12
jne _get_AAD_loop2\num_initial_blocks\operation
_get_AAD_loop2_done\num_initial_blocks\operation:
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data
xor %r11, %r11 # initialise the data pointer offset as zero
mov %arg5, %rax # %rax = *Y0
movdqu (%rax), \XMM0 # XMM0 = Y0
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, \XMM0
.if (\i == 5) || (\i == 6) || (\i == 7)
-.irpc index, \i_seq
- paddd ONE(%rip), \XMM0 # INCR Y0
- movdqa \XMM0, %xmm\index
- movdqa SHUF_MASK(%rip), %xmm14
- PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap
-.endr
-.irpc index, \i_seq
- pxor 16*0(%arg1), %xmm\index
-.endr
-.irpc index, \i_seq
- movaps 0x10(%rdi), \TMP1
- AESENC \TMP1, %xmm\index # Round 1
-.endr
-.irpc index, \i_seq
- movaps 0x20(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
+ MOVADQ ONE(%RIP),\TMP1
+ MOVADQ 0(%arg1),\TMP2
.irpc index, \i_seq
- movaps 0x30(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
+ paddd \TMP1, \XMM0 # INCR Y0
+ MOVADQ \XMM0, %xmm\index
+ PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap
+ pxor \TMP2, %xmm\index
.endr
-.irpc index, \i_seq
- movaps 0x40(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
-.irpc index, \i_seq
- movaps 0x50(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
-.irpc index, \i_seq
- movaps 0x60(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
-.irpc index, \i_seq
- movaps 0x70(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
-.irpc index, \i_seq
- movaps 0x80(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
-.endr
-.irpc index, \i_seq
- movaps 0x90(%arg1), \TMP1
- AESENC \TMP1, %xmm\index # Round 2
+ lea 0x10(%arg1),%r10
+ mov keysize,%eax
+ shr $2,%eax # 128->4, 192->6, 256->8
+ add $5,%eax # 128->9, 192->11, 256->13
+
+aes_loop_initial_enc\num_initial_blocks:
+ MOVADQ (%r10),\TMP1
+.irpc index, \i_seq
+ AESENC \TMP1, %xmm\index
.endr
+ add $16,%r10
+ sub $1,%eax
+ jnz aes_loop_initial_enc\num_initial_blocks
+
+ MOVADQ (%r10), \TMP1
.irpc index, \i_seq
- movaps 0xa0(%arg1), \TMP1
- AESENCLAST \TMP1, %xmm\index # Round 10
+ AESENCLAST \TMP1, %xmm\index # Last Round
.endr
.irpc index, \i_seq
movdqu (%arg3 , %r11, 1), \TMP1
movdqu %xmm\index, (%arg2 , %r11, 1)
# write back plaintext/ciphertext for num_initial_blocks
add $16, %r11
-
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, %xmm\index
# prepare plaintext/ciphertext for GHASH computation
* Precomputations for HashKey parallel with encryption of first 4 blocks.
* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
*/
- paddd ONE(%rip), \XMM0 # INCR Y0
- movdqa \XMM0, \XMM1
- movdqa SHUF_MASK(%rip), %xmm14
+ MOVADQ ONE(%RIP),\TMP1
+ paddd \TMP1, \XMM0 # INCR Y0
+ MOVADQ \XMM0, \XMM1
PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
- paddd ONE(%rip), \XMM0 # INCR Y0
- movdqa \XMM0, \XMM2
- movdqa SHUF_MASK(%rip), %xmm14
+ paddd \TMP1, \XMM0 # INCR Y0
+ MOVADQ \XMM0, \XMM2
PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
- paddd ONE(%rip), \XMM0 # INCR Y0
- movdqa \XMM0, \XMM3
- movdqa SHUF_MASK(%rip), %xmm14
+ paddd \TMP1, \XMM0 # INCR Y0
+ MOVADQ \XMM0, \XMM3
PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
- paddd ONE(%rip), \XMM0 # INCR Y0
- movdqa \XMM0, \XMM4
- movdqa SHUF_MASK(%rip), %xmm14
+ paddd \TMP1, \XMM0 # INCR Y0
+ MOVADQ \XMM0, \XMM4
PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
- pxor 16*0(%arg1), \XMM1
- pxor 16*0(%arg1), \XMM2
- pxor 16*0(%arg1), \XMM3
- pxor 16*0(%arg1), \XMM4
+ MOVADQ 0(%arg1),\TMP1
+ pxor \TMP1, \XMM1
+ pxor \TMP1, \XMM2
+ pxor \TMP1, \XMM3
+ pxor \TMP1, \XMM4
movdqa \TMP3, \TMP5
pshufd $78, \TMP3, \TMP1
pxor \TMP3, \TMP1
pshufd $78, \TMP5, \TMP1
pxor \TMP5, \TMP1
movdqa \TMP1, HashKey_4_k(%rsp)
- movaps 0xa0(%arg1), \TMP2
+ lea 0xa0(%arg1),%r10
+ mov keysize,%eax
+ shr $2,%eax # 128->4, 192->6, 256->8
+ sub $4,%eax # 128->0, 192->2, 256->4
+ jz aes_loop_pre_enc_done\num_initial_blocks
+
+aes_loop_pre_enc\num_initial_blocks:
+ MOVADQ (%r10),\TMP2
+.irpc index, 1234
+ AESENC \TMP2, %xmm\index
+.endr
+ add $16,%r10
+ sub $1,%eax
+ jnz aes_loop_pre_enc\num_initial_blocks
+
+aes_loop_pre_enc_done\num_initial_blocks:
+ MOVADQ (%r10), \TMP2
AESENCLAST \TMP2, \XMM1
AESENCLAST \TMP2, \XMM2
AESENCLAST \TMP2, \XMM3
movdqu \XMM4, 16*3(%arg2 , %r11 , 1)
add $64, %r11
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
pxor \XMMDst, \XMM1
# combine GHASHed value with the corresponding ciphertext
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
- movdqa SHUF_MASK(%rip), %xmm14
PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
_initial_blocks_done\num_initial_blocks\operation:
AESENC \TMP3, \XMM3
AESENC \TMP3, \XMM4
PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0
- movaps 0xa0(%arg1), \TMP3
+ lea 0xa0(%arg1),%r10
+ mov keysize,%eax
+ shr $2,%eax # 128->4, 192->6, 256->8
+ sub $4,%eax # 128->0, 192->2, 256->4
+ jz aes_loop_par_enc_done
+
+aes_loop_par_enc:
+ MOVADQ (%r10),\TMP3
+.irpc index, 1234
+ AESENC \TMP3, %xmm\index
+.endr
+ add $16,%r10
+ sub $1,%eax
+ jnz aes_loop_par_enc
+
+aes_loop_par_enc_done:
+ MOVADQ (%r10), \TMP3
AESENCLAST \TMP3, \XMM1 # Round 10
AESENCLAST \TMP3, \XMM2
AESENCLAST \TMP3, \XMM3
AESENC \TMP3, \XMM3
AESENC \TMP3, \XMM4
PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0
- movaps 0xa0(%arg1), \TMP3
- AESENCLAST \TMP3, \XMM1 # Round 10
+ lea 0xa0(%arg1),%r10
+ mov keysize,%eax
+ shr $2,%eax # 128->4, 192->6, 256->8
+ sub $4,%eax # 128->0, 192->2, 256->4
+ jz aes_loop_par_dec_done
+
+aes_loop_par_dec:
+ MOVADQ (%r10),\TMP3
+.irpc index, 1234
+ AESENC \TMP3, %xmm\index
+.endr
+ add $16,%r10
+ sub $1,%eax
+ jnz aes_loop_par_dec
+
+aes_loop_par_dec_done:
+ MOVADQ (%r10), \TMP3
+ AESENCLAST \TMP3, \XMM1 # last round
AESENCLAST \TMP3, \XMM2
AESENCLAST \TMP3, \XMM3
AESENCLAST \TMP3, \XMM4
pxor \TMP6, \XMMDst # reduced result is in XMMDst
.endm
-/* Encryption of a single block done*/
-.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1
- pxor (%arg1), \XMM0
- movaps 16(%arg1), \TMP1
- AESENC \TMP1, \XMM0
- movaps 32(%arg1), \TMP1
- AESENC \TMP1, \XMM0
- movaps 48(%arg1), \TMP1
- AESENC \TMP1, \XMM0
- movaps 64(%arg1), \TMP1
- AESENC \TMP1, \XMM0
- movaps 80(%arg1), \TMP1
- AESENC \TMP1, \XMM0
- movaps 96(%arg1), \TMP1
- AESENC \TMP1, \XMM0
- movaps 112(%arg1), \TMP1
- AESENC \TMP1, \XMM0
- movaps 128(%arg1), \TMP1
- AESENC \TMP1, \XMM0
- movaps 144(%arg1), \TMP1
- AESENC \TMP1, \XMM0
- movaps 160(%arg1), \TMP1
- AESENCLAST \TMP1, \XMM0
-.endm
+/* Encryption of a single block
+* uses eax & r10
+*/
+.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1
+ pxor (%arg1), \XMM0
+ mov keysize,%eax
+ shr $2,%eax # 128->4, 192->6, 256->8
+ add $5,%eax # 128->9, 192->11, 256->13
+ lea 16(%arg1), %r10 # get first expanded key address
+
+_esb_loop_\@:
+ MOVADQ (%r10),\TMP1
+ AESENC \TMP1,\XMM0
+ add $16,%r10
+ sub $1,%eax
+ jnz _esb_loop_\@
+
+ MOVADQ (%r10),\TMP1
+ AESENCLAST \TMP1,\XMM0
+.endm
/*****************************************************************************
* void aesni_gcm_dec(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
* u8 *out, // Plaintext output. Encrypt in-place is allowed.
#include <asm/crypto/glue_helper.h>
#endif
+
/* This data is stored at the end of the crypto_tfm struct.
* It's a type of per "session" data storage location.
* This needs to be 16 byte aligned.
u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
u8 *auth_tag, unsigned long auth_tag_len)
{
- if (plaintext_len < AVX_GEN2_OPTSIZE) {
+ struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
+ if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){
aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
aad_len, auth_tag, auth_tag_len);
} else {
u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
u8 *auth_tag, unsigned long auth_tag_len)
{
- if (ciphertext_len < AVX_GEN2_OPTSIZE) {
+ struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
+ if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad,
aad_len, auth_tag, auth_tag_len);
} else {
u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
u8 *auth_tag, unsigned long auth_tag_len)
{
- if (plaintext_len < AVX_GEN2_OPTSIZE) {
+ struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
+ if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad,
aad_len, auth_tag, auth_tag_len);
} else if (plaintext_len < AVX_GEN4_OPTSIZE) {
u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
u8 *auth_tag, unsigned long auth_tag_len)
{
- if (ciphertext_len < AVX_GEN2_OPTSIZE) {
+ struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
+ if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey,
aad, aad_len, auth_tag, auth_tag_len);
} else if (ciphertext_len < AVX_GEN4_OPTSIZE) {
kernel_fpu_begin();
while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
- nbytes & AES_BLOCK_MASK, walk.iv);
+ nbytes & AES_BLOCK_MASK, walk.iv);
nbytes &= AES_BLOCK_SIZE - 1;
err = blkcipher_walk_done(desc, &walk, nbytes);
}
}
/*Account for 4 byte nonce at the end.*/
key_len -= 4;
- if (key_len != AES_KEYSIZE_128) {
+ if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
+ key_len != AES_KEYSIZE_256) {
crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
return -EINVAL;
}
__be32 counter = cpu_to_be32(1);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+ u32 key_len = ctx->aes_key_expanded.key_length;
void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv_tab[16+AESNI_ALIGN];
/* to 8 or 12 bytes */
if (unlikely(req->assoclen != 8 && req->assoclen != 12))
return -EINVAL;
+ if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16))
+ return -EINVAL;
+ if (unlikely(key_len != AES_KEYSIZE_128 &&
+ key_len != AES_KEYSIZE_192 &&
+ key_len != AES_KEYSIZE_256))
+ return -EINVAL;
+
/* IV below built */
for (i = 0; i < 4; i++)
*(iv+i) = ctx->nonce[i];
int retval = 0;
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+ u32 key_len = ctx->aes_key_expanded.key_length;
void *aes_ctx = &(ctx->aes_key_expanded);
unsigned long auth_tag_len = crypto_aead_authsize(tfm);
u8 iv_and_authTag[32+AESNI_ALIGN];
if (unlikely((req->cryptlen < auth_tag_len) ||
(req->assoclen != 8 && req->assoclen != 12)))
return -EINVAL;
+ if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16))
+ return -EINVAL;
+ if (unlikely(key_len != AES_KEYSIZE_128 &&
+ key_len != AES_KEYSIZE_192 &&
+ key_len != AES_KEYSIZE_256))
+ return -EINVAL;
+
/* Assuming we are supporting rfc4106 64-bit extended */
/* sequence numbers We need to have the AAD length */
/* equal to 8 or 12 bytes */
MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized");
MODULE_ALIAS_CRYPTO("des3_ede");
MODULE_ALIAS_CRYPTO("des3_ede-asm");
-MODULE_ALIAS_CRYPTO("des");
-MODULE_ALIAS_CRYPTO("des-asm");
MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>");
help
MD5 message digest algorithm (RFC1321).
+config CRYPTO_MD5_OCTEON
+ tristate "MD5 digest algorithm (OCTEON)"
+ depends on CPU_CAVIUM_OCTEON
+ select CRYPTO_MD5
+ select CRYPTO_HASH
+ help
+ MD5 message digest algorithm (RFC1321) implemented
+ using OCTEON crypto instructions, when available.
+
config CRYPTO_MD5_SPARC64
tristate "MD5 digest algorithm (SPARC64)"
depends on SPARC64
This option enables the user-spaces interface for symmetric
key cipher algorithms.
+config CRYPTO_USER_API_RNG
+ tristate "User-space interface for random number generator algorithms"
+ depends on NET
+ select CRYPTO_RNG
+ select CRYPTO_USER_API
+ help
+ This option enables the user-spaces interface for random
+ number generator algorithms.
+
config CRYPTO_HASH_INFO
bool
obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o
obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o
obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o
+obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o
#
# generic algorithms and the async_tx api
static inline u8 *ablkcipher_get_spot(u8 *start, unsigned int len)
{
u8 *end_page = (u8 *)(((unsigned long)(start + len - 1)) & PAGE_MASK);
+
return max(start, end_page);
}
if (n == len_this_page)
break;
n -= len_this_page;
- scatterwalk_start(&walk->out, scatterwalk_sg_next(walk->out.sg));
+ scatterwalk_start(&walk->out, sg_next(walk->out.sg));
}
return bsize;
walk->iv = req->info;
if (unlikely(((unsigned long)walk->iv & alignmask))) {
int err = ablkcipher_copy_iv(walk, tfm, alignmask);
+
if (err)
return err;
}
if (IS_ERR(inst))
goto put_tmpl;
- if ((err = crypto_register_instance(tmpl, inst))) {
+ err = crypto_register_instance(tmpl, inst);
+ if (err) {
tmpl->free(inst);
goto put_tmpl;
}
if (IS_ERR(inst))
goto put_tmpl;
- if ((err = crypto_register_instance(tmpl, inst))) {
+ err = crypto_register_instance(tmpl, inst);
+ if (err) {
tmpl->free(inst);
goto put_tmpl;
}
err = type->setkey(ask->private, key, keylen);
out:
- sock_kfree_s(sk, key, keylen);
+ sock_kzfree_s(sk, key, keylen);
return err;
}
goto unlock;
err = alg_setkey(sk, optval, optlen);
+ break;
+ case ALG_SET_AEAD_AUTHSIZE:
+ if (sock->state == SS_CONNECTED)
+ goto unlock;
+ if (!type->setauthsize)
+ goto unlock;
+ err = type->setauthsize(ask->private, optlen);
}
unlock:
if (cmsg->cmsg_level != SOL_ALG)
continue;
- switch(cmsg->cmsg_type) {
+ switch (cmsg->cmsg_type) {
case ALG_SET_IV:
if (cmsg->cmsg_len < CMSG_LEN(sizeof(*con->iv)))
return -EINVAL;
if (offset & alignmask) {
unsigned int unaligned = alignmask + 1 - (offset & alignmask);
+
if (nbytes > unaligned)
nbytes = unaligned;
}
if (!walk->total)
return 0;
- walk->sg = scatterwalk_sg_next(walk->sg);
+ walk->sg = sg_next(walk->sg);
return hash_walk_new_entry(walk);
}
list = &tmpl->instances;
hlist_for_each_entry(inst, list, list) {
int err = crypto_remove_alg(&inst->alg, &users);
+
BUG_ON(err);
}
--- /dev/null
+/*
+ * algif_rng: User-space interface for random number generators
+ *
+ * This file provides the user-space API for random number generators.
+ *
+ * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, and the entire permission notice in its entirety,
+ * including the disclaimer of warranties.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * ALTERNATIVELY, this product may be distributed under the terms of
+ * the GNU General Public License, in which case the provisions of the GPL2
+ * are required INSTEAD OF the above restrictions. (This clause is
+ * necessary due to a potential bad interaction between the GPL and
+ * the restrictions contained in a BSD-style copyright.)
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
+ * WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+ * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <linux/module.h>
+#include <crypto/rng.h>
+#include <linux/random.h>
+#include <crypto/if_alg.h>
+#include <linux/net.h>
+#include <net/sock.h>
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
+MODULE_DESCRIPTION("User-space interface for random number generators");
+
+struct rng_ctx {
+#define MAXSIZE 128
+ unsigned int len;
+ struct crypto_rng *drng;
+};
+
+static int rng_recvmsg(struct kiocb *unused, struct socket *sock,
+ struct msghdr *msg, size_t len, int flags)
+{
+ struct sock *sk = sock->sk;
+ struct alg_sock *ask = alg_sk(sk);
+ struct rng_ctx *ctx = ask->private;
+ int err = -EFAULT;
+ int genlen = 0;
+ u8 result[MAXSIZE];
+
+ if (len == 0)
+ return 0;
+ if (len > MAXSIZE)
+ len = MAXSIZE;
+
+ /*
+ * although not strictly needed, this is a precaution against coding
+ * errors
+ */
+ memset(result, 0, len);
+
+ /*
+ * The enforcement of a proper seeding of an RNG is done within an
+ * RNG implementation. Some RNGs (DRBG, krng) do not need specific
+ * seeding as they automatically seed. The X9.31 DRNG will return
+ * an error if it was not seeded properly.
+ */
+ genlen = crypto_rng_get_bytes(ctx->drng, result, len);
+ if (genlen < 0)
+ return genlen;
+
+ err = memcpy_to_msg(msg, result, len);
+ memzero_explicit(result, genlen);
+
+ return err ? err : len;
+}
+
+static struct proto_ops algif_rng_ops = {
+ .family = PF_ALG,
+
+ .connect = sock_no_connect,
+ .socketpair = sock_no_socketpair,
+ .getname = sock_no_getname,
+ .ioctl = sock_no_ioctl,
+ .listen = sock_no_listen,
+ .shutdown = sock_no_shutdown,
+ .getsockopt = sock_no_getsockopt,
+ .mmap = sock_no_mmap,
+ .bind = sock_no_bind,
+ .accept = sock_no_accept,
+ .setsockopt = sock_no_setsockopt,
+ .poll = sock_no_poll,
+ .sendmsg = sock_no_sendmsg,
+ .sendpage = sock_no_sendpage,
+
+ .release = af_alg_release,
+ .recvmsg = rng_recvmsg,
+};
+
+static void *rng_bind(const char *name, u32 type, u32 mask)
+{
+ return crypto_alloc_rng(name, type, mask);
+}
+
+static void rng_release(void *private)
+{
+ crypto_free_rng(private);
+}
+
+static void rng_sock_destruct(struct sock *sk)
+{
+ struct alg_sock *ask = alg_sk(sk);
+ struct rng_ctx *ctx = ask->private;
+
+ sock_kfree_s(sk, ctx, ctx->len);
+ af_alg_release_parent(sk);
+}
+
+static int rng_accept_parent(void *private, struct sock *sk)
+{
+ struct rng_ctx *ctx;
+ struct alg_sock *ask = alg_sk(sk);
+ unsigned int len = sizeof(*ctx);
+
+ ctx = sock_kmalloc(sk, len, GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+
+ ctx->len = len;
+
+ /*
+ * No seeding done at that point -- if multiple accepts are
+ * done on one RNG instance, each resulting FD points to the same
+ * state of the RNG.
+ */
+
+ ctx->drng = private;
+ ask->private = ctx;
+ sk->sk_destruct = rng_sock_destruct;
+
+ return 0;
+}
+
+static int rng_setkey(void *private, const u8 *seed, unsigned int seedlen)
+{
+ /*
+ * Check whether seedlen is of sufficient size is done in RNG
+ * implementations.
+ */
+ return crypto_rng_reset(private, (u8 *)seed, seedlen);
+}
+
+static const struct af_alg_type algif_type_rng = {
+ .bind = rng_bind,
+ .release = rng_release,
+ .accept = rng_accept_parent,
+ .setkey = rng_setkey,
+ .ops = &algif_rng_ops,
+ .name = "rng",
+ .owner = THIS_MODULE
+};
+
+static int __init rng_init(void)
+{
+ return af_alg_register_type(&algif_type_rng);
+}
+
+static void __exit rng_exit(void)
+{
+ int err = af_alg_unregister_type(&algif_type_rng);
+ BUG_ON(err);
+}
+
+module_init(rng_init);
+module_exit(rng_exit);
sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
sg = sgl->sg;
+ sg_unmark_end(sg + sgl->cur);
do {
i = sgl->cur;
plen = min_t(int, len, PAGE_SIZE);
sgl->cur++;
} while (len && sgl->cur < MAX_SGL_ENTS);
+ if (!size)
+ sg_mark_end(sg + sgl->cur - 1);
+
ctx->merge = plen & (PAGE_SIZE - 1);
}
ctx->merge = 0;
sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
+ if (sgl->cur)
+ sg_unmark_end(sgl->sg + sgl->cur - 1);
+
+ sg_mark_end(sgl->sg + sgl->cur);
get_page(page);
sg_set_page(sgl->sg + sgl->cur, page, size, offset);
sgl->cur++;
if (!is_power_of_2(alg->cra_blocksize))
goto out_put_alg;
+ if (strncmp(alg->cra_name, "cbc(", 4))
+ goto out_put_alg;
+
inst = crypto_alloc_instance("cts", alg);
if (IS_ERR(inst))
goto out_put_alg;
inst->alg.cra_blkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
inst->alg.cra_blkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
- inst->alg.cra_blkcipher.geniv = "seqiv";
-
inst->alg.cra_ctxsize = sizeof(struct crypto_cts_ctx);
inst->alg.cra_init = crypto_cts_init_tfm;
*/
#include <crypto/drbg.h>
-#include <linux/string.h>
/***************************************************************
* Backend cipher definitions available to DRBG
* function. Thus, the function implicitly knows the size of the
* buffer.
*
- * The FIPS test can be called in an endless loop until it returns
- * true. Although the code looks like a potential for a deadlock, it
- * is not the case, because returning a false cannot mathematically
- * occur (except once when a reseed took place and the updated state
- * would is now set up such that the generation of new value returns
- * an identical one -- this is most unlikely and would happen only once).
- * Thus, if this function repeatedly returns false and thus would cause
- * a deadlock, the integrity of the entire kernel is lost.
- *
* @drbg DRBG handle
* @buf output buffer of random data to be checked
*
return false;
}
ret = memcmp(drbg->prev, buf, drbg_blocklen(drbg));
+ if (!ret)
+ panic("DRBG continuous self test failed\n");
memcpy(drbg->prev, buf, drbg_blocklen(drbg));
/* the test shall pass when the two compared values are not equal */
return ret != 0;
ret = 0;
out:
- memzero_explicit(iv, drbg_blocklen(drbg));
- memzero_explicit(temp, drbg_statelen(drbg));
- memzero_explicit(pad, drbg_blocklen(drbg));
+ memset(iv, 0, drbg_blocklen(drbg));
+ memset(temp, 0, drbg_statelen(drbg));
+ memset(pad, 0, drbg_blocklen(drbg));
return ret;
}
ret = 0;
out:
- memzero_explicit(temp, drbg_statelen(drbg) + drbg_blocklen(drbg));
+ memset(temp, 0, drbg_statelen(drbg) + drbg_blocklen(drbg));
if (2 != reseed)
- memzero_explicit(df_data, drbg_statelen(drbg));
+ memset(df_data, 0, drbg_statelen(drbg));
return ret;
}
len = ret;
out:
- memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg));
+ memset(drbg->scratchpad, 0, drbg_blocklen(drbg));
return len;
}
}
out:
- memzero_explicit(tmp, drbg_blocklen(drbg));
+ memset(tmp, 0, drbg_blocklen(drbg));
return ret;
}
ret = drbg_hash_df(drbg, drbg->C, drbg_statelen(drbg), &datalist2);
out:
- memzero_explicit(drbg->scratchpad, drbg_statelen(drbg));
+ memset(drbg->scratchpad, 0, drbg_statelen(drbg));
return ret;
}
drbg->scratchpad, drbg_blocklen(drbg));
out:
- memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg));
+ memset(drbg->scratchpad, 0, drbg_blocklen(drbg));
return ret;
}
}
out:
- memzero_explicit(drbg->scratchpad,
+ memset(drbg->scratchpad, 0,
(drbg_statelen(drbg) + drbg_blocklen(drbg)));
return len;
}
drbg_add_buf(drbg->V, drbg_statelen(drbg), u.req, 8);
out:
- memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg));
+ memset(drbg->scratchpad, 0, drbg_blocklen(drbg));
return len;
}
walk->offset += PAGE_SIZE - 1;
walk->offset &= PAGE_MASK;
if (walk->offset >= walk->sg->offset + walk->sg->length)
- scatterwalk_start(walk, scatterwalk_sg_next(walk->sg));
+ scatterwalk_start(walk, sg_next(walk->sg));
}
}
break;
offset += sg->length;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
scatterwalk_advance(&walk, start - offset);
do {
offset += sg->length;
n++;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
/* num_bytes is too large */
if (unlikely(!sg && (num_bytes < offset)))
if (IS_ERR(inst))
goto out;
+ if (inst->alg.cra_ablkcipher.ivsize < sizeof(u64)) {
+ skcipher_geniv_free(inst);
+ inst = ERR_PTR(-EINVAL);
+ goto out;
+ }
+
inst->alg.cra_ablkcipher.givencrypt = seqiv_givencrypt_first;
inst->alg.cra_init = seqiv_init;
if (IS_ERR(inst))
goto out;
+ if (inst->alg.cra_aead.ivsize < sizeof(u64)) {
+ aead_geniv_free(inst);
+ inst = ERR_PTR(-EINVAL);
+ goto out;
+ }
+
inst->alg.cra_aead.givencrypt = seqiv_aead_givencrypt_first;
inst->alg.cra_init = seqiv_aead_init;
int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE;
int k, rem;
- np = (np > XBUFSIZE) ? XBUFSIZE : np;
- rem = buflen % PAGE_SIZE;
if (np > XBUFSIZE) {
rem = PAGE_SIZE;
np = XBUFSIZE;
+ } else {
+ rem = buflen % PAGE_SIZE;
}
+
sg_init_table(sg, np);
- for (k = 0; k < np; ++k) {
- if (k == (np-1))
- sg_set_buf(&sg[k], xbuf[k], rem);
- else
- sg_set_buf(&sg[k], xbuf[k], PAGE_SIZE);
- }
+ np--;
+ for (k = 0; k < np; k++)
+ sg_set_buf(&sg[k], xbuf[k], PAGE_SIZE);
+
+ sg_set_buf(&sg[k], xbuf[k], rem);
}
static void test_aead_speed(const char *algo, int enc, unsigned int secs,
struct scatterlist *sgout;
const char *e;
void *assoc;
- char iv[MAX_IVLEN];
+ char *iv;
char *xbuf[XBUFSIZE];
char *xoutbuf[XBUFSIZE];
char *axbuf[XBUFSIZE];
unsigned int *b_size;
unsigned int iv_len;
+ iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
+ if (!iv)
+ return;
+
if (aad_size >= PAGE_SIZE) {
pr_err("associate data length (%u) too big\n", aad_size);
- return;
+ goto out_noxbuf;
}
if (enc == ENCRYPT)
iv_len = crypto_aead_ivsize(tfm);
if (iv_len)
- memset(&iv, 0xff, iv_len);
+ memset(iv, 0xff, iv_len);
crypto_aead_clear_flags(tfm, ~0);
printk(KERN_INFO "test %u (%d bit key, %d byte blocks): ",
out_noaxbuf:
testmgr_free_buf(xbuf);
out_noxbuf:
+ kfree(iv);
return;
}
if (ret == -EINPROGRESS || ret == -EBUSY) {
struct tcrypt_result *tr = req->base.data;
- ret = wait_for_completion_interruptible(&tr->completion);
- if (!ret)
- ret = tr->err;
+ wait_for_completion(&tr->completion);
reinit_completion(&tr->completion);
+ ret = tr->err;
}
return ret;
}
if (ret == -EINPROGRESS || ret == -EBUSY) {
struct tcrypt_result *tr = req->base.data;
- ret = wait_for_completion_interruptible(&tr->completion);
- if (!ret)
- ret = tr->err;
+ wait_for_completion(&tr->completion);
reinit_completion(&tr->completion);
+ ret = tr->err;
}
return ret;
static int wait_async_op(struct tcrypt_result *tr, int ret)
{
if (ret == -EINPROGRESS || ret == -EBUSY) {
- ret = wait_for_completion_interruptible(&tr->completion);
- if (!ret)
- ret = tr->err;
+ wait_for_completion(&tr->completion);
reinit_completion(&tr->completion);
+ ret = tr->err;
}
return ret;
}
break;
case -EINPROGRESS:
case -EBUSY:
- ret = wait_for_completion_interruptible(
- &tresult.completion);
- if (!ret && !(ret = tresult.err)) {
- reinit_completion(&tresult.completion);
+ wait_for_completion(&tresult.completion);
+ reinit_completion(&tresult.completion);
+ ret = tresult.err;
+ if (!ret)
break;
- }
/* fall through */
default:
printk(KERN_ERR "alg: hash: digest failed "
struct scatterlist *sgout;
const char *e, *d;
struct tcrypt_result result;
- unsigned int authsize;
+ unsigned int authsize, iv_len;
void *input;
void *output;
void *assoc;
memcpy(input, template[i].input, template[i].ilen);
memcpy(assoc, template[i].assoc, template[i].alen);
+ iv_len = crypto_aead_ivsize(tfm);
if (template[i].iv)
- memcpy(iv, template[i].iv, MAX_IVLEN);
+ memcpy(iv, template[i].iv, iv_len);
else
- memset(iv, 0, MAX_IVLEN);
+ memset(iv, 0, iv_len);
crypto_aead_clear_flags(tfm, ~0);
if (template[i].wk)
break;
case -EINPROGRESS:
case -EBUSY:
- ret = wait_for_completion_interruptible(
- &result.completion);
- if (!ret && !(ret = result.err)) {
- reinit_completion(&result.completion);
+ wait_for_completion(&result.completion);
+ reinit_completion(&result.completion);
+ ret = result.err;
+ if (!ret)
break;
- }
case -EBADMSG:
if (template[i].novrfy)
/* verification failure was expected */
break;
case -EINPROGRESS:
case -EBUSY:
- ret = wait_for_completion_interruptible(
- &result.completion);
- if (!ret && !(ret = result.err)) {
- reinit_completion(&result.completion);
+ wait_for_completion(&result.completion);
+ reinit_completion(&result.completion);
+ ret = result.err;
+ if (!ret)
break;
- }
case -EBADMSG:
if (template[i].novrfy)
/* verification failure was expected */
break;
case -EINPROGRESS:
case -EBUSY:
- ret = wait_for_completion_interruptible(
- &result.completion);
- if (!ret && !((ret = result.err))) {
- reinit_completion(&result.completion);
+ wait_for_completion(&result.completion);
+ reinit_completion(&result.completion);
+ ret = result.err;
+ if (!ret)
break;
- }
/* fall through */
default:
pr_err("alg: skcipher%s: %s failed on test %d for %s: ret=%d\n",
break;
case -EINPROGRESS:
case -EBUSY:
- ret = wait_for_completion_interruptible(
- &result.completion);
- if (!ret && !((ret = result.err))) {
- reinit_completion(&result.completion);
+ wait_for_completion(&result.completion);
+ reinit_completion(&result.completion);
+ ret = result.err;
+ if (!ret)
break;
- }
/* fall through */
default:
pr_err("alg: skcipher%s: %s failed on chunk test %d for %s: ret=%d\n",
}, {
.alg = "rfc4106(gcm(aes))",
.test = alg_test_aead,
+ .fips_allowed = 1,
.suite = {
.aead = {
.enc = {
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/random.h>
+#include <linux/err.h>
#include <asm/uaccess.h>
static struct hwrng *current_rng;
static struct task_struct *hwrng_fill;
static LIST_HEAD(rng_list);
+/* Protects rng_list and current_rng */
static DEFINE_MUTEX(rng_mutex);
+/* Protects rng read functions, data_avail, rng_buffer and rng_fillbuf */
+static DEFINE_MUTEX(reading_mutex);
static int data_avail;
static u8 *rng_buffer, *rng_fillbuf;
static unsigned short current_quality;
MODULE_PARM_DESC(default_quality,
"default entropy content of hwrng per mill");
+static void drop_current_rng(void);
+static int hwrng_init(struct hwrng *rng);
static void start_khwrngd(void);
static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size,
unsigned char bytes[16];
int bytes_read;
+ mutex_lock(&reading_mutex);
bytes_read = rng_get_data(rng, bytes, sizeof(bytes), 1);
+ mutex_unlock(&reading_mutex);
if (bytes_read > 0)
add_device_randomness(bytes, bytes_read);
}
-static inline int hwrng_init(struct hwrng *rng)
+static inline void cleanup_rng(struct kref *kref)
{
+ struct hwrng *rng = container_of(kref, struct hwrng, ref);
+
+ if (rng->cleanup)
+ rng->cleanup(rng);
+
+ complete(&rng->cleanup_done);
+}
+
+static int set_current_rng(struct hwrng *rng)
+{
+ int err;
+
+ BUG_ON(!mutex_is_locked(&rng_mutex));
+
+ err = hwrng_init(rng);
+ if (err)
+ return err;
+
+ drop_current_rng();
+ current_rng = rng;
+
+ return 0;
+}
+
+static void drop_current_rng(void)
+{
+ BUG_ON(!mutex_is_locked(&rng_mutex));
+ if (!current_rng)
+ return;
+
+ /* decrease last reference for triggering the cleanup */
+ kref_put(¤t_rng->ref, cleanup_rng);
+ current_rng = NULL;
+}
+
+/* Returns ERR_PTR(), NULL or refcounted hwrng */
+static struct hwrng *get_current_rng(void)
+{
+ struct hwrng *rng;
+
+ if (mutex_lock_interruptible(&rng_mutex))
+ return ERR_PTR(-ERESTARTSYS);
+
+ rng = current_rng;
+ if (rng)
+ kref_get(&rng->ref);
+
+ mutex_unlock(&rng_mutex);
+ return rng;
+}
+
+static void put_rng(struct hwrng *rng)
+{
+ /*
+ * Hold rng_mutex here so we serialize in case they set_current_rng
+ * on rng again immediately.
+ */
+ mutex_lock(&rng_mutex);
+ if (rng)
+ kref_put(&rng->ref, cleanup_rng);
+ mutex_unlock(&rng_mutex);
+}
+
+static int hwrng_init(struct hwrng *rng)
+{
+ if (kref_get_unless_zero(&rng->ref))
+ goto skip_init;
+
if (rng->init) {
int ret;
if (ret)
return ret;
}
+
+ kref_init(&rng->ref);
+ reinit_completion(&rng->cleanup_done);
+
+skip_init:
add_early_randomness(rng);
current_quality = rng->quality ? : default_quality;
return 0;
}
-static inline void hwrng_cleanup(struct hwrng *rng)
-{
- if (rng && rng->cleanup)
- rng->cleanup(rng);
-}
-
static int rng_dev_open(struct inode *inode, struct file *filp)
{
/* enforce read-only access to this chrdev */
int wait) {
int present;
+ BUG_ON(!mutex_is_locked(&reading_mutex));
if (rng->read)
return rng->read(rng, (void *)buffer, size, wait);
ssize_t ret = 0;
int err = 0;
int bytes_read, len;
+ struct hwrng *rng;
while (size) {
- if (mutex_lock_interruptible(&rng_mutex)) {
- err = -ERESTARTSYS;
+ rng = get_current_rng();
+ if (IS_ERR(rng)) {
+ err = PTR_ERR(rng);
goto out;
}
-
- if (!current_rng) {
+ if (!rng) {
err = -ENODEV;
- goto out_unlock;
+ goto out;
}
+ mutex_lock(&reading_mutex);
if (!data_avail) {
- bytes_read = rng_get_data(current_rng, rng_buffer,
+ bytes_read = rng_get_data(rng, rng_buffer,
rng_buffer_size(),
!(filp->f_flags & O_NONBLOCK));
if (bytes_read < 0) {
err = bytes_read;
- goto out_unlock;
+ goto out_unlock_reading;
}
data_avail = bytes_read;
}
if (!data_avail) {
if (filp->f_flags & O_NONBLOCK) {
err = -EAGAIN;
- goto out_unlock;
+ goto out_unlock_reading;
}
} else {
len = data_avail;
if (copy_to_user(buf + ret, rng_buffer + data_avail,
len)) {
err = -EFAULT;
- goto out_unlock;
+ goto out_unlock_reading;
}
size -= len;
ret += len;
}
- mutex_unlock(&rng_mutex);
+ mutex_unlock(&reading_mutex);
+ put_rng(rng);
if (need_resched())
schedule_timeout_interruptible(1);
}
out:
return ret ? : err;
-out_unlock:
- mutex_unlock(&rng_mutex);
+
+out_unlock_reading:
+ mutex_unlock(&reading_mutex);
+ put_rng(rng);
goto out;
}
err = -ENODEV;
list_for_each_entry(rng, &rng_list, list) {
if (strcmp(rng->name, buf) == 0) {
- if (rng == current_rng) {
- err = 0;
- break;
- }
- err = hwrng_init(rng);
- if (err)
- break;
- hwrng_cleanup(current_rng);
- current_rng = rng;
err = 0;
+ if (rng != current_rng)
+ err = set_current_rng(rng);
break;
}
}
struct device_attribute *attr,
char *buf)
{
- int err;
ssize_t ret;
- const char *name = "none";
+ struct hwrng *rng;
- err = mutex_lock_interruptible(&rng_mutex);
- if (err)
- return -ERESTARTSYS;
- if (current_rng)
- name = current_rng->name;
- ret = snprintf(buf, PAGE_SIZE, "%s\n", name);
- mutex_unlock(&rng_mutex);
+ rng = get_current_rng();
+ if (IS_ERR(rng))
+ return PTR_ERR(rng);
+
+ ret = snprintf(buf, PAGE_SIZE, "%s\n", rng ? rng->name : "none");
+ put_rng(rng);
return ret;
}
NULL);
-static void unregister_miscdev(void)
+static void __exit unregister_miscdev(void)
{
device_remove_file(rng_miscdev.this_device, &dev_attr_rng_available);
device_remove_file(rng_miscdev.this_device, &dev_attr_rng_current);
misc_deregister(&rng_miscdev);
}
-static int register_miscdev(void)
+static int __init register_miscdev(void)
{
int err;
long rc;
while (!kthread_should_stop()) {
- if (!current_rng)
+ struct hwrng *rng;
+
+ rng = get_current_rng();
+ if (IS_ERR(rng) || !rng)
break;
- rc = rng_get_data(current_rng, rng_fillbuf,
+ mutex_lock(&reading_mutex);
+ rc = rng_get_data(rng, rng_fillbuf,
rng_buffer_size(), 1);
+ mutex_unlock(&reading_mutex);
+ put_rng(rng);
if (rc <= 0) {
pr_warn("hwrng: no data available\n");
msleep_interruptible(10000);
continue;
}
+ /* Outside lock, sure, but y'know: randomness. */
add_hwgenerator_randomness((void *)rng_fillbuf, rc,
rc * current_quality * 8 >> 10);
}
goto out_unlock;
}
+ init_completion(&rng->cleanup_done);
+ complete(&rng->cleanup_done);
+
old_rng = current_rng;
- if (!old_rng) {
- err = hwrng_init(rng);
- if (err)
- goto out_unlock;
- current_rng = rng;
- }
err = 0;
if (!old_rng) {
- err = register_miscdev();
- if (err) {
- hwrng_cleanup(rng);
- current_rng = NULL;
+ err = set_current_rng(rng);
+ if (err)
goto out_unlock;
- }
}
- INIT_LIST_HEAD(&rng->list);
list_add_tail(&rng->list, &rng_list);
if (old_rng && !rng->init) {
void hwrng_unregister(struct hwrng *rng)
{
- int err;
-
mutex_lock(&rng_mutex);
list_del(&rng->list);
if (current_rng == rng) {
- hwrng_cleanup(rng);
- if (list_empty(&rng_list)) {
- current_rng = NULL;
- } else {
- current_rng = list_entry(rng_list.prev, struct hwrng, list);
- err = hwrng_init(current_rng);
- if (err)
- current_rng = NULL;
+ drop_current_rng();
+ if (!list_empty(&rng_list)) {
+ struct hwrng *tail;
+
+ tail = list_entry(rng_list.prev, struct hwrng, list);
+
+ set_current_rng(tail);
}
}
+
if (list_empty(&rng_list)) {
- unregister_miscdev();
+ mutex_unlock(&rng_mutex);
if (hwrng_fill)
kthread_stop(hwrng_fill);
- }
+ } else
+ mutex_unlock(&rng_mutex);
- mutex_unlock(&rng_mutex);
+ wait_for_completion(&rng->cleanup_done);
}
EXPORT_SYMBOL_GPL(hwrng_unregister);
-static void __exit hwrng_exit(void)
+static int __init hwrng_modinit(void)
+{
+ return register_miscdev();
+}
+
+static void __exit hwrng_modexit(void)
{
mutex_lock(&rng_mutex);
BUG_ON(current_rng);
kfree(rng_buffer);
kfree(rng_fillbuf);
mutex_unlock(&rng_mutex);
+
+ unregister_miscdev();
}
-module_exit(hwrng_exit);
+module_init(hwrng_modinit);
+module_exit(hwrng_modexit);
MODULE_DESCRIPTION("H/W Random Number Generator (RNG) driver");
MODULE_LICENSE("GPL");
bool hwrng_removed;
};
-
static void random_recv_done(struct virtqueue *vq)
{
struct virtrng_info *vi = vq->vdev->priv;
#include "crypto4xx_sa.h"
#include "crypto4xx_core.h"
-u32 get_dynamic_sa_offset_iv_field(struct crypto4xx_ctx *ctx)
-{
- u32 offset;
- union dynamic_sa_contents cts;
-
- if (ctx->direction == DIR_INBOUND)
- cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_in))->sa_contents;
- else
- cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_out))->sa_contents;
- offset = cts.bf.key_size
- + cts.bf.inner_size
- + cts.bf.outer_size
- + cts.bf.spi
- + cts.bf.seq_num0
- + cts.bf.seq_num1
- + cts.bf.seq_num_mask0
- + cts.bf.seq_num_mask1
- + cts.bf.seq_num_mask2
- + cts.bf.seq_num_mask3;
-
- return sizeof(struct dynamic_sa_ctl) + offset * 4;
-}
-
u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx)
{
u32 offset;
dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,
DMA_TO_DEVICE);
err_map_in:
+err_alloc:
free_page((unsigned long)dd->buf_out);
free_page((unsigned long)dd->buf_in);
-err_alloc:
if (err)
pr_err("error: %d\n", err);
return err;
struct atmel_sha_dev *dd;
unsigned long flags;
-
- /* fallback stuff */
- struct crypto_shash *fallback;
-
};
#define ATMEL_SHA_QUEUE_LENGTH 50
return atmel_sha_init(req) ?: atmel_sha_finup(req);
}
-static int atmel_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
+static int atmel_sha_cra_init(struct crypto_tfm *tfm)
{
- struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm);
- const char *alg_name = crypto_tfm_alg_name(tfm);
-
- /* Allocate a fallback and abort if it failed. */
- tctx->fallback = crypto_alloc_shash(alg_name, 0,
- CRYPTO_ALG_NEED_FALLBACK);
- if (IS_ERR(tctx->fallback)) {
- pr_err("atmel-sha: fallback driver '%s' could not be loaded.\n",
- alg_name);
- return PTR_ERR(tctx->fallback);
- }
crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
sizeof(struct atmel_sha_reqctx) +
SHA_BUFFER_LEN + SHA512_BLOCK_SIZE);
return 0;
}
-static int atmel_sha_cra_init(struct crypto_tfm *tfm)
-{
- return atmel_sha_cra_init_alg(tfm, NULL);
-}
-
-static void atmel_sha_cra_exit(struct crypto_tfm *tfm)
-{
- struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm);
-
- crypto_free_shash(tctx->fallback);
- tctx->fallback = NULL;
-}
-
static struct ahash_alg sha_1_256_algs[] = {
{
.init = atmel_sha_init,
.cra_name = "sha1",
.cra_driver_name = "atmel-sha1",
.cra_priority = 100,
- .cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_NEED_FALLBACK,
+ .cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct atmel_sha_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
.cra_init = atmel_sha_cra_init,
- .cra_exit = atmel_sha_cra_exit,
}
}
},
.cra_name = "sha256",
.cra_driver_name = "atmel-sha256",
.cra_priority = 100,
- .cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_NEED_FALLBACK,
+ .cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct atmel_sha_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
.cra_init = atmel_sha_cra_init,
- .cra_exit = atmel_sha_cra_exit,
}
}
},
.cra_name = "sha224",
.cra_driver_name = "atmel-sha224",
.cra_priority = 100,
- .cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_NEED_FALLBACK,
+ .cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA224_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct atmel_sha_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
.cra_init = atmel_sha_cra_init,
- .cra_exit = atmel_sha_cra_exit,
}
}
};
.cra_name = "sha384",
.cra_driver_name = "atmel-sha384",
.cra_priority = 100,
- .cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_NEED_FALLBACK,
+ .cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA384_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct atmel_sha_ctx),
.cra_alignmask = 0x3,
.cra_module = THIS_MODULE,
.cra_init = atmel_sha_cra_init,
- .cra_exit = atmel_sha_cra_exit,
}
}
},
.cra_name = "sha512",
.cra_driver_name = "atmel-sha512",
.cra_priority = 100,
- .cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_NEED_FALLBACK,
+ .cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA512_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct atmel_sha_ctx),
.cra_alignmask = 0x3,
.cra_module = THIS_MODULE,
.cra_init = atmel_sha_cra_init,
- .cra_exit = atmel_sha_cra_exit,
}
}
},
dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,
DMA_TO_DEVICE);
err_map_in:
+err_alloc:
free_page((unsigned long)dd->buf_out);
free_page((unsigned long)dd->buf_in);
-err_alloc:
if (err)
pr_err("error: %d\n", err);
return err;
while (!sg_is_last(sg)) {
sg_nents++;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
return sg_nents;
ret = platform_driver_register(&bfin_crypto_crc_driver);
if (ret) {
- pr_info(KERN_ERR "unable to register driver\n");
+ pr_err("unable to register driver\n");
return ret;
}
in_options = 0;
} else {
src_dma = edesc->sec4_sg_dma;
- sec4_sg_index += (iv_contig ? 0 : 1) + edesc->src_nents;
+ sec4_sg_index += edesc->src_nents + 1;
in_options = LDST_SGF;
}
append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
if (!all_contig) {
if (!is_gcm) {
sg_to_sec4_sg(req->assoc,
- (assoc_nents ? : 1),
+ assoc_nents,
edesc->sec4_sg +
sec4_sg_index, 0);
- sec4_sg_index += assoc_nents ? : 1;
+ sec4_sg_index += assoc_nents;
}
dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
if (is_gcm) {
sg_to_sec4_sg(req->assoc,
- (assoc_nents ? : 1),
+ assoc_nents,
edesc->sec4_sg +
sec4_sg_index, 0);
- sec4_sg_index += assoc_nents ? : 1;
+ sec4_sg_index += assoc_nents;
}
sg_to_sec4_sg_last(req->src,
- (src_nents ? : 1),
+ src_nents,
edesc->sec4_sg +
sec4_sg_index, 0);
- sec4_sg_index += src_nents ? : 1;
+ sec4_sg_index += src_nents;
}
if (dst_nents) {
sg_to_sec4_sg_last(req->dst, dst_nents,
{
struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
struct caam_ctrl __iomem *ctrl;
- struct rng4tst __iomem *r4tst;
u32 *desc, status, rdsta_val;
int ret = 0, sh_idx;
ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl;
- r4tst = &ctrl->r4tst[0];
-
desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL);
if (!desc)
return -ENOMEM;
* without any error (HW optimizations for later
* CAAM eras), then try again.
*/
- rdsta_val =
- rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK;
+ rdsta_val = rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK;
if (status || !(rdsta_val & (1 << sh_idx)))
ret = -EAGAIN;
if (ret)
else
snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id);
- dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n",
- status, error, idx_str, idx,
- cha_str, cha_err_code,
- err_str, err_err_code);
+ /*
+ * CCB ICV check failures are part of normal operation life;
+ * we leave the upper layers to do what they want with them.
+ */
+ if (err_id != JRSTA_CCBERR_ERRID_ICVCHK)
+ dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n",
+ status, error, idx_str, idx,
+ cha_str, cha_err_code,
+ err_str, err_err_code);
}
static void report_jump_status(struct device *jrdev, const u32 status,
if (error) {
dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
jrp->ridx, jrp->irq);
- irq_dispose_mapping(jrp->irq);
- jrp->irq = 0;
- return -EINVAL;
+ goto out_kill_deq;
}
error = caam_reset_hw_jr(dev);
if (error)
- return error;
+ goto out_free_irq;
+ error = -ENOMEM;
jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
&inpbusaddr, GFP_KERNEL);
+ if (!jrp->inpring)
+ goto out_free_irq;
jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) *
JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
+ if (!jrp->outring)
+ goto out_free_inpring;
jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH,
GFP_KERNEL);
-
- if ((jrp->inpring == NULL) || (jrp->outring == NULL) ||
- (jrp->entinfo == NULL)) {
- dev_err(dev, "can't allocate job rings for %d\n",
- jrp->ridx);
- return -ENOMEM;
- }
+ if (!jrp->entinfo)
+ goto out_free_outring;
for (i = 0; i < JOBR_DEPTH; i++)
jrp->entinfo[i].desc_addr_dma = !0;
(JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
return 0;
+
+out_free_outring:
+ dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
+ jrp->outring, outbusaddr);
+out_free_inpring:
+ dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
+ jrp->inpring, inpbusaddr);
+ dev_err(dev, "can't allocate job rings for %d\n", jrp->ridx);
+out_free_irq:
+ free_irq(jrp->irq, dev);
+out_kill_deq:
+ tasklet_kill(&jrp->irqtask);
+ return error;
}
/* Now do the platform independent part */
error = caam_jr_init(jrdev); /* now turn on hardware */
- if (error)
+ if (error) {
+ irq_dispose_mapping(jrpriv->irq);
return error;
+ }
jrpriv->dev = jrdev;
spin_lock(&driver_data.jr_alloc_lock);
dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg),
sg_dma_len(sg), offset);
sec4_sg_ptr++;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
sg_count--;
}
return sec4_sg_ptr - 1;
nbytes -= sg->length;
if (!sg_is_last(sg) && (sg + 1)->length == 0)
*chained = true;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
return sg_nents;
int i;
for (i = 0; i < nents; i++) {
dma_map_sg(dev, sg, 1, dir);
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
} else {
dma_map_sg(dev, sg, nents, dir);
int i;
for (i = 0; i < nents; i++) {
dma_unmap_sg(dev, sg, 1, dir);
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
} else {
dma_unmap_sg(dev, sg, nents, dir);
#ifdef CONFIG_X86
static const struct x86_cpu_id ccp_support[] = {
{ X86_VENDOR_AMD, 22, },
+ { },
};
#endif
struct buffer_desc *buf, gfp_t flags,
enum dma_data_direction dir)
{
- for (;nbytes > 0; sg = scatterwalk_sg_next(sg)) {
+ for (; nbytes > 0; sg = sg_next(sg)) {
unsigned len = min(nbytes, sg->length);
struct buffer_desc *next_buf;
u32 next_buf_phys;
break;
offset += sg->length;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
return (start + nbytes > offset + sg->length);
}
break;
offset += sg_src->length;
- sg_src = scatterwalk_sg_next(sg_src);
+ sg_src = sg_next(sg_src);
}
/* start - offset is the number of bytes to advance in the scatterlist
while (len && (nx_sg - nx_dst) < sglen) {
n = scatterwalk_clamp(&walk, len);
if (!n) {
- /* In cases where we have scatterlist chain scatterwalk_sg_next
+ /* In cases where we have scatterlist chain sg_next
* handles with it properly */
- scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg));
+ scatterwalk_start(&walk, sg_next(walk.sg));
n = scatterwalk_clamp(&walk, len);
}
dst = scatterwalk_map(&walk);
scatterwalk_advance(&dd->in_walk, 4);
if (dd->in_sg->length == _calc_walked(in)) {
- dd->in_sg = scatterwalk_sg_next(dd->in_sg);
+ dd->in_sg = sg_next(dd->in_sg);
if (dd->in_sg) {
scatterwalk_start(&dd->in_walk,
dd->in_sg);
*dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i));
scatterwalk_advance(&dd->out_walk, 4);
if (dd->out_sg->length == _calc_walked(out)) {
- dd->out_sg = scatterwalk_sg_next(dd->out_sg);
+ dd->out_sg = sg_next(dd->out_sg);
if (dd->out_sg) {
scatterwalk_start(&dd->out_walk,
dd->out_sg);
scatterwalk_advance(&dd->in_walk, 4);
if (dd->in_sg->length == _calc_walked(in)) {
- dd->in_sg = scatterwalk_sg_next(dd->in_sg);
+ dd->in_sg = sg_next(dd->in_sg);
if (dd->in_sg) {
scatterwalk_start(&dd->in_walk,
dd->in_sg);
*dst = omap_des_read(dd, DES_REG_DATA_N(dd, i));
scatterwalk_advance(&dd->out_walk, 4);
if (dd->out_sg->length == _calc_walked(out)) {
- dd->out_sg = scatterwalk_sg_next(dd->out_sg);
+ dd->out_sg = sg_next(dd->out_sg);
if (dd->out_sg) {
scatterwalk_start(&dd->out_walk,
dd->out_sg);
}
}
- dd->total -= DES_BLOCK_SIZE;
+ BUG_ON(dd->total < DES_BLOCK_SIZE);
- BUG_ON(dd->total < 0);
+ dd->total -= DES_BLOCK_SIZE;
/* Clear IRQ status */
status &= ~DES_REG_IRQ_DATA_OUT;
#ifndef ADF_ACCEL_DEVICES_H_
#define ADF_ACCEL_DEVICES_H_
#include <linux/module.h>
-#include <linux/atomic.h>
#include <linux/list.h>
#include <linux/proc_fs.h>
#include <linux/io.h>
int (*alloc_irq)(struct adf_accel_dev *accel_dev);
void (*free_irq)(struct adf_accel_dev *accel_dev);
void (*enable_error_correction)(struct adf_accel_dev *accel_dev);
+ int (*init_admin_comms)(struct adf_accel_dev *accel_dev);
+ void (*exit_admin_comms)(struct adf_accel_dev *accel_dev);
+ int (*init_arb)(struct adf_accel_dev *accel_dev);
+ void (*exit_arb)(struct adf_accel_dev *accel_dev);
+ void (*enable_ints)(struct adf_accel_dev *accel_dev);
const char *fw_name;
uint32_t pci_dev_id;
uint32_t fuses;
struct work_struct reset_work;
};
-#define PPDSTAT_OFFSET 0x7E
static void adf_dev_restore(struct adf_accel_dev *accel_dev)
{
struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
struct pci_dev *parent = pdev->bus->self;
- uint16_t ppdstat = 0, bridge_ctl = 0;
- int pending = 0;
+ uint16_t bridge_ctl = 0;
pr_info("QAT: Resetting device qat_dev%d\n", accel_dev->accel_id);
- pci_read_config_word(pdev, PPDSTAT_OFFSET, &ppdstat);
- pending = ppdstat & PCI_EXP_DEVSTA_TRPND;
- if (pending) {
- int ctr = 0;
-
- do {
- msleep(100);
- pci_read_config_word(pdev, PPDSTAT_OFFSET, &ppdstat);
- pending = ppdstat & PCI_EXP_DEVSTA_TRPND;
- } while (pending && ctr++ < 10);
- }
- if (pending)
+ if (!pci_wait_for_pending_transaction(pdev))
pr_info("QAT: Transaction still in progress. Proceeding\n");
pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl);
adf_dev_restarting_notify(accel_dev);
adf_dev_stop(accel_dev);
+ adf_dev_shutdown(accel_dev);
adf_dev_restore(accel_dev);
- if (adf_dev_start(accel_dev)) {
+ if (adf_dev_init(accel_dev) || adf_dev_start(accel_dev)) {
/* The device hanged and we can't restart it so stop here */
dev_err(&GET_DEV(accel_dev), "Restart device failed\n");
kfree(reset_data);
{
struct adf_reset_dev_data *reset_data;
- if (adf_dev_started(accel_dev) &&
- !test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
+ if (!adf_dev_started(accel_dev) ||
+ test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
return 0;
set_bit(ADF_STATUS_RESTARTING, &accel_dev->status);
#include <linux/seq_file.h>
#include "adf_accel_devices.h"
#include "adf_cfg.h"
+#include "adf_common_drv.h"
static DEFINE_MUTEX(qat_cfg_read_lock);
down_write(&dev_cfg_data->lock);
adf_cfg_section_del_all(&dev_cfg_data->sec_list);
up_write(&dev_cfg_data->lock);
+ clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
}
/**
int adf_dev_init(struct adf_accel_dev *accel_dev);
int adf_dev_start(struct adf_accel_dev *accel_dev);
int adf_dev_stop(struct adf_accel_dev *accel_dev);
-int adf_dev_shutdown(struct adf_accel_dev *accel_dev);
+void adf_dev_shutdown(struct adf_accel_dev *accel_dev);
int adf_ctl_dev_register(void);
void adf_ctl_dev_unregister(void);
if (adf_dev_stop(accel_dev)) {
pr_err("QAT: Failed to stop qat_dev%d\n", id);
ret = -EFAULT;
+ } else {
+ adf_dev_shutdown(accel_dev);
}
}
}
if (!adf_dev_started(accel_dev)) {
pr_info("QAT: Starting acceleration device qat_dev%d.\n",
ctl_data->device_id);
- ret = adf_dev_start(accel_dev);
+ ret = adf_dev_init(accel_dev);
+ if (!ret)
+ ret = adf_dev_start(accel_dev);
} else {
pr_info("QAT: Acceleration device qat_dev%d already started.\n",
ctl_data->device_id);
if (ret) {
pr_err("QAT: Failed to start qat_dev%d\n", ctl_data->device_id);
adf_dev_stop(accel_dev);
+ adf_dev_shutdown(accel_dev);
}
out:
kfree(ctl_data);
EXPORT_SYMBOL_GPL(adf_service_unregister);
/**
- * adf_dev_start() - Start acceleration service for the given accel device
- * @accel_dev: Pointer to acceleration device.
+ * adf_dev_init() - Init data structures and services for the given accel device
+ * @accel_dev: Pointer to acceleration device.
*
- * Function notifies all the registered services that the acceleration device
- * is ready to be used.
- * To be used by QAT device specific drivers.
+ * Initialize the ring data structures and the admin comms and arbitration
+ * services.
*
* Return: 0 on success, error code othewise.
*/
-int adf_dev_start(struct adf_accel_dev *accel_dev)
+int adf_dev_init(struct adf_accel_dev *accel_dev)
{
struct service_hndl *service;
struct list_head *list_itr;
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ if (!hw_data) {
+ dev_err(&GET_DEV(accel_dev),
+ "QAT: Failed to init device - hw_data not set\n");
+ return -EFAULT;
+ }
+
if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) {
pr_info("QAT: Device not configured\n");
return -EFAULT;
}
- set_bit(ADF_STATUS_STARTING, &accel_dev->status);
+
+ if (adf_init_etr_data(accel_dev)) {
+ dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n");
+ return -EFAULT;
+ }
+
+ if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) {
+ dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n");
+ return -EFAULT;
+ }
+
+ if (hw_data->init_arb && hw_data->init_arb(accel_dev)) {
+ dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n");
+ return -EFAULT;
+ }
+
+ hw_data->enable_ints(accel_dev);
if (adf_ae_init(accel_dev)) {
pr_err("QAT: Failed to initialise Acceleration Engine\n");
hw_data->enable_error_correction(accel_dev);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(adf_dev_init);
+
+/**
+ * adf_dev_start() - Start acceleration service for the given accel device
+ * @accel_dev: Pointer to acceleration device.
+ *
+ * Function notifies all the registered services that the acceleration device
+ * is ready to be used.
+ * To be used by QAT device specific drivers.
+ *
+ * Return: 0 on success, error code othewise.
+ */
+int adf_dev_start(struct adf_accel_dev *accel_dev)
+{
+ struct service_hndl *service;
+ struct list_head *list_itr;
+
+ set_bit(ADF_STATUS_STARTING, &accel_dev->status);
+
if (adf_ae_start(accel_dev)) {
pr_err("QAT: AE Start Failed\n");
return -EFAULT;
*/
int adf_dev_stop(struct adf_accel_dev *accel_dev)
{
- struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct service_hndl *service;
struct list_head *list_itr;
- int ret, wait = 0;
+ bool wait = false;
+ int ret;
if (!adf_dev_started(accel_dev) &&
!test_bit(ADF_STATUS_STARTING, &accel_dev->status)) {
return 0;
}
- clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
if (!ret) {
clear_bit(accel_dev->accel_id, &service->start_status);
} else if (ret == -EAGAIN) {
- wait = 1;
+ wait = true;
clear_bit(accel_dev->accel_id, &service->start_status);
}
}
if (wait)
msleep(100);
- if (adf_dev_started(accel_dev)) {
+ if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) {
if (adf_ae_stop(accel_dev))
pr_err("QAT: failed to stop AE\n");
else
clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
}
+ return 0;
+}
+EXPORT_SYMBOL_GPL(adf_dev_stop);
+
+/**
+ * adf_dev_shutdown() - shutdown acceleration services and data strucutures
+ * @accel_dev: Pointer to acceleration device
+ *
+ * Cleanup the ring data structures and the admin comms and arbitration
+ * services.
+ */
+void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
+{
+ struct adf_hw_device_data *hw_data = accel_dev->hw_device;
+ struct service_hndl *service;
+ struct list_head *list_itr;
+
+ if (!hw_data) {
+ dev_err(&GET_DEV(accel_dev),
+ "QAT: Failed to shutdown device - hw_data not set\n");
+ return;
+ }
+
if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) {
if (adf_ae_fw_release(accel_dev))
pr_err("QAT: Failed to release the ucode\n");
if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
adf_cfg_del_all(accel_dev);
- return 0;
+ if (hw_data->exit_arb)
+ hw_data->exit_arb(accel_dev);
+
+ if (hw_data->exit_admin_comms)
+ hw_data->exit_admin_comms(accel_dev);
+
+ adf_cleanup_etr_data(accel_dev);
}
-EXPORT_SYMBOL_GPL(adf_dev_stop);
+EXPORT_SYMBOL_GPL(adf_dev_shutdown);
int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev)
{
#define ADF_TRANSPORT_INTRN_H
#include <linux/interrupt.h>
-#include <linux/atomic.h>
#include <linux/spinlock_types.h>
#include "adf_transport.h"
struct icp_qat_hw_cipher_algo_blk {
struct icp_qat_hw_cipher_aes256_f8 aes;
-};
+} __aligned(64);
#endif
#include "icp_qat_fw.h"
#include "icp_qat_fw_la.h"
-#define QAT_AES_HW_CONFIG_ENC(alg) \
+#define QAT_AES_HW_CONFIG_CBC_ENC(alg) \
ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
- ICP_QAT_HW_CIPHER_NO_CONVERT, \
- ICP_QAT_HW_CIPHER_ENCRYPT)
+ ICP_QAT_HW_CIPHER_NO_CONVERT, \
+ ICP_QAT_HW_CIPHER_ENCRYPT)
-#define QAT_AES_HW_CONFIG_DEC(alg) \
+#define QAT_AES_HW_CONFIG_CBC_DEC(alg) \
ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
- ICP_QAT_HW_CIPHER_KEY_CONVERT, \
- ICP_QAT_HW_CIPHER_DECRYPT)
+ ICP_QAT_HW_CIPHER_KEY_CONVERT, \
+ ICP_QAT_HW_CIPHER_DECRYPT)
static atomic_t active_dev;
};
} __aligned(64);
-#define MAX_AUTH_STATE_SIZE sizeof(struct icp_qat_hw_auth_algo_blk)
-
-struct qat_auth_state {
- uint8_t data[MAX_AUTH_STATE_SIZE + 64];
-} __aligned(64);
-
-struct qat_alg_session_ctx {
+struct qat_alg_aead_ctx {
struct qat_alg_cd *enc_cd;
- dma_addr_t enc_cd_paddr;
struct qat_alg_cd *dec_cd;
+ dma_addr_t enc_cd_paddr;
dma_addr_t dec_cd_paddr;
- struct icp_qat_fw_la_bulk_req enc_fw_req_tmpl;
- struct icp_qat_fw_la_bulk_req dec_fw_req_tmpl;
- struct qat_crypto_instance *inst;
- struct crypto_tfm *tfm;
+ struct icp_qat_fw_la_bulk_req enc_fw_req;
+ struct icp_qat_fw_la_bulk_req dec_fw_req;
struct crypto_shash *hash_tfm;
enum icp_qat_hw_auth_algo qat_hash_alg;
+ struct qat_crypto_instance *inst;
+ struct crypto_tfm *tfm;
uint8_t salt[AES_BLOCK_SIZE];
- spinlock_t lock; /* protects qat_alg_session_ctx struct */
+ spinlock_t lock; /* protects qat_alg_aead_ctx struct */
+};
+
+struct qat_alg_ablkcipher_ctx {
+ struct icp_qat_hw_cipher_algo_blk *enc_cd;
+ struct icp_qat_hw_cipher_algo_blk *dec_cd;
+ dma_addr_t enc_cd_paddr;
+ dma_addr_t dec_cd_paddr;
+ struct icp_qat_fw_la_bulk_req enc_fw_req;
+ struct icp_qat_fw_la_bulk_req dec_fw_req;
+ struct qat_crypto_instance *inst;
+ struct crypto_tfm *tfm;
+ spinlock_t lock; /* protects qat_alg_ablkcipher_ctx struct */
};
static int get_current_node(void)
}
static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
- struct qat_alg_session_ctx *ctx,
+ struct qat_alg_aead_ctx *ctx,
const uint8_t *auth_key,
unsigned int auth_keylen)
{
- struct qat_auth_state auth_state;
SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
struct sha1_state sha1;
struct sha256_state sha256;
struct sha512_state sha512;
int block_size = crypto_shash_blocksize(ctx->hash_tfm);
int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
- uint8_t *ipad = auth_state.data;
- uint8_t *opad = ipad + block_size;
+ char ipad[block_size];
+ char opad[block_size];
__be32 *hash_state_out;
__be64 *hash512_state_out;
int i, offset;
- memzero_explicit(auth_state.data, MAX_AUTH_STATE_SIZE + 64);
+ memset(ipad, 0, block_size);
+ memset(opad, 0, block_size);
shash->tfm = ctx->hash_tfm;
shash->flags = 0x0;
if (auth_keylen > block_size) {
- char buff[SHA512_BLOCK_SIZE];
int ret = crypto_shash_digest(shash, auth_key,
- auth_keylen, buff);
+ auth_keylen, ipad);
if (ret)
return ret;
- memcpy(ipad, buff, digest_size);
- memcpy(opad, buff, digest_size);
- memzero_explicit(ipad + digest_size, block_size - digest_size);
- memzero_explicit(opad + digest_size, block_size - digest_size);
+ memcpy(opad, ipad, digest_size);
} else {
memcpy(ipad, auth_key, auth_keylen);
memcpy(opad, auth_key, auth_keylen);
- memzero_explicit(ipad + auth_keylen, block_size - auth_keylen);
- memzero_explicit(opad + auth_keylen, block_size - auth_keylen);
}
for (i = 0; i < block_size; i++) {
header->comn_req_flags =
ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
QAT_COMN_PTR_TYPE_SGL);
- ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
- ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
ICP_QAT_FW_LA_PARTIAL_NONE);
ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
ICP_QAT_FW_LA_NO_UPDATE_STATE);
}
-static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx,
- int alg, struct crypto_authenc_keys *keys)
+static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx,
+ int alg,
+ struct crypto_authenc_keys *keys)
{
struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
struct icp_qat_hw_auth_algo_blk *hash =
(struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
- struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req_tmpl;
+ struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
void *ptr = &req_tmpl->cd_ctrl;
struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
/* CD setup */
- cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg);
+ cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);
memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
hash->sha.inner_setup.auth_config.config =
ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
/* Request setup */
qat_alg_init_common_hdr(header);
header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
+ ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
ICP_QAT_FW_LA_RET_AUTH_RES);
ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
return 0;
}
-static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx,
- int alg, struct crypto_authenc_keys *keys)
+static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx,
+ int alg,
+ struct crypto_authenc_keys *keys)
{
struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
(struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
sizeof(struct icp_qat_hw_auth_setup) +
roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
- struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req_tmpl;
+ struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
void *ptr = &req_tmpl->cd_ctrl;
sizeof(struct icp_qat_fw_la_cipher_req_params));
/* CD setup */
- cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg);
+ cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);
memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
hash->sha.inner_setup.auth_config.config =
ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
/* Request setup */
qat_alg_init_common_hdr(header);
header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
+ ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
ICP_QAT_FW_LA_NO_RET_AUTH_RES);
ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
return 0;
}
-static int qat_alg_init_sessions(struct qat_alg_session_ctx *ctx,
- const uint8_t *key, unsigned int keylen)
+static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx,
+ struct icp_qat_fw_la_bulk_req *req,
+ struct icp_qat_hw_cipher_algo_blk *cd,
+ const uint8_t *key, unsigned int keylen)
{
- struct crypto_authenc_keys keys;
- int alg;
+ struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
+ struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
+ struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
- if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE))
- return -EFAULT;
+ memcpy(cd->aes.key, key, keylen);
+ qat_alg_init_common_hdr(header);
+ header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
+ cd_pars->u.s.content_desc_params_sz =
+ sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
+ /* Cipher CD config setup */
+ cd_ctrl->cipher_key_sz = keylen >> 3;
+ cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
+ cd_ctrl->cipher_cfg_offset = 0;
+ ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
+ ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
+}
- if (crypto_authenc_extractkeys(&keys, key, keylen))
- goto bad_key;
+static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx,
+ int alg, const uint8_t *key,
+ unsigned int keylen)
+{
+ struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
+ struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
+ struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
+
+ qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen);
+ cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
+ enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);
+}
- switch (keys.enckeylen) {
+static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx,
+ int alg, const uint8_t *key,
+ unsigned int keylen)
+{
+ struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
+ struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
+ struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
+
+ qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen);
+ cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
+ dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);
+}
+
+static int qat_alg_validate_key(int key_len, int *alg)
+{
+ switch (key_len) {
case AES_KEYSIZE_128:
- alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
+ *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
break;
case AES_KEYSIZE_192:
- alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
+ *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
break;
case AES_KEYSIZE_256:
- alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
+ *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
break;
default:
- goto bad_key;
+ return -EINVAL;
}
+ return 0;
+}
- if (qat_alg_init_enc_session(ctx, alg, &keys))
+static int qat_alg_aead_init_sessions(struct qat_alg_aead_ctx *ctx,
+ const uint8_t *key, unsigned int keylen)
+{
+ struct crypto_authenc_keys keys;
+ int alg;
+
+ if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE))
+ return -EFAULT;
+
+ if (crypto_authenc_extractkeys(&keys, key, keylen))
+ goto bad_key;
+
+ if (qat_alg_validate_key(keys.enckeylen, &alg))
+ goto bad_key;
+
+ if (qat_alg_aead_init_enc_session(ctx, alg, &keys))
goto error;
- if (qat_alg_init_dec_session(ctx, alg, &keys))
+ if (qat_alg_aead_init_dec_session(ctx, alg, &keys))
goto error;
return 0;
return -EFAULT;
}
-static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key,
- unsigned int keylen)
+static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx,
+ const uint8_t *key,
+ unsigned int keylen)
+{
+ int alg;
+
+ if (qat_alg_validate_key(keylen, &alg))
+ goto bad_key;
+
+ qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen);
+ qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen);
+ return 0;
+bad_key:
+ crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+}
+
+static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key,
+ unsigned int keylen)
{
- struct qat_alg_session_ctx *ctx = crypto_aead_ctx(tfm);
+ struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
struct device *dev;
spin_lock(&ctx->lock);
if (ctx->enc_cd) {
/* rekeying */
dev = &GET_DEV(ctx->inst->accel_dev);
- memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd));
- memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd));
- memzero_explicit(&ctx->enc_fw_req_tmpl,
- sizeof(struct icp_qat_fw_la_bulk_req));
- memzero_explicit(&ctx->dec_fw_req_tmpl,
- sizeof(struct icp_qat_fw_la_bulk_req));
+ memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
+ memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
+ memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
+ memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
} else {
/* new key */
int node = get_current_node();
dev = &GET_DEV(inst->accel_dev);
ctx->inst = inst;
- ctx->enc_cd = dma_zalloc_coherent(dev,
- sizeof(struct qat_alg_cd),
+ ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
&ctx->enc_cd_paddr,
GFP_ATOMIC);
if (!ctx->enc_cd) {
spin_unlock(&ctx->lock);
return -ENOMEM;
}
- ctx->dec_cd = dma_zalloc_coherent(dev,
- sizeof(struct qat_alg_cd),
+ ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
&ctx->dec_cd_paddr,
GFP_ATOMIC);
if (!ctx->dec_cd) {
}
}
spin_unlock(&ctx->lock);
- if (qat_alg_init_sessions(ctx, key, keylen))
+ if (qat_alg_aead_init_sessions(ctx, key, keylen))
goto out_free_all;
return 0;
out_free_all:
- memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd));
+ memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
dma_free_coherent(dev, sizeof(struct qat_alg_cd),
ctx->dec_cd, ctx->dec_cd_paddr);
ctx->dec_cd = NULL;
out_free_enc:
- memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd));
+ memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
dma_free_coherent(dev, sizeof(struct qat_alg_cd),
ctx->enc_cd, ctx->enc_cd_paddr);
ctx->enc_cd = NULL;
dma_addr_t blp = qat_req->buf.blp;
dma_addr_t blpout = qat_req->buf.bloutp;
size_t sz = qat_req->buf.sz;
- int i, bufs = bl->num_bufs;
+ size_t sz_out = qat_req->buf.sz_out;
+ int i;
for (i = 0; i < bl->num_bufs; i++)
dma_unmap_single(dev, bl->bufers[i].addr,
kfree(bl);
if (blp != blpout) {
/* If out of place operation dma unmap only data */
- int bufless = bufs - blout->num_mapped_bufs;
+ int bufless = blout->num_bufs - blout->num_mapped_bufs;
- for (i = bufless; i < bufs; i++) {
+ for (i = bufless; i < blout->num_bufs; i++) {
dma_unmap_single(dev, blout->bufers[i].addr,
blout->bufers[i].len,
DMA_BIDIRECTIONAL);
}
- dma_unmap_single(dev, blpout, sz, DMA_TO_DEVICE);
+ dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE);
kfree(blout);
}
}
struct qat_crypto_request *qat_req)
{
struct device *dev = &GET_DEV(inst->accel_dev);
- int i, bufs = 0, n = sg_nents(sgl), assoc_n = sg_nents(assoc);
+ int i, bufs = 0, sg_nctr = 0;
+ int n = sg_nents(sgl), assoc_n = sg_nents(assoc);
struct qat_alg_buf_list *bufl;
struct qat_alg_buf_list *buflout = NULL;
dma_addr_t blp;
dma_addr_t bloutp = 0;
struct scatterlist *sg;
- size_t sz = sizeof(struct qat_alg_buf_list) +
+ size_t sz_out, sz = sizeof(struct qat_alg_buf_list) +
((1 + n + assoc_n) * sizeof(struct qat_alg_buf));
if (unlikely(!n))
return -EINVAL;
- bufl = kmalloc_node(sz, GFP_ATOMIC,
+ bufl = kzalloc_node(sz, GFP_ATOMIC,
dev_to_node(&GET_DEV(inst->accel_dev)));
if (unlikely(!bufl))
return -ENOMEM;
goto err;
bufs++;
}
- bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen,
- DMA_BIDIRECTIONAL);
- bufl->bufers[bufs].len = ivlen;
- if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
- goto err;
- bufs++;
+ if (ivlen) {
+ bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen,
+ DMA_BIDIRECTIONAL);
+ bufl->bufers[bufs].len = ivlen;
+ if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
+ goto err;
+ bufs++;
+ }
for_each_sg(sgl, sg, n, i) {
- int y = i + bufs;
+ int y = sg_nctr + bufs;
+
+ if (!sg->length)
+ continue;
bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg),
sg->length,
bufl->bufers[y].len = sg->length;
if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr)))
goto err;
+ sg_nctr++;
}
- bufl->num_bufs = n + bufs;
+ bufl->num_bufs = sg_nctr + bufs;
qat_req->buf.bl = bufl;
qat_req->buf.blp = blp;
qat_req->buf.sz = sz;
if (sgl != sglout) {
struct qat_alg_buf *bufers;
- buflout = kmalloc_node(sz, GFP_ATOMIC,
+ n = sg_nents(sglout);
+ sz_out = sizeof(struct qat_alg_buf_list) +
+ ((1 + n + assoc_n) * sizeof(struct qat_alg_buf));
+ sg_nctr = 0;
+ buflout = kzalloc_node(sz_out, GFP_ATOMIC,
dev_to_node(&GET_DEV(inst->accel_dev)));
if (unlikely(!buflout))
goto err;
- bloutp = dma_map_single(dev, buflout, sz, DMA_TO_DEVICE);
+ bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE);
if (unlikely(dma_mapping_error(dev, bloutp)))
goto err;
bufers = buflout->bufers;
bufers[i].addr = bufl->bufers[i].addr;
}
for_each_sg(sglout, sg, n, i) {
- int y = i + bufs;
+ int y = sg_nctr + bufs;
+
+ if (!sg->length)
+ continue;
bufers[y].addr = dma_map_single(dev, sg_virt(sg),
sg->length,
DMA_BIDIRECTIONAL);
- buflout->bufers[y].len = sg->length;
if (unlikely(dma_mapping_error(dev, bufers[y].addr)))
goto err;
+ bufers[y].len = sg->length;
+ sg_nctr++;
}
- buflout->num_bufs = n + bufs;
- buflout->num_mapped_bufs = n;
+ buflout->num_bufs = sg_nctr + bufs;
+ buflout->num_mapped_bufs = sg_nctr;
qat_req->buf.blout = buflout;
qat_req->buf.bloutp = bloutp;
+ qat_req->buf.sz_out = sz_out;
} else {
/* Otherwise set the src and dst to the same address */
qat_req->buf.bloutp = qat_req->buf.blp;
+ qat_req->buf.sz_out = 0;
}
return 0;
err:
dev_err(dev, "Failed to map buf for dma\n");
- for_each_sg(sgl, sg, n + bufs, i) {
- if (!dma_mapping_error(dev, bufl->bufers[i].addr)) {
+ sg_nctr = 0;
+ for (i = 0; i < n + bufs; i++)
+ if (!dma_mapping_error(dev, bufl->bufers[i].addr))
dma_unmap_single(dev, bufl->bufers[i].addr,
bufl->bufers[i].len,
DMA_BIDIRECTIONAL);
- }
- }
+
if (!dma_mapping_error(dev, blp))
dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
kfree(bufl);
if (sgl != sglout && buflout) {
- for_each_sg(sglout, sg, n, i) {
- int y = i + bufs;
-
- if (!dma_mapping_error(dev, buflout->bufers[y].addr))
- dma_unmap_single(dev, buflout->bufers[y].addr,
- buflout->bufers[y].len,
+ n = sg_nents(sglout);
+ for (i = bufs; i < n + bufs; i++)
+ if (!dma_mapping_error(dev, buflout->bufers[i].addr))
+ dma_unmap_single(dev, buflout->bufers[i].addr,
+ buflout->bufers[i].len,
DMA_BIDIRECTIONAL);
- }
if (!dma_mapping_error(dev, bloutp))
- dma_unmap_single(dev, bloutp, sz, DMA_TO_DEVICE);
+ dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE);
kfree(buflout);
}
return -ENOMEM;
}
-void qat_alg_callback(void *resp)
+static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
+ struct qat_crypto_request *qat_req)
{
- struct icp_qat_fw_la_resp *qat_resp = resp;
- struct qat_crypto_request *qat_req =
- (void *)(__force long)qat_resp->opaque_data;
- struct qat_alg_session_ctx *ctx = qat_req->ctx;
+ struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
struct qat_crypto_instance *inst = ctx->inst;
- struct aead_request *areq = qat_req->areq;
+ struct aead_request *areq = qat_req->aead_req;
uint8_t stat_filed = qat_resp->comn_resp.comn_status;
int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
areq->base.complete(&areq->base, res);
}
-static int qat_alg_dec(struct aead_request *areq)
+static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
+ struct qat_crypto_request *qat_req)
+{
+ struct qat_alg_ablkcipher_ctx *ctx = qat_req->ablkcipher_ctx;
+ struct qat_crypto_instance *inst = ctx->inst;
+ struct ablkcipher_request *areq = qat_req->ablkcipher_req;
+ uint8_t stat_filed = qat_resp->comn_resp.comn_status;
+ int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
+
+ qat_alg_free_bufl(inst, qat_req);
+ if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
+ res = -EINVAL;
+ areq->base.complete(&areq->base, res);
+}
+
+void qat_alg_callback(void *resp)
+{
+ struct icp_qat_fw_la_resp *qat_resp = resp;
+ struct qat_crypto_request *qat_req =
+ (void *)(__force long)qat_resp->opaque_data;
+
+ qat_req->cb(qat_resp, qat_req);
+}
+
+static int qat_alg_aead_dec(struct aead_request *areq)
{
struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
- struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
struct qat_crypto_request *qat_req = aead_request_ctx(areq);
struct icp_qat_fw_la_cipher_req_params *cipher_param;
struct icp_qat_fw_la_auth_req_params *auth_param;
return ret;
msg = &qat_req->req;
- *msg = ctx->dec_fw_req_tmpl;
- qat_req->ctx = ctx;
- qat_req->areq = areq;
+ *msg = ctx->dec_fw_req;
+ qat_req->aead_ctx = ctx;
+ qat_req->aead_req = areq;
+ qat_req->cb = qat_aead_alg_callback;
qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
return -EINPROGRESS;
}
-static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv,
- int enc_iv)
+static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv,
+ int enc_iv)
{
struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
- struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
struct qat_crypto_request *qat_req = aead_request_ctx(areq);
struct icp_qat_fw_la_cipher_req_params *cipher_param;
struct icp_qat_fw_la_auth_req_params *auth_param;
return ret;
msg = &qat_req->req;
- *msg = ctx->enc_fw_req_tmpl;
- qat_req->ctx = ctx;
- qat_req->areq = areq;
+ *msg = ctx->enc_fw_req;
+ qat_req->aead_ctx = ctx;
+ qat_req->aead_req = areq;
+ qat_req->cb = qat_aead_alg_callback;
qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
return -EINPROGRESS;
}
-static int qat_alg_enc(struct aead_request *areq)
+static int qat_alg_aead_enc(struct aead_request *areq)
{
- return qat_alg_enc_internal(areq, areq->iv, 0);
+ return qat_alg_aead_enc_internal(areq, areq->iv, 0);
}
-static int qat_alg_genivenc(struct aead_givcrypt_request *req)
+static int qat_alg_aead_genivenc(struct aead_givcrypt_request *req)
{
struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq);
struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
- struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
__be64 seq;
memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE);
seq = cpu_to_be64(req->seq);
memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t),
&seq, sizeof(uint64_t));
- return qat_alg_enc_internal(&req->areq, req->giv, 1);
+ return qat_alg_aead_enc_internal(&req->areq, req->giv, 1);
}
-static int qat_alg_init(struct crypto_tfm *tfm,
- enum icp_qat_hw_auth_algo hash, const char *hash_name)
+static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
+ const uint8_t *key,
+ unsigned int keylen)
{
- struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ struct device *dev;
+
+ spin_lock(&ctx->lock);
+ if (ctx->enc_cd) {
+ /* rekeying */
+ dev = &GET_DEV(ctx->inst->accel_dev);
+ memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
+ memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
+ memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
+ memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
+ } else {
+ /* new key */
+ int node = get_current_node();
+ struct qat_crypto_instance *inst =
+ qat_crypto_get_instance_node(node);
+ if (!inst) {
+ spin_unlock(&ctx->lock);
+ return -EINVAL;
+ }
+
+ dev = &GET_DEV(inst->accel_dev);
+ ctx->inst = inst;
+ ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
+ &ctx->enc_cd_paddr,
+ GFP_ATOMIC);
+ if (!ctx->enc_cd) {
+ spin_unlock(&ctx->lock);
+ return -ENOMEM;
+ }
+ ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
+ &ctx->dec_cd_paddr,
+ GFP_ATOMIC);
+ if (!ctx->dec_cd) {
+ spin_unlock(&ctx->lock);
+ goto out_free_enc;
+ }
+ }
+ spin_unlock(&ctx->lock);
+ if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen))
+ goto out_free_all;
+
+ return 0;
+
+out_free_all:
+ memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd));
+ dma_free_coherent(dev, sizeof(*ctx->enc_cd),
+ ctx->dec_cd, ctx->dec_cd_paddr);
+ ctx->dec_cd = NULL;
+out_free_enc:
+ memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd));
+ dma_free_coherent(dev, sizeof(*ctx->dec_cd),
+ ctx->enc_cd, ctx->enc_cd_paddr);
+ ctx->enc_cd = NULL;
+ return -ENOMEM;
+}
+
+static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)
+{
+ struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
+ struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
+ struct icp_qat_fw_la_cipher_req_params *cipher_param;
+ struct icp_qat_fw_la_bulk_req *msg;
+ int ret, ctr = 0;
+
+ ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst,
+ NULL, 0, qat_req);
+ if (unlikely(ret))
+ return ret;
+
+ msg = &qat_req->req;
+ *msg = ctx->enc_fw_req;
+ qat_req->ablkcipher_ctx = ctx;
+ qat_req->ablkcipher_req = req;
+ qat_req->cb = qat_ablkcipher_alg_callback;
+ qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
+ qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
+ qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
+ cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
+ cipher_param->cipher_length = req->nbytes;
+ cipher_param->cipher_offset = 0;
+ memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
+ do {
+ ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
+ } while (ret == -EAGAIN && ctr++ < 10);
+
+ if (ret == -EAGAIN) {
+ qat_alg_free_bufl(ctx->inst, qat_req);
+ return -EBUSY;
+ }
+ return -EINPROGRESS;
+}
+
+static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
+{
+ struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
+ struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
+ struct icp_qat_fw_la_cipher_req_params *cipher_param;
+ struct icp_qat_fw_la_bulk_req *msg;
+ int ret, ctr = 0;
+
+ ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst,
+ NULL, 0, qat_req);
+ if (unlikely(ret))
+ return ret;
+
+ msg = &qat_req->req;
+ *msg = ctx->dec_fw_req;
+ qat_req->ablkcipher_ctx = ctx;
+ qat_req->ablkcipher_req = req;
+ qat_req->cb = qat_ablkcipher_alg_callback;
+ qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
+ qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
+ qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
+ cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
+ cipher_param->cipher_length = req->nbytes;
+ cipher_param->cipher_offset = 0;
+ memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
+ do {
+ ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
+ } while (ret == -EAGAIN && ctr++ < 10);
+
+ if (ret == -EAGAIN) {
+ qat_alg_free_bufl(ctx->inst, qat_req);
+ return -EBUSY;
+ }
+ return -EINPROGRESS;
+}
+
+static int qat_alg_aead_init(struct crypto_tfm *tfm,
+ enum icp_qat_hw_auth_algo hash,
+ const char *hash_name)
+{
+ struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
- memzero_explicit(ctx, sizeof(*ctx));
ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
if (IS_ERR(ctx->hash_tfm))
return -EFAULT;
return 0;
}
-static int qat_alg_sha1_init(struct crypto_tfm *tfm)
+static int qat_alg_aead_sha1_init(struct crypto_tfm *tfm)
{
- return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
+ return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
}
-static int qat_alg_sha256_init(struct crypto_tfm *tfm)
+static int qat_alg_aead_sha256_init(struct crypto_tfm *tfm)
{
- return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
+ return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
}
-static int qat_alg_sha512_init(struct crypto_tfm *tfm)
+static int qat_alg_aead_sha512_init(struct crypto_tfm *tfm)
{
- return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
+ return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
}
-static void qat_alg_exit(struct crypto_tfm *tfm)
+static void qat_alg_aead_exit(struct crypto_tfm *tfm)
{
- struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
struct qat_crypto_instance *inst = ctx->inst;
struct device *dev;
dev = &GET_DEV(inst->accel_dev);
if (ctx->enc_cd) {
- memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd));
+ memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
dma_free_coherent(dev, sizeof(struct qat_alg_cd),
ctx->enc_cd, ctx->enc_cd_paddr);
}
if (ctx->dec_cd) {
- memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd));
+ memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
dma_free_coherent(dev, sizeof(struct qat_alg_cd),
ctx->dec_cd, ctx->dec_cd_paddr);
}
qat_crypto_put_instance(inst);
}
+static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm)
+{
+ struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ spin_lock_init(&ctx->lock);
+ tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
+ sizeof(struct qat_crypto_request);
+ ctx->tfm = tfm;
+ return 0;
+}
+
+static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm)
+{
+ struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct qat_crypto_instance *inst = ctx->inst;
+ struct device *dev;
+
+ if (!inst)
+ return;
+
+ dev = &GET_DEV(inst->accel_dev);
+ if (ctx->enc_cd) {
+ memset(ctx->enc_cd, 0,
+ sizeof(struct icp_qat_hw_cipher_algo_blk));
+ dma_free_coherent(dev,
+ sizeof(struct icp_qat_hw_cipher_algo_blk),
+ ctx->enc_cd, ctx->enc_cd_paddr);
+ }
+ if (ctx->dec_cd) {
+ memset(ctx->dec_cd, 0,
+ sizeof(struct icp_qat_hw_cipher_algo_blk));
+ dma_free_coherent(dev,
+ sizeof(struct icp_qat_hw_cipher_algo_blk),
+ ctx->dec_cd, ctx->dec_cd_paddr);
+ }
+ qat_crypto_put_instance(inst);
+}
+
static struct crypto_alg qat_algs[] = { {
.cra_name = "authenc(hmac(sha1),cbc(aes))",
.cra_driver_name = "qat_aes_cbc_hmac_sha1",
.cra_priority = 4001,
.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct qat_alg_session_ctx),
+ .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_aead_type,
.cra_module = THIS_MODULE,
- .cra_init = qat_alg_sha1_init,
- .cra_exit = qat_alg_exit,
+ .cra_init = qat_alg_aead_sha1_init,
+ .cra_exit = qat_alg_aead_exit,
.cra_u = {
.aead = {
- .setkey = qat_alg_setkey,
- .decrypt = qat_alg_dec,
- .encrypt = qat_alg_enc,
- .givencrypt = qat_alg_genivenc,
+ .setkey = qat_alg_aead_setkey,
+ .decrypt = qat_alg_aead_dec,
+ .encrypt = qat_alg_aead_enc,
+ .givencrypt = qat_alg_aead_genivenc,
.ivsize = AES_BLOCK_SIZE,
.maxauthsize = SHA1_DIGEST_SIZE,
},
.cra_priority = 4001,
.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct qat_alg_session_ctx),
+ .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_aead_type,
.cra_module = THIS_MODULE,
- .cra_init = qat_alg_sha256_init,
- .cra_exit = qat_alg_exit,
+ .cra_init = qat_alg_aead_sha256_init,
+ .cra_exit = qat_alg_aead_exit,
.cra_u = {
.aead = {
- .setkey = qat_alg_setkey,
- .decrypt = qat_alg_dec,
- .encrypt = qat_alg_enc,
- .givencrypt = qat_alg_genivenc,
+ .setkey = qat_alg_aead_setkey,
+ .decrypt = qat_alg_aead_dec,
+ .encrypt = qat_alg_aead_enc,
+ .givencrypt = qat_alg_aead_genivenc,
.ivsize = AES_BLOCK_SIZE,
.maxauthsize = SHA256_DIGEST_SIZE,
},
.cra_priority = 4001,
.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct qat_alg_session_ctx),
+ .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_aead_type,
.cra_module = THIS_MODULE,
- .cra_init = qat_alg_sha512_init,
- .cra_exit = qat_alg_exit,
+ .cra_init = qat_alg_aead_sha512_init,
+ .cra_exit = qat_alg_aead_exit,
.cra_u = {
.aead = {
- .setkey = qat_alg_setkey,
- .decrypt = qat_alg_dec,
- .encrypt = qat_alg_enc,
- .givencrypt = qat_alg_genivenc,
+ .setkey = qat_alg_aead_setkey,
+ .decrypt = qat_alg_aead_dec,
+ .encrypt = qat_alg_aead_enc,
+ .givencrypt = qat_alg_aead_genivenc,
.ivsize = AES_BLOCK_SIZE,
.maxauthsize = SHA512_DIGEST_SIZE,
},
},
+}, {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "qat_aes_cbc",
+ .cra_priority = 4001,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = qat_alg_ablkcipher_init,
+ .cra_exit = qat_alg_ablkcipher_exit,
+ .cra_u = {
+ .ablkcipher = {
+ .setkey = qat_alg_ablkcipher_setkey,
+ .decrypt = qat_alg_ablkcipher_decrypt,
+ .encrypt = qat_alg_ablkcipher_encrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ },
+ },
} };
int qat_algs_register(void)
int i;
for (i = 0; i < ARRAY_SIZE(qat_algs); i++)
- qat_algs[i].cra_flags = CRYPTO_ALG_TYPE_AEAD |
- CRYPTO_ALG_ASYNC;
+ qat_algs[i].cra_flags =
+ (qat_algs[i].cra_type == &crypto_aead_type) ?
+ CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC :
+ CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
+
return crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs));
}
return 0;
struct qat_alg_buf_list *blout;
dma_addr_t bloutp;
size_t sz;
+ size_t sz_out;
};
+struct qat_crypto_request;
+
struct qat_crypto_request {
struct icp_qat_fw_la_bulk_req req;
- struct qat_alg_session_ctx *ctx;
- struct aead_request *areq;
+ union {
+ struct qat_alg_aead_ctx *aead_ctx;
+ struct qat_alg_ablkcipher_ctx *ablkcipher_ctx;
+ };
+ union {
+ struct aead_request *aead_req;
+ struct ablkcipher_request *ablkcipher_req;
+ };
struct qat_crypto_request_buffs buf;
+ void (*cb)(struct icp_qat_fw_la_resp *resp,
+ struct qat_crypto_request *req);
};
+
#endif
*/
#include <adf_accel_devices.h>
#include "adf_dh895xcc_hw_data.h"
+#include "adf_common_drv.h"
#include "adf_drv.h"
/* Worker thread to service arbiter mappings based on dev SKUs */
}
}
+static void adf_enable_ints(struct adf_accel_dev *accel_dev)
+{
+ void __iomem *addr;
+
+ addr = (&GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR])->virt_addr;
+
+ /* Enable bundle and misc interrupts */
+ ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET,
+ ADF_DH895XCC_SMIA0_MASK);
+ ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET,
+ ADF_DH895XCC_SMIA1_MASK);
+}
+
void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data)
{
hw_data->dev_class = &dh895xcc_class;
hw_data->get_misc_bar_id = get_misc_bar_id;
hw_data->get_sku = get_sku;
hw_data->fw_name = ADF_DH895XCC_FW;
+ hw_data->init_admin_comms = adf_init_admin_comms;
+ hw_data->exit_admin_comms = adf_exit_admin_comms;
+ hw_data->init_arb = adf_init_arb;
+ hw_data->exit_arb = adf_exit_arb;
+ hw_data->enable_ints = adf_enable_ints;
}
void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data)
struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev;
int i;
- adf_exit_admin_comms(accel_dev);
- adf_exit_arb(accel_dev);
- adf_cleanup_etr_data(accel_dev);
+ adf_dev_shutdown(accel_dev);
for (i = 0; i < ADF_PCI_MAX_BARS; i++) {
struct adf_bar *bar = &accel_pci_dev->pci_bars[i];
kfree(accel_dev);
}
-static int qat_dev_start(struct adf_accel_dev *accel_dev)
+static int adf_dev_configure(struct adf_accel_dev *accel_dev)
{
int cpus = num_online_cpus();
int banks = GET_MAX_BANKS(accel_dev);
goto err;
set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
- return adf_dev_start(accel_dev);
+ return 0;
err:
dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n");
return -EINVAL;
struct adf_accel_dev *accel_dev;
struct adf_accel_pci *accel_pci_dev;
struct adf_hw_device_data *hw_data;
- void __iomem *pmisc_bar_addr = NULL;
char name[ADF_DEVICE_NAME_LENGTH];
unsigned int i, bar_nr;
int ret;
ret = -EFAULT;
goto out_err;
}
- if (i == ADF_DH895XCC_PMISC_BAR)
- pmisc_bar_addr = bar->virt_addr;
}
pci_set_master(pdev);
goto out_err;
}
- if (adf_init_etr_data(accel_dev)) {
- dev_err(&pdev->dev, "Failed initialize etr\n");
- ret = -EFAULT;
- goto out_err;
- }
-
- if (adf_init_admin_comms(accel_dev)) {
- dev_err(&pdev->dev, "Failed initialize admin comms\n");
- ret = -EFAULT;
- goto out_err;
- }
-
- if (adf_init_arb(accel_dev)) {
- dev_err(&pdev->dev, "Failed initialize hw arbiter\n");
- ret = -EFAULT;
- goto out_err;
- }
if (pci_save_state(pdev)) {
dev_err(&pdev->dev, "Failed to save pci state\n");
ret = -ENOMEM;
goto out_err;
}
- /* Enable bundle and misc interrupts */
- ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET,
- ADF_DH895XCC_SMIA0_MASK);
- ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET,
- ADF_DH895XCC_SMIA1_MASK);
+ ret = adf_dev_configure(accel_dev);
+ if (ret)
+ goto out_err;
+
+ ret = adf_dev_init(accel_dev);
+ if (ret)
+ goto out_err;
- ret = qat_dev_start(accel_dev);
+ ret = adf_dev_start(accel_dev);
if (ret) {
adf_dev_stop(accel_dev);
goto out_err;
err = dma_map_sg(dev, sg, 1, dir);
if (!err)
return -EFAULT;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
} else {
err = dma_map_sg(dev, sg, nents, dir);
if (chained)
while (sg) {
dma_unmap_sg(dev, sg, 1, dir);
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
else
dma_unmap_sg(dev, sg, nents, dir);
nbytes -= sg->length;
if (!sg_is_last(sg) && (sg + 1)->length == 0 && chained)
*chained = true;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
return nents;
break;
len += sg_dma_len(sg);
sg_last = sg;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
if (!sg_last)
break;
}
nbytes -= sg->length;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
return nbytes;
if (unlikely(chained))
while (sg) {
dma_map_sg(dev, sg, 1, dir);
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
else
dma_map_sg(dev, sg, nents, dir);
{
while (sg) {
dma_unmap_sg(dev, sg, 1, dir);
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
}
link_tbl_ptr->j_extent = 0;
link_tbl_ptr++;
cryptlen -= sg_dma_len(sg);
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
/* adjust (decrease) last one (or two) entry's len to cryptlen */
nbytes -= sg->length;
if (!sg_is_last(sg) && (sg + 1)->length == 0)
*chained = true;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
return sg_nents;
.dst_addr = device_data->phybase + CRYP_DMA_TX_FIFO,
.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
.dst_maxburst = 4,
- };
+ };
struct dma_slave_config cryp2mem = {
.direction = DMA_DEV_TO_MEM,
.src_addr = device_data->phybase + CRYP_DMA_RX_FIFO,
.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES,
.src_maxburst = 4,
- };
+ };
dma_cap_zero(device_data->dma.mask);
dma_cap_set(DMA_SLAVE, device_data->dma.mask);
while (nbytes > 0) {
nbytes -= sg->length;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
nents++;
}
static SIMPLE_DEV_PM_OPS(ux500_cryp_pm, ux500_cryp_suspend, ux500_cryp_resume);
static const struct of_device_id ux500_cryp_match[] = {
- { .compatible = "stericsson,ux500-cryp" },
- { },
+ { .compatible = "stericsson,ux500-cryp" },
+ { },
};
static struct platform_driver cryp_driver = {
void (*release)(void *private);
int (*setkey)(void *private, const u8 *key, unsigned int keylen);
int (*accept)(void *private, struct sock *sk);
+ int (*setauthsize)(void *private, unsigned int authsize);
struct proto_ops *ops;
struct module *owner;
sg1[num - 1].page_link |= 0x01;
}
-static inline struct scatterlist *scatterwalk_sg_next(struct scatterlist *sg)
-{
- if (sg_is_last(sg))
- return NULL;
-
- return (++sg)->length ? sg : sg_chain_ptr(sg);
-}
-
static inline void scatterwalk_crypto_chain(struct scatterlist *head,
struct scatterlist *sg,
int chain, int num)
{
if (chain) {
head->length += sg->length;
- sg = scatterwalk_sg_next(sg);
+ sg = sg_next(sg);
}
if (sg)
* cipher operation completes.
*
* The callback function is registered with the ablkcipher_request handle and
- * must comply with the following template:
+ * must comply with the following template
*
* void callback_function(struct crypto_async_request *req, int error)
*/
*
* For encryption, the source is treated as the plaintext and the
* destination is the ciphertext. For a decryption operation, the use is
- * reversed: the source is the ciphertext and the destination is the plaintext.
+ * reversed - the source is the ciphertext and the destination is the plaintext.
*/
static inline void ablkcipher_request_set_crypt(
struct ablkcipher_request *req,
*/
static inline int crypto_aead_decrypt(struct aead_request *req)
{
+ if (req->cryptlen < crypto_aead_authsize(crypto_aead_reqtfm(req)))
+ return -EINVAL;
+
return crypto_aead_crt(crypto_aead_reqtfm(req))->decrypt(req);
}
* completes
*
* The callback function is registered with the aead_request handle and
- * must comply with the following template:
+ * must comply with the following template
*
* void callback_function(struct crypto_async_request *req, int error)
*/
*
* For encryption, the source is treated as the plaintext and the
* destination is the ciphertext. For a decryption operation, the use is
- * reversed: the source is the ciphertext and the destination is the plaintext.
+ * reversed - the source is the ciphertext and the destination is the plaintext.
*
* IMPORTANT NOTE AEAD requires an authentication tag (MAC). For decryption,
* the caller must concatenate the ciphertext followed by the
#ifndef LINUX_HWRANDOM_H_
#define LINUX_HWRANDOM_H_
+#include <linux/completion.h>
#include <linux/types.h>
#include <linux/list.h>
+#include <linux/kref.h>
/**
* struct hwrng - Hardware Random Number Generator driver
/* internal. */
struct list_head list;
+ struct kref ref;
+ struct completion cleanup_done;
};
/** Register a new Hardware Random Number Generator driver. */
* @s: Pointer to the start of the area.
* @count: The size of the area.
*
+ * Note: usually using memset() is just fine (!), but in cases
+ * where clearing out _local_ data at the end of a scope is
+ * necessary, memzero_explicit() should be used instead in
+ * order to prevent the compiler from optimising away zeroing.
+ *
* memzero_explicit() doesn't need an arch-specific version as
* it just invokes the one of memset() implicitly.
*/
projects / linux-drm-fsl-dcu.git / commitdiff