From 309a7e514da7d53e05b5d053594f6aabb0d382b5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Feb 2026 13:34:47 -0800 Subject: [PATCH 01/66] lib/crypto: aes: Add support for CBC-based MACs Add support for CBC-based MACs to the AES library, specifically AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC. Of these three algorithms, AES-CMAC is the most modern and the most commonly used. Use cases for the AES-CMAC library include the kernel's SMB client and server, and the bluetooth and mac80211 drivers. Support for AES-XCBC-MAC and AES-CBC-MAC is included so that there will be no performance regression in the "xcbc(aes)" and "ccm(aes)" support in the traditional crypto API once the arm64-optimized code is migrated into the library. AES-XCBC-MAC is given its own key preparation function but is otherwise identical to AES-CMAC and just reuses the AES-CMAC structs and functions. The implementation automatically uses the optimized AES key expansion and single block en/decryption functions. It also allows architectures to provide an optimized implementation of aes_cbcmac_blocks(), which allows the existing arm64-optimized code for these modes to be used. Just put the code for these modes directly in the libaes module rather than in a separate module. This is simpler, it makes it easier to share code between AES modes, and it increases the amount of inlining that is possible. (Indeed, for these reasons, most of the architecture-optimized AES code already provides multiple modes per module. x86 for example has only a single aesni-intel module. So to a large extent, this design choice just reflects the status quo.) However, since there are a lot of AES modes, there's still some value in omitting modes that are not needed at all in a given kernel. Therefore, make these modes an optional feature of libaes, controlled by CONFIG_CRYPTO_LIB_AES_CBC_MACS. This seems like a good middle ground. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260218213501.136844-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/aes-cbc-macs.h | 154 ++++++++++++++++++++++++++ lib/crypto/Kconfig | 10 ++ lib/crypto/aes.c | 198 ++++++++++++++++++++++++++++++++++ 3 files changed, 362 insertions(+) create mode 100644 include/crypto/aes-cbc-macs.h diff --git a/include/crypto/aes-cbc-macs.h b/include/crypto/aes-cbc-macs.h new file mode 100644 index 000000000000..e61df108b926 --- /dev/null +++ b/include/crypto/aes-cbc-macs.h @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Support for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CBC_MACS_H +#define _CRYPTO_AES_CBC_MACS_H + +#include + +/** + * struct aes_cmac_key - Prepared key for AES-CMAC or AES-XCBC-MAC + * @aes: The AES key for cipher block chaining + * @k_final: Finalization subkeys for the final block. + * k_final[0] (CMAC K1, XCBC-MAC K2) is used if it's a full block. + * k_final[1] (CMAC K2, XCBC-MAC K3) is used if it's a partial block. + */ +struct aes_cmac_key { + struct aes_enckey aes; + union { + u8 b[AES_BLOCK_SIZE]; + __be64 w[2]; + } k_final[2]; +}; + +/** + * struct aes_cmac_ctx - Context for computing an AES-CMAC or AES-XCBC-MAC value + * @key: Pointer to the key struct. A pointer is used rather than a copy of the + * struct, since the key struct size may be large. It is assumed that the + * key lives at least as long as the context. + * @partial_len: Number of bytes that have been XOR'ed into @h since the last + * AES encryption. This is 0 if no data has been processed yet, + * or between 1 and AES_BLOCK_SIZE inclusive otherwise. + * @h: The current chaining value + */ +struct aes_cmac_ctx { + const struct aes_cmac_key *key; + size_t partial_len; + u8 h[AES_BLOCK_SIZE]; +}; + +/** + * aes_cmac_preparekey() - Prepare a key for AES-CMAC + * @key: (output) The key struct to initialize + * @in_key: The raw AES key + * @key_len: Length of the raw key in bytes. The supported values are + * AES_KEYSIZE_128, AES_KEYSIZE_192, and AES_KEYSIZE_256. + * + * Context: Any context. + * Return: 0 on success or -EINVAL if the given key length is invalid. No other + * errors are possible, so callers that always pass a valid key length + * don't need to check for errors. + */ +int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, + size_t key_len); + +/** + * aes_xcbcmac_preparekey() - Prepare a key for AES-XCBC-MAC + * @key: (output) The key struct to initialize + * @in_key: The raw key. As per the AES-XCBC-MAC specification (RFC 3566), this + * is 128 bits, matching the internal use of AES-128. + * + * AES-XCBC-MAC and AES-CMAC are the same except for the key preparation. After + * that step, AES-XCBC-MAC is supported via the aes_cmac_* functions. + * + * New users should use AES-CMAC instead of AES-XCBC-MAC. + * + * Context: Any context. + */ +void aes_xcbcmac_preparekey(struct aes_cmac_key *key, + const u8 in_key[at_least AES_KEYSIZE_128]); + +/** + * aes_cmac_init() - Start computing an AES-CMAC or AES-XCBC-MAC value + * @ctx: (output) The context to initialize + * @key: The key to use. Note that a pointer to the key is saved in the + * context, so the key must live at least as long as the context. + * + * This supports both AES-CMAC and AES-XCBC-MAC. Which one is done depends on + * whether aes_cmac_preparekey() or aes_xcbcmac_preparekey() was called. + */ +static inline void aes_cmac_init(struct aes_cmac_ctx *ctx, + const struct aes_cmac_key *key) +{ + *ctx = (struct aes_cmac_ctx){ .key = key }; +} + +/** + * aes_cmac_update() - Update an AES-CMAC or AES-XCBC-MAC context with more data + * @ctx: The context to update; must have been initialized + * @data: The message data + * @data_len: The data length in bytes. Doesn't need to be block-aligned. + * + * This can be called any number of times. + * + * Context: Any context. + */ +void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len); + +/** + * aes_cmac_final() - Finish computing an AES-CMAC or AES-XCBC-MAC value + * @ctx: The context to finalize; must have been initialized + * @out: (output) The resulting MAC + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[at_least AES_BLOCK_SIZE]); + +/** + * aes_cmac() - Compute AES-CMAC or AES-XCBC-MAC in one shot + * @key: The key to use + * @data: The message data + * @data_len: The data length in bytes + * @out: (output) The resulting AES-CMAC or AES-XCBC-MAC value + * + * This supports both AES-CMAC and AES-XCBC-MAC. Which one is done depends on + * whether aes_cmac_preparekey() or aes_xcbcmac_preparekey() was called. + * + * Context: Any context. + */ +static inline void aes_cmac(const struct aes_cmac_key *key, const u8 *data, + size_t data_len, u8 out[at_least AES_BLOCK_SIZE]) +{ + struct aes_cmac_ctx ctx; + + aes_cmac_init(&ctx, key); + aes_cmac_update(&ctx, data, data_len); + aes_cmac_final(&ctx, out); +} + +/* + * AES-CBC-MAC support. This is provided only for use by the implementation of + * AES-CCM. It should have no other users. Warning: unlike AES-CMAC and + * AES-XCBC-MAC, AES-CBC-MAC isn't a secure MAC for variable-length messages. + */ +struct aes_cbcmac_ctx { + const struct aes_enckey *key; + size_t partial_len; + u8 h[AES_BLOCK_SIZE]; +}; +static inline void aes_cbcmac_init(struct aes_cbcmac_ctx *ctx, + const struct aes_enckey *key) +{ + *ctx = (struct aes_cbcmac_ctx){ .key = key }; +} +void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, + size_t data_len); +void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, + u8 out[at_least AES_BLOCK_SIZE]); + +#endif /* _CRYPTO_AES_CBC_MACS_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 032f9755f999..42ec51645915 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -10,6 +10,8 @@ config CRYPTO_LIB_UTILS config CRYPTO_LIB_AES tristate + # Select dependencies of modes that are part of libaes. + select CRYPTO_LIB_UTILS if CRYPTO_LIB_AES_CBC_MACS config CRYPTO_LIB_AES_ARCH bool @@ -28,6 +30,14 @@ config CRYPTO_LIB_AESCFB select CRYPTO_LIB_AES select CRYPTO_LIB_UTILS +config CRYPTO_LIB_AES_CBC_MACS + tristate + select CRYPTO_LIB_AES + help + The AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC library functions. Select + this if your module uses any of the functions from + . + config CRYPTO_LIB_AESGCM tristate select CRYPTO_LIB_AES diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index b73e19f1bb95..39deae6105c0 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -4,7 +4,9 @@ * Copyright 2026 Google LLC */ +#include #include +#include #include #include #include @@ -512,6 +514,202 @@ void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], } EXPORT_SYMBOL(aes_decrypt); +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) + +#ifndef aes_cbcmac_blocks_arch +static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, + bool enc_after) +{ + return false; +} +#endif + +/* This assumes nblocks >= 1. */ +static void aes_cbcmac_blocks(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, bool enc_after) +{ + if (aes_cbcmac_blocks_arch(h, key, data, nblocks, enc_before, + enc_after)) + return; + + if (enc_before) + aes_encrypt(key, h, h); + for (; nblocks > 1; nblocks--) { + crypto_xor(h, data, AES_BLOCK_SIZE); + data += AES_BLOCK_SIZE; + aes_encrypt(key, h, h); + } + crypto_xor(h, data, AES_BLOCK_SIZE); + if (enc_after) + aes_encrypt(key, h, h); +} + +int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, + size_t key_len) +{ + u64 hi, lo, mask; + int err; + + /* Prepare the AES key. */ + err = aes_prepareenckey(&key->aes, in_key, key_len); + if (err) + return err; + + /* + * Prepare the subkeys K1 and K2 by encrypting the all-zeroes block, + * then multiplying by 'x' and 'x^2' (respectively) in GF(2^128). + * Reference: NIST SP 800-38B, Section 6.1 "Subkey Generation". + */ + memset(key->k_final[0].b, 0, AES_BLOCK_SIZE); + aes_encrypt(&key->aes, key->k_final[0].b, key->k_final[0].b); + hi = be64_to_cpu(key->k_final[0].w[0]); + lo = be64_to_cpu(key->k_final[0].w[1]); + for (int i = 0; i < 2; i++) { + mask = ((s64)hi >> 63) & 0x87; + hi = (hi << 1) ^ (lo >> 63); + lo = (lo << 1) ^ mask; + key->k_final[i].w[0] = cpu_to_be64(hi); + key->k_final[i].w[1] = cpu_to_be64(lo); + } + return 0; +} +EXPORT_SYMBOL_GPL(aes_cmac_preparekey); + +void aes_xcbcmac_preparekey(struct aes_cmac_key *key, + const u8 in_key[AES_KEYSIZE_128]) +{ + static const u8 constants[3][AES_BLOCK_SIZE] = { + { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, + { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, + { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, + }; + u8 new_aes_key[AES_BLOCK_SIZE]; + + static_assert(AES_BLOCK_SIZE == AES_KEYSIZE_128); + aes_prepareenckey(&key->aes, in_key, AES_BLOCK_SIZE); + aes_encrypt(&key->aes, new_aes_key, constants[0]); + aes_encrypt(&key->aes, key->k_final[0].b, constants[1]); + aes_encrypt(&key->aes, key->k_final[1].b, constants[2]); + aes_prepareenckey(&key->aes, new_aes_key, AES_BLOCK_SIZE); + memzero_explicit(new_aes_key, AES_BLOCK_SIZE); +} +EXPORT_SYMBOL_GPL(aes_xcbcmac_preparekey); + +void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len) +{ + bool enc_before = false; + size_t nblocks; + + if (ctx->partial_len) { + /* XOR data into a pending block. */ + size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); + + crypto_xor(&ctx->h[ctx->partial_len], data, l); + data += l; + data_len -= l; + ctx->partial_len += l; + if (data_len == 0) { + /* + * Either the pending block hasn't been filled yet, or + * no more data was given so it's not yet known whether + * the block is the final block. + */ + return; + } + /* Pending block has been filled and isn't the final block. */ + enc_before = true; + } + + nblocks = data_len / AES_BLOCK_SIZE; + data_len %= AES_BLOCK_SIZE; + if (nblocks == 0) { + /* 0 additional full blocks, then optionally a partial block */ + if (enc_before) + aes_encrypt(&ctx->key->aes, ctx->h, ctx->h); + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; + } else if (data_len != 0) { + /* 1 or more additional full blocks, then a partial block */ + aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, + enc_before, /* enc_after= */ true); + data += nblocks * AES_BLOCK_SIZE; + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; + } else { + /* + * 1 or more additional full blocks only. Encryption of the + * last block is delayed until it's known whether it's the final + * block in the message or not. + */ + aes_cbcmac_blocks(ctx->h, &ctx->key->aes, data, nblocks, + enc_before, /* enc_after= */ false); + ctx->partial_len = AES_BLOCK_SIZE; + } +} +EXPORT_SYMBOL_GPL(aes_cmac_update); + +void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) +{ + if (ctx->partial_len == AES_BLOCK_SIZE) { + /* Final block is a full block. Use k_final[0]. */ + crypto_xor(ctx->h, ctx->key->k_final[0].b, AES_BLOCK_SIZE); + } else { + /* Final block is a partial block. Pad, and use k_final[1]. */ + ctx->h[ctx->partial_len] ^= 0x80; + crypto_xor(ctx->h, ctx->key->k_final[1].b, AES_BLOCK_SIZE); + } + aes_encrypt(&ctx->key->aes, out, ctx->h); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(aes_cmac_final); + +void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, + size_t data_len) +{ + bool enc_before = false; + size_t nblocks; + + if (ctx->partial_len) { + size_t l = min(data_len, AES_BLOCK_SIZE - ctx->partial_len); + + crypto_xor(&ctx->h[ctx->partial_len], data, l); + data += l; + data_len -= l; + ctx->partial_len += l; + if (ctx->partial_len < AES_BLOCK_SIZE) + return; + enc_before = true; + } + + nblocks = data_len / AES_BLOCK_SIZE; + data_len %= AES_BLOCK_SIZE; + if (nblocks == 0) { + if (enc_before) + aes_encrypt(ctx->key, ctx->h, ctx->h); + } else { + aes_cbcmac_blocks(ctx->h, ctx->key, data, nblocks, enc_before, + /* enc_after= */ true); + data += nblocks * AES_BLOCK_SIZE; + } + crypto_xor(ctx->h, data, data_len); + ctx->partial_len = data_len; +} +EXPORT_SYMBOL_NS_GPL(aes_cbcmac_update, "CRYPTO_INTERNAL"); + +void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) +{ + if (ctx->partial_len) + aes_encrypt(ctx->key, out, ctx->h); + else + memcpy(out, ctx->h, AES_BLOCK_SIZE); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); +#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ + #ifdef aes_mod_init_arch static int __init aes_mod_init(void) { From 7137cbf2b5c9feb6302d6da116eab2047c5f05d2 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Feb 2026 13:34:48 -0800 Subject: [PATCH 02/66] crypto: aes - Add cmac, xcbc, and cbcmac algorithms using library Update the "aes" module to implement "cmac(aes)", "xcbc(aes)", and "cbcmac(aes)" algorithms using the corresponding library functions, and register these with the crypto_shash API. Each algorithm is included only if the corresponding existing kconfig option is enabled. This allows the architecture-optimized implementations of these algorithms to continue to be accessible via the crypto_shash API once they are migrated into the library. For "xcbc(aes)", I also fixed the bug where AES key lengths other than 128 bits were allowed, so that this bug didn't have to be implemented in the library. The AES-XCBC-MAC specification (RFC 3566) is clear that key lengths other than 128 bits MUST NOT be supported. AES-XCBC-MAC derives a 128-bit subkey internally, so the nonstandard support for longer AES keys didn't really work: AES-128 was still used internally. In the unlikely event that someone is actually relying on the broken and nonstandard support for longer AES-XCBC-MAC keys, we can fairly easily reintroduce it. But it seems unnecessary: the only user of "xcbc(aes)" seems to be IPsec, which uses 128-bit keys with it. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260218213501.136844-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 2 + crypto/aes.c | 183 ++++++++++++++++++++++++++- crypto/testmgr.c | 10 +- drivers/crypto/starfive/jh7110-aes.c | 2 +- 4 files changed, 190 insertions(+), 7 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index b4bb85e8e226..c977d40f906d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -361,6 +361,8 @@ config CRYPTO_AES tristate "AES (Advanced Encryption Standard)" select CRYPTO_ALGAPI select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM + select CRYPTO_HASH if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM help AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3) diff --git a/crypto/aes.c b/crypto/aes.c index ae8385df0ce5..6bf23eb0503f 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -5,8 +5,10 @@ * Copyright 2026 Google LLC */ +#include #include #include +#include #include static_assert(__alignof__(struct aes_key) <= CRYPTO_MINALIGN); @@ -33,6 +35,98 @@ static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) aes_decrypt(key, out, in); } +static_assert(__alignof__(struct aes_cmac_key) <= CRYPTO_MINALIGN); +#define AES_CMAC_KEY(tfm) ((struct aes_cmac_key *)crypto_shash_ctx(tfm)) +#define AES_CMAC_CTX(desc) ((struct aes_cmac_ctx *)shash_desc_ctx(desc)) + +static int __maybe_unused crypto_aes_cmac_setkey(struct crypto_shash *tfm, + const u8 *in_key, + unsigned int key_len) +{ + return aes_cmac_preparekey(AES_CMAC_KEY(tfm), in_key, key_len); +} + +static int __maybe_unused crypto_aes_xcbc_setkey(struct crypto_shash *tfm, + const u8 *in_key, + unsigned int key_len) +{ + if (key_len != AES_KEYSIZE_128) + return -EINVAL; + aes_xcbcmac_preparekey(AES_CMAC_KEY(tfm), in_key); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_init(struct shash_desc *desc) +{ + aes_cmac_init(AES_CMAC_CTX(desc), AES_CMAC_KEY(desc->tfm)); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_update(struct shash_desc *desc, + const u8 *data, + unsigned int len) +{ + aes_cmac_update(AES_CMAC_CTX(desc), data, len); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_final(struct shash_desc *desc, + u8 *out) +{ + aes_cmac_final(AES_CMAC_CTX(desc), out); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_digest(struct shash_desc *desc, + const u8 *data, + unsigned int len, u8 *out) +{ + aes_cmac(AES_CMAC_KEY(desc->tfm), data, len, out); + return 0; +} + +static_assert(__alignof__(struct aes_enckey) <= CRYPTO_MINALIGN); +#define AES_CBCMAC_KEY(tfm) ((struct aes_enckey *)crypto_shash_ctx(tfm)) +#define AES_CBCMAC_CTX(desc) ((struct aes_cbcmac_ctx *)shash_desc_ctx(desc)) + +static int __maybe_unused crypto_aes_cbcmac_setkey(struct crypto_shash *tfm, + const u8 *in_key, + unsigned int key_len) +{ + return aes_prepareenckey(AES_CBCMAC_KEY(tfm), in_key, key_len); +} + +static int __maybe_unused crypto_aes_cbcmac_init(struct shash_desc *desc) +{ + aes_cbcmac_init(AES_CBCMAC_CTX(desc), AES_CBCMAC_KEY(desc->tfm)); + return 0; +} + +static int __maybe_unused crypto_aes_cbcmac_update(struct shash_desc *desc, + const u8 *data, + unsigned int len) +{ + aes_cbcmac_update(AES_CBCMAC_CTX(desc), data, len); + return 0; +} + +static int __maybe_unused crypto_aes_cbcmac_final(struct shash_desc *desc, + u8 *out) +{ + aes_cbcmac_final(AES_CBCMAC_CTX(desc), out); + return 0; +} + +static int __maybe_unused crypto_aes_cbcmac_digest(struct shash_desc *desc, + const u8 *data, + unsigned int len, u8 *out) +{ + aes_cbcmac_init(AES_CBCMAC_CTX(desc), AES_CBCMAC_KEY(desc->tfm)); + aes_cbcmac_update(AES_CBCMAC_CTX(desc), data, len); + aes_cbcmac_final(AES_CBCMAC_CTX(desc), out); + return 0; +} + static struct crypto_alg alg = { .cra_name = "aes", .cra_driver_name = "aes-lib", @@ -48,19 +142,106 @@ static struct crypto_alg alg = { .cia_decrypt = crypto_aes_decrypt } } }; +static struct shash_alg mac_algs[] = { +#if IS_ENABLED(CONFIG_CRYPTO_CMAC) + { + .base.cra_name = "cmac(aes)", + .base.cra_driver_name = "cmac-aes-lib", + .base.cra_priority = 300, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_cmac_key), + .base.cra_module = THIS_MODULE, + .digestsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_cmac_setkey, + .init = crypto_aes_cmac_init, + .update = crypto_aes_cmac_update, + .final = crypto_aes_cmac_final, + .digest = crypto_aes_cmac_digest, + .descsize = sizeof(struct aes_cmac_ctx), + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XCBC) + { + /* + * Note that the only difference between xcbc(aes) and cmac(aes) + * is the preparekey function. + */ + .base.cra_name = "xcbc(aes)", + .base.cra_driver_name = "xcbc-aes-lib", + .base.cra_priority = 300, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_cmac_key), + .base.cra_module = THIS_MODULE, + .digestsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_xcbc_setkey, + .init = crypto_aes_cmac_init, + .update = crypto_aes_cmac_update, + .final = crypto_aes_cmac_final, + .digest = crypto_aes_cmac_digest, + .descsize = sizeof(struct aes_cmac_ctx), + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CCM) + { + .base.cra_name = "cbcmac(aes)", + .base.cra_driver_name = "cbcmac-aes-lib", + .base.cra_priority = 300, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_enckey), + .base.cra_module = THIS_MODULE, + .digestsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_cbcmac_setkey, + .init = crypto_aes_cbcmac_init, + .update = crypto_aes_cbcmac_update, + .final = crypto_aes_cbcmac_final, + .digest = crypto_aes_cbcmac_digest, + .descsize = sizeof(struct aes_cbcmac_ctx), + }, +#endif +}; + static int __init crypto_aes_mod_init(void) { - return crypto_register_alg(&alg); + int err = crypto_register_alg(&alg); + + if (err) + return err; + + if (ARRAY_SIZE(mac_algs) > 0) { + err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs)); + if (err) + goto err_unregister_alg; + } /* Else, CONFIG_CRYPTO_HASH might not be enabled. */ + return 0; + +err_unregister_alg: + crypto_unregister_alg(&alg); + return err; } module_init(crypto_aes_mod_init); static void __exit crypto_aes_mod_exit(void) { + if (ARRAY_SIZE(mac_algs) > 0) + crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); crypto_unregister_alg(&alg); } module_exit(crypto_aes_mod_exit); MODULE_DESCRIPTION("Crypto API support for AES block cipher"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); MODULE_LICENSE("GPL"); MODULE_ALIAS_CRYPTO("aes"); MODULE_ALIAS_CRYPTO("aes-lib"); +#if IS_ENABLED(CONFIG_CRYPTO_CMAC) +MODULE_ALIAS_CRYPTO("cmac(aes)"); +MODULE_ALIAS_CRYPTO("cmac-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XCBC) +MODULE_ALIAS_CRYPTO("xcbc(aes)"); +MODULE_ALIAS_CRYPTO("xcbc-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CCM) +MODULE_ALIAS_CRYPTO("cbcmac(aes)"); +MODULE_ALIAS_CRYPTO("cbcmac-aes-lib"); +#endif diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 4985411dedae..fec950f1628b 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4388,7 +4388,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, { #endif .alg = "cbcmac(aes)", - .generic_driver = "cbcmac(aes-lib)", + .generic_driver = "cbcmac-aes-lib", .test = alg_test_hash, .suite = { .hash = __VECS(aes_cbcmac_tv_template) @@ -4401,7 +4401,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "ccm(aes)", - .generic_driver = "ccm_base(ctr(aes-lib),cbcmac(aes-lib))", + .generic_driver = "ccm_base(ctr(aes-lib),cbcmac-aes-lib)", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -4429,7 +4429,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, }, { .alg = "cmac(aes)", - .generic_driver = "cmac(aes-lib)", + .generic_driver = "cmac-aes-lib", .fips_allowed = 1, .test = alg_test_hash, .suite = { @@ -5326,7 +5326,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4309(ccm(aes))", - .generic_driver = "rfc4309(ccm_base(ctr(aes-lib),cbcmac(aes-lib)))", + .generic_driver = "rfc4309(ccm_base(ctr(aes-lib),cbcmac-aes-lib))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -5515,7 +5515,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "xcbc(aes)", - .generic_driver = "xcbc(aes-lib)", + .generic_driver = "xcbc-aes-lib", .test = alg_test_hash, .suite = { .hash = __VECS(aes_xcbc128_tv_template) diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index c1dc1e43e117..2e2d97d17e6c 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -1003,7 +1003,7 @@ static int starfive_aes_ctr_init_tfm(struct crypto_skcipher *tfm) static int starfive_aes_ccm_init_tfm(struct crypto_aead *tfm) { - return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-lib),cbcmac(aes-lib))"); + return starfive_aes_aead_init_tfm(tfm, "ccm_base(ctr(aes-lib),cbcmac-aes-lib)"); } static int starfive_aes_gcm_init_tfm(struct crypto_aead *tfm) From f8f08d7cc43237e91e3aedf7b67d015d24c38fcc Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Feb 2026 13:34:49 -0800 Subject: [PATCH 03/66] crypto: arm64/aes - Fix 32-bit aes_mac_update() arg treated as 64-bit Since the 'enc_after' argument to neon_aes_mac_update() and ce_aes_mac_update() has type 'int', it needs to be accessed using the corresponding 32-bit register, not the 64-bit register. The upper half of the corresponding 64-bit register may contain garbage. Fixes: 4860620da7e5 ("crypto: arm64/aes - add NEON/Crypto Extensions CBCMAC/CMAC/XCBC driver") Cc: stable@vger.kernel.org Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260218213501.136844-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/crypto/aes-modes.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index 0e834a2c062c..e793478f37c1 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -838,7 +838,7 @@ AES_FUNC_START(aes_mac_update) encrypt_block v0, w2, x1, x7, w8 eor v0.16b, v0.16b, v4.16b cmp w3, wzr - csinv x5, x6, xzr, eq + csinv w5, w6, wzr, eq cbz w5, .Lmacout encrypt_block v0, w2, x1, x7, w8 st1 {v0.16b}, [x4] /* return dg */ @@ -852,7 +852,7 @@ AES_FUNC_START(aes_mac_update) eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ subs w3, w3, #1 - csinv x5, x6, xzr, eq + csinv w5, w6, wzr, eq cbz w5, .Lmacout .Lmacenc: From 4b908403209252e59ecad4c068bf967fa3f07525 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Feb 2026 13:34:50 -0800 Subject: [PATCH 04/66] lib/crypto: arm64/aes: Move assembly code for AES modes into libaes To migrate the support for CBC-based MACs into libaes, the corresponding arm64 assembly code needs to be moved there. However, the arm64 AES assembly code groups many AES modes together; individual modes aren't easily separable. (This isn't unique to arm64; other architectures organize their AES modes similarly.) Since the other AES modes will be migrated into the library eventually too, just move the full assembly files for the AES modes into the library. (This is similar to what I already did for PowerPC and SPARC.) Specifically: move the assembly files aes-ce.S, aes-modes.S, and aes-neon.S and their build rules; declare the assembly functions in ; and export the assembly functions from libaes. Note that the exports and public declarations of the assembly functions are temporary. They exist only to keep arch/arm64/crypto/ working until the AES modes are fully moved into the library. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260218213501.136844-5-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/crypto/Makefile | 4 +- arch/arm64/crypto/aes-ce-ccm-glue.c | 4 -- arch/arm64/crypto/aes-glue.c | 47 +------------ arch/arm64/crypto/aes-neonbs-glue.c | 15 +--- include/crypto/aes.h | 69 +++++++++++++++++++ lib/crypto/Makefile | 4 +- .../crypto => lib/crypto/arm64}/aes-ce.S | 3 +- .../crypto => lib/crypto/arm64}/aes-modes.S | 2 +- .../crypto => lib/crypto/arm64}/aes-neon.S | 2 +- lib/crypto/arm64/aes.h | 30 ++++++++ 10 files changed, 109 insertions(+), 71 deletions(-) rename {arch/arm64/crypto => lib/crypto/arm64}/aes-ce.S (96%) rename {arch/arm64/crypto => lib/crypto/arm64}/aes-modes.S (99%) rename {arch/arm64/crypto => lib/crypto/arm64}/aes-neon.S (99%) diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index 3574e917bc37..8a8e3e551ed3 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -33,10 +33,10 @@ obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o obj-$(CONFIG_CRYPTO_AES_ARM64_CE_BLK) += aes-ce-blk.o -aes-ce-blk-y := aes-glue-ce.o aes-ce.o +aes-ce-blk-y := aes-glue-ce.o obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o -aes-neon-blk-y := aes-glue-neon.o aes-neon.o +aes-neon-blk-y := aes-glue-neon.o obj-$(CONFIG_CRYPTO_AES_ARM64_BS) += aes-neon-bs.o aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index db371ac051fc..45aed0073283 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -31,10 +31,6 @@ static int num_rounds(struct crypto_aes_ctx *ctx) return 6 + ctx->key_length / 4; } -asmlinkage u32 ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds, - int blocks, u8 dg[], int enc_before, - int enc_after); - asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes, u32 const rk[], u32 rounds, u8 mac[], u8 ctr[], u8 const final_iv[]); diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 92f43e1cd097..fd7c3a560a71 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -71,48 +71,9 @@ MODULE_ALIAS_CRYPTO("xcbc(aes)"); MODULE_ALIAS_CRYPTO("cbcmac(aes)"); MODULE_AUTHOR("Ard Biesheuvel "); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); MODULE_LICENSE("GPL v2"); -/* defined in aes-modes.S */ -asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks); -asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks); - -asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks, u8 iv[]); -asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks, u8 iv[]); - -asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int bytes, u8 const iv[]); -asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int bytes, u8 const iv[]); - -asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int bytes, u8 ctr[]); - -asmlinkage void aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int bytes, u8 ctr[], int byte_ctr); - -asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], - int rounds, int bytes, u32 const rk2[], u8 iv[], - int first); -asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], - int rounds, int bytes, u32 const rk2[], u8 iv[], - int first); - -asmlinkage void aes_essiv_cbc_encrypt(u8 out[], u8 const in[], u32 const rk1[], - int rounds, int blocks, u8 iv[], - u32 const rk2[]); -asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], - int rounds, int blocks, u8 iv[], - u32 const rk2[]); - -asmlinkage int aes_mac_update(u8 const in[], u32 const rk[], int rounds, - int blocks, u8 dg[], int enc_before, - int enc_after); - struct crypto_aes_xts_ctx { struct crypto_aes_ctx key1; struct crypto_aes_ctx __aligned(8) key2; @@ -971,13 +932,7 @@ unregister_ciphers: #ifdef USE_V8_CRYPTO_EXTENSIONS module_cpu_feature_match(AES, aes_init); -EXPORT_SYMBOL_NS(ce_aes_mac_update, "CRYPTO_INTERNAL"); #else module_init(aes_init); -EXPORT_SYMBOL(neon_aes_ecb_encrypt); -EXPORT_SYMBOL(neon_aes_cbc_encrypt); -EXPORT_SYMBOL(neon_aes_ctr_encrypt); -EXPORT_SYMBOL(neon_aes_xts_encrypt); -EXPORT_SYMBOL(neon_aes_xts_decrypt); #endif module_exit(aes_exit); diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c index cb87c8fc66b3..7630a7bf5da9 100644 --- a/arch/arm64/crypto/aes-neonbs-glue.c +++ b/arch/arm64/crypto/aes-neonbs-glue.c @@ -17,6 +17,7 @@ MODULE_AUTHOR("Ard Biesheuvel "); MODULE_DESCRIPTION("Bit sliced AES using NEON instructions"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("ecb(aes)"); @@ -42,20 +43,6 @@ asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, int blocks, u8 iv[]); -/* borrowed from aes-neon-blk.ko */ -asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks); -asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks, u8 iv[]); -asmlinkage void neon_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int bytes, u8 ctr[]); -asmlinkage void neon_aes_xts_encrypt(u8 out[], u8 const in[], - u32 const rk1[], int rounds, int bytes, - u32 const rk2[], u8 iv[], int first); -asmlinkage void neon_aes_xts_decrypt(u8 out[], u8 const in[], - u32 const rk1[], int rounds, int bytes, - u32 const rk2[], u8 iv[], int first); - struct aesbs_ctx { u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32]; int rounds; diff --git a/include/crypto/aes.h b/include/crypto/aes.h index cbf1cc96db52..91bf4667d3e9 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -167,6 +167,75 @@ int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, #ifdef CONFIG_ARM64 int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len); +asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void neon_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void neon_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void neon_aes_cbc_cts_encrypt(u8 out[], u8 const in[], + u32 const rk[], int rounds, int bytes, + u8 const iv[]); +asmlinkage void neon_aes_cbc_cts_decrypt(u8 out[], u8 const in[], + u32 const rk[], int rounds, int bytes, + u8 const iv[]); +asmlinkage void neon_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[]); +asmlinkage void neon_aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[], + int byte_ctr); +asmlinkage void neon_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void neon_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void neon_aes_essiv_cbc_encrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], + u32 const rk2[]); +asmlinkage void neon_aes_essiv_cbc_decrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], + u32 const rk2[]); +asmlinkage int neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + int blocks, u8 dg[], int enc_before, + int enc_after); + +asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 const iv[]); +asmlinkage void ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 const iv[]); +asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[]); +asmlinkage void ce_aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[], + int byte_ctr); +asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void ce_aes_essiv_cbc_encrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], u32 const rk2[]); +asmlinkage void ce_aes_essiv_cbc_decrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], u32 const rk2[]); +asmlinkage int ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + int blocks, u8 dg[], int enc_before, + int enc_after); #elif defined(CONFIG_PPC) void ppc_expand_key_128(u32 *key_enc, const u8 *key); void ppc_expand_key_192(u32 *key_enc, const u8 *key); diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 725eef05b758..c05d4b4e8e82 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -26,7 +26,9 @@ libaes-$(CONFIG_ARM) += arm/aes-cipher-core.o ifeq ($(CONFIG_ARM64),y) libaes-y += arm64/aes-cipher-core.o -libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o +libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o \ + arm64/aes-ce.o \ + arm64/aes-neon.o endif ifeq ($(CONFIG_PPC),y) diff --git a/arch/arm64/crypto/aes-ce.S b/lib/crypto/arm64/aes-ce.S similarity index 96% rename from arch/arm64/crypto/aes-ce.S rename to lib/crypto/arm64/aes-ce.S index b262eaa9170c..b853e02f7b1e 100644 --- a/arch/arm64/crypto/aes-ce.S +++ b/lib/crypto/arm64/aes-ce.S @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with - * Crypto Extensions + * AES cipher for ARMv8 with Crypto Extensions * * Copyright (C) 2013 - 2017 Linaro Ltd */ diff --git a/arch/arm64/crypto/aes-modes.S b/lib/crypto/arm64/aes-modes.S similarity index 99% rename from arch/arm64/crypto/aes-modes.S rename to lib/crypto/arm64/aes-modes.S index e793478f37c1..f4df6f84a3c7 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/lib/crypto/arm64/aes-modes.S @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES + * Chaining mode wrappers for AES * * Copyright (C) 2013 - 2017 Linaro Ltd */ diff --git a/arch/arm64/crypto/aes-neon.S b/lib/crypto/arm64/aes-neon.S similarity index 99% rename from arch/arm64/crypto/aes-neon.S rename to lib/crypto/arm64/aes-neon.S index 3a8961b6ea51..f37b1dbd887f 100644 --- a/arch/arm64/crypto/aes-neon.S +++ b/lib/crypto/arm64/aes-neon.S @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * linux/arch/arm64/crypto/aes-neon.S - AES cipher for ARMv8 NEON + * AES cipher for ARMv8 NEON * * Copyright (C) 2013 - 2017 Linaro Ltd. */ diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h index 63eea6271ef9..69f465c668f0 100644 --- a/lib/crypto/arm64/aes.h +++ b/lib/crypto/arm64/aes.h @@ -126,6 +126,36 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, } EXPORT_SYMBOL(ce_aes_expandkey); +#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) +EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_ctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_mac_update, "CRYPTO_INTERNAL"); + +EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_ctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL"); +#endif + static void aes_encrypt_arch(const struct aes_enckey *key, u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]) From 58286738b159ca93d41438a6ddcc2ea5333191b4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Feb 2026 13:34:51 -0800 Subject: [PATCH 05/66] lib/crypto: arm64/aes: Migrate optimized CBC-based MACs into library Instead of exposing the arm64-optimized CMAC, XCBC-MAC, and CBC-MAC code via arm64-specific crypto_shash algorithms, instead just implement the aes_cbcmac_blocks_arch() library function. This is much simpler, it makes the corresponding library functions be arm64-optimized, and it fixes the longstanding issue where this optimized code was disabled by default. The corresponding algorithms still remain available through crypto_shash, but individual architectures no longer need to handle it. Note that to be compatible with the library using 'size_t' lengths, the type of the return value and 'blocks' parameter to the assembly functions had to be changed to 'size_t', and the assembly code had to be updated accordingly to use the corresponding 64-bit registers. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260218213501.136844-6-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/crypto/Kconfig | 2 +- arch/arm64/crypto/aes-glue.c | 214 +---------------------------------- include/crypto/aes.h | 9 +- lib/crypto/arm64/aes-modes.S | 19 ++-- lib/crypto/arm64/aes.h | 48 +++++++- 5 files changed, 61 insertions(+), 231 deletions(-) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 81ed892b3b72..82794afaffc9 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -144,7 +144,7 @@ config CRYPTO_AES_ARM64_CE_CCM select CRYPTO_ALGAPI select CRYPTO_AES_ARM64_CE_BLK select CRYPTO_AEAD - select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CBC_MACS help AEAD cipher: AES cipher algorithms (FIPS-197) with CCM (Counter with Cipher Block Chaining-Message Authentication Code) diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index fd7c3a560a71..85497cfe76d8 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -37,7 +36,6 @@ #define aes_xctr_encrypt ce_aes_xctr_encrypt #define aes_xts_encrypt ce_aes_xts_encrypt #define aes_xts_decrypt ce_aes_xts_decrypt -#define aes_mac_update ce_aes_mac_update MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 Crypto Extensions"); #else #define MODE "neon" @@ -54,7 +52,6 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 Crypto Extensions"); #define aes_xctr_encrypt neon_aes_xctr_encrypt #define aes_xts_encrypt neon_aes_xts_encrypt #define aes_xts_decrypt neon_aes_xts_decrypt -#define aes_mac_update neon_aes_mac_update MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS/XCTR using ARMv8 NEON"); #endif #if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) @@ -66,9 +63,6 @@ MODULE_ALIAS_CRYPTO("xctr(aes)"); #endif MODULE_ALIAS_CRYPTO("cts(cbc(aes))"); MODULE_ALIAS_CRYPTO("essiv(cbc(aes),sha256)"); -MODULE_ALIAS_CRYPTO("cmac(aes)"); -MODULE_ALIAS_CRYPTO("xcbc(aes)"); -MODULE_ALIAS_CRYPTO("cbcmac(aes)"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_IMPORT_NS("CRYPTO_INTERNAL"); @@ -84,15 +78,6 @@ struct crypto_aes_essiv_cbc_ctx { struct crypto_aes_ctx __aligned(8) key2; }; -struct mac_tfm_ctx { - struct crypto_aes_ctx key; - u8 __aligned(8) consts[]; -}; - -struct mac_desc_ctx { - u8 dg[AES_BLOCK_SIZE]; -}; - static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key, unsigned int key_len) { @@ -723,211 +708,14 @@ static struct skcipher_alg aes_algs[] = { { .decrypt = essiv_cbc_decrypt, } }; -static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); - - return aes_expandkey(&ctx->key, in_key, key_len); -} - -static void cmac_gf128_mul_by_x(be128 *y, const be128 *x) -{ - u64 a = be64_to_cpu(x->a); - u64 b = be64_to_cpu(x->b); - - y->a = cpu_to_be64((a << 1) | (b >> 63)); - y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0)); -} - -static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); - be128 *consts = (be128 *)ctx->consts; - int rounds = 6 + key_len / 4; - int err; - - err = cbcmac_setkey(tfm, in_key, key_len); - if (err) - return err; - - /* encrypt the zero vector */ - scoped_ksimd() - aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, - ctx->key.key_enc, rounds, 1); - - cmac_gf128_mul_by_x(consts, consts); - cmac_gf128_mul_by_x(consts + 1, consts); - - return 0; -} - -static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key, - unsigned int key_len) -{ - static u8 const ks[3][AES_BLOCK_SIZE] = { - { [0 ... AES_BLOCK_SIZE - 1] = 0x1 }, - { [0 ... AES_BLOCK_SIZE - 1] = 0x2 }, - { [0 ... AES_BLOCK_SIZE - 1] = 0x3 }, - }; - - struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); - int rounds = 6 + key_len / 4; - u8 key[AES_BLOCK_SIZE]; - int err; - - err = cbcmac_setkey(tfm, in_key, key_len); - if (err) - return err; - - scoped_ksimd() { - aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1); - aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2); - } - - return cbcmac_setkey(tfm, key, sizeof(key)); -} - -static int mac_init(struct shash_desc *desc) -{ - struct mac_desc_ctx *ctx = shash_desc_ctx(desc); - - memset(ctx->dg, 0, AES_BLOCK_SIZE); - return 0; -} - -static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, - u8 dg[], int enc_before) -{ - int rounds = 6 + ctx->key_length / 4; - int rem; - - do { - scoped_ksimd() - rem = aes_mac_update(in, ctx->key_enc, rounds, blocks, - dg, enc_before, !enc_before); - in += (blocks - rem) * AES_BLOCK_SIZE; - blocks = rem; - } while (blocks); -} - -static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len) -{ - struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct mac_desc_ctx *ctx = shash_desc_ctx(desc); - int blocks = len / AES_BLOCK_SIZE; - - len %= AES_BLOCK_SIZE; - mac_do_update(&tctx->key, p, blocks, ctx->dg, 0); - return len; -} - -static int cbcmac_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct mac_desc_ctx *ctx = shash_desc_ctx(desc); - - if (len) { - crypto_xor(ctx->dg, src, len); - mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1); - } - memcpy(out, ctx->dg, AES_BLOCK_SIZE); - return 0; -} - -static int cmac_finup(struct shash_desc *desc, const u8 *src, unsigned int len, - u8 *out) -{ - struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct mac_desc_ctx *ctx = shash_desc_ctx(desc); - u8 *consts = tctx->consts; - - crypto_xor(ctx->dg, src, len); - if (len != AES_BLOCK_SIZE) { - ctx->dg[len] ^= 0x80; - consts += AES_BLOCK_SIZE; - } - mac_do_update(&tctx->key, consts, 1, ctx->dg, 0); - memcpy(out, ctx->dg, AES_BLOCK_SIZE); - return 0; -} - -static struct shash_alg mac_algs[] = { { - .base.cra_name = "cmac(aes)", - .base.cra_driver_name = "cmac-aes-" MODE, - .base.cra_priority = PRIO, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINAL_NONZERO, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + - 2 * AES_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = AES_BLOCK_SIZE, - .init = mac_init, - .update = mac_update, - .finup = cmac_finup, - .setkey = cmac_setkey, - .descsize = sizeof(struct mac_desc_ctx), -}, { - .base.cra_name = "xcbc(aes)", - .base.cra_driver_name = "xcbc-aes-" MODE, - .base.cra_priority = PRIO, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINAL_NONZERO, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mac_tfm_ctx) + - 2 * AES_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = AES_BLOCK_SIZE, - .init = mac_init, - .update = mac_update, - .finup = cmac_finup, - .setkey = xcbc_setkey, - .descsize = sizeof(struct mac_desc_ctx), -}, { - .base.cra_name = "cbcmac(aes)", - .base.cra_driver_name = "cbcmac-aes-" MODE, - .base.cra_priority = PRIO, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mac_tfm_ctx), - .base.cra_module = THIS_MODULE, - - .digestsize = AES_BLOCK_SIZE, - .init = mac_init, - .update = mac_update, - .finup = cbcmac_finup, - .setkey = cbcmac_setkey, - .descsize = sizeof(struct mac_desc_ctx), -} }; - static void aes_exit(void) { - crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } static int __init aes_init(void) { - int err; - - err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); - if (err) - return err; - - err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs)); - if (err) - goto unregister_ciphers; - - return 0; - -unregister_ciphers: - crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); - return err; + return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } #ifdef USE_V8_CRYPTO_EXTENSIONS diff --git a/include/crypto/aes.h b/include/crypto/aes.h index 91bf4667d3e9..3feb4105c2a2 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -200,9 +200,6 @@ asmlinkage void neon_aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], int rounds, int blocks, u8 iv[], u32 const rk2[]); -asmlinkage int neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds, - int blocks, u8 dg[], int enc_before, - int enc_after); asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], int rounds, int blocks); @@ -233,9 +230,9 @@ asmlinkage void ce_aes_essiv_cbc_encrypt(u8 out[], u8 const in[], asmlinkage void ce_aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], int rounds, int blocks, u8 iv[], u32 const rk2[]); -asmlinkage int ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds, - int blocks, u8 dg[], int enc_before, - int enc_after); +asmlinkage size_t ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + size_t blocks, u8 dg[], int enc_before, + int enc_after); #elif defined(CONFIG_PPC) void ppc_expand_key_128(u32 *key_enc, const u8 *key); void ppc_expand_key_192(u32 *key_enc, const u8 *key); diff --git a/lib/crypto/arm64/aes-modes.S b/lib/crypto/arm64/aes-modes.S index f4df6f84a3c7..fc89cd02b642 100644 --- a/lib/crypto/arm64/aes-modes.S +++ b/lib/crypto/arm64/aes-modes.S @@ -815,9 +815,11 @@ AES_FUNC_START(aes_xts_decrypt) b .Lxtsdecctsout AES_FUNC_END(aes_xts_decrypt) +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) /* - * aes_mac_update(u8 const in[], u32 const rk[], int rounds, - * int blocks, u8 dg[], int enc_before, int enc_after) + * size_t aes_mac_update(u8 const in[], u32 const rk[], int rounds, + * size_t blocks, u8 dg[], int enc_before, + * int enc_after); */ AES_FUNC_START(aes_mac_update) ld1 {v0.16b}, [x4] /* get dg */ @@ -827,7 +829,7 @@ AES_FUNC_START(aes_mac_update) encrypt_block v0, w2, x1, x7, w8 .Lmacloop4x: - subs w3, w3, #4 + subs x3, x3, #4 bmi .Lmac1x ld1 {v1.16b-v4.16b}, [x0], #64 /* get next pt block */ eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ @@ -837,7 +839,7 @@ AES_FUNC_START(aes_mac_update) eor v0.16b, v0.16b, v3.16b encrypt_block v0, w2, x1, x7, w8 eor v0.16b, v0.16b, v4.16b - cmp w3, wzr + cmp x3, xzr csinv w5, w6, wzr, eq cbz w5, .Lmacout encrypt_block v0, w2, x1, x7, w8 @@ -845,13 +847,13 @@ AES_FUNC_START(aes_mac_update) cond_yield .Lmacout, x7, x8 b .Lmacloop4x .Lmac1x: - add w3, w3, #4 + add x3, x3, #4 .Lmacloop: - cbz w3, .Lmacout + cbz x3, .Lmacout ld1 {v1.16b}, [x0], #16 /* get next pt block */ eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ - subs w3, w3, #1 + subs x3, x3, #1 csinv w5, w6, wzr, eq cbz w5, .Lmacout @@ -861,6 +863,7 @@ AES_FUNC_START(aes_mac_update) .Lmacout: st1 {v0.16b}, [x4] /* return dg */ - mov w0, w3 + mov x0, x3 ret AES_FUNC_END(aes_mac_update) +#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h index 69f465c668f0..78e7b4e5f120 100644 --- a/lib/crypto/arm64/aes.h +++ b/lib/crypto/arm64/aes.h @@ -11,6 +11,7 @@ #include #include +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes); struct aes_block { @@ -28,6 +29,9 @@ asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE], asmlinkage u32 __aes_ce_sub(u32 l); asmlinkage void __aes_ce_invert(struct aes_block *out, const struct aes_block *in); +asmlinkage size_t neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + size_t blocks, u8 dg[], int enc_before, + int enc_after); /* * Expand an AES key using the crypto extensions if supported and usable or @@ -139,7 +143,6 @@ EXPORT_SYMBOL_NS_GPL(neon_aes_xts_encrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(neon_aes_xts_decrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); -EXPORT_SYMBOL_NS_GPL(neon_aes_mac_update, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_encrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_decrypt, "CRYPTO_INTERNAL"); @@ -153,6 +156,8 @@ EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); +#endif +#if IS_MODULE(CONFIG_CRYPTO_AES_ARM64_CE_CCM) EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL"); #endif @@ -184,11 +189,48 @@ static void aes_decrypt_arch(const struct aes_key *key, } } +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) +#define aes_cbcmac_blocks_arch aes_cbcmac_blocks_arch +static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, + bool enc_after) +{ + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + static_branch_likely(&have_neon) && likely(may_use_simd())) { + do { + size_t rem; + + scoped_ksimd() { + if (static_branch_likely(&have_aes)) + rem = ce_aes_mac_update( + data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + else + rem = neon_aes_mac_update( + data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + } + data += (nblocks - rem) * AES_BLOCK_SIZE; + nblocks = rem; + enc_before = false; + } while (nblocks); + return true; + } + return false; +} +#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ + #ifdef CONFIG_KERNEL_MODE_NEON #define aes_mod_init_arch aes_mod_init_arch static void aes_mod_init_arch(void) { - if (cpu_have_named_feature(AES)) - static_branch_enable(&have_aes); + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_neon); + if (cpu_have_named_feature(AES)) + static_branch_enable(&have_aes); + } } #endif /* CONFIG_KERNEL_MODE_NEON */ From a348fd1f6eee5b8f5bf159c9d95d35cc54d17699 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Feb 2026 13:34:52 -0800 Subject: [PATCH 06/66] lib/crypto: tests: Add KUnit tests for CBC-based MACs Add a KUnit test suite for the AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC library functions. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260218213501.136844-7-ebiggers@kernel.org Link: https://lore.kernel.org/r/20260306001917.24105-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/.kunitconfig | 3 + lib/crypto/tests/Kconfig | 9 + lib/crypto/tests/Makefile | 1 + lib/crypto/tests/aes-cmac-testvecs.h | 181 ++++++++++++++++++++ lib/crypto/tests/aes_cbc_macs_kunit.c | 228 ++++++++++++++++++++++++++ scripts/crypto/gen-hash-testvecs.py | 31 +++- 6 files changed, 450 insertions(+), 3 deletions(-) create mode 100644 lib/crypto/tests/aes-cmac-testvecs.h create mode 100644 lib/crypto/tests/aes_cbc_macs_kunit.c diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig index 6b2ce28ae509..8cfd213bded9 100644 --- a/lib/crypto/.kunitconfig +++ b/lib/crypto/.kunitconfig @@ -5,8 +5,10 @@ CONFIG_KUNIT=y # since they are hidden symbols. CONFIG_CRYPTO=y CONFIG_CRYPTO_ADIANTUM=y +CONFIG_CRYPTO_AES=y CONFIG_CRYPTO_BLAKE2B=y CONFIG_CRYPTO_CHACHA20POLY1305=y +CONFIG_CRYPTO_CMAC=y CONFIG_CRYPTO_HCTR2=y CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MLDSA=y @@ -20,6 +22,7 @@ CONFIG_NET=y CONFIG_NETDEVICES=y CONFIG_WIREGUARD=y +CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST=y diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 0de289b429a9..0d71de3da15d 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -1,5 +1,14 @@ # SPDX-License-Identifier: GPL-2.0-or-later +config CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST + tristate "KUnit tests for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_AES_CBC_MACS + default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC message + authentication codes. + config CRYPTO_LIB_BLAKE2B_KUNIT_TEST tristate "KUnit tests for BLAKE2b" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_BLAKE2B diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile index f4262379f56c..f864e0ffbee4 100644 --- a/lib/crypto/tests/Makefile +++ b/lib/crypto/tests/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later +obj-$(CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST) += aes_cbc_macs_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o obj-$(CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST) += curve25519_kunit.o diff --git a/lib/crypto/tests/aes-cmac-testvecs.h b/lib/crypto/tests/aes-cmac-testvecs.h new file mode 100644 index 000000000000..8177862ba5a3 --- /dev/null +++ b/lib/crypto/tests/aes-cmac-testvecs.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py aes-cmac */ + +static const struct { + size_t data_len; + u8 digest[AES_BLOCK_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x9a, 0xeb, 0x94, 0xc1, 0xe9, 0xc1, 0x57, 0x49, + 0x7e, 0xba, 0x66, 0x47, 0x9f, 0x03, 0x2c, 0x5b, + }, + }, + { + .data_len = 1, + .digest = { + 0x52, 0xef, 0x65, 0xda, 0x7b, 0x92, 0x0c, 0x0f, + 0xdd, 0xd6, 0xb9, 0x68, 0x3f, 0xcc, 0x5f, 0xea, + }, + }, + { + .data_len = 2, + .digest = { + 0xc3, 0x95, 0x15, 0xea, 0x16, 0x33, 0xbe, 0x5a, + 0xd4, 0x2c, 0x25, 0x06, 0x15, 0xc6, 0x10, 0x19, + }, + }, + { + .data_len = 3, + .digest = { + 0x82, 0x41, 0x41, 0xd5, 0x33, 0x26, 0x0b, 0xb6, + 0xc8, 0xf7, 0x8d, 0x76, 0x8a, 0xfc, 0x0e, 0xe4, + }, + }, + { + .data_len = 16, + .digest = { + 0x94, 0x09, 0x80, 0x07, 0xba, 0x7c, 0xed, 0xd2, + 0x74, 0x72, 0x30, 0x26, 0xb5, 0x11, 0x64, 0xc1, + }, + }, + { + .data_len = 32, + .digest = { + 0xeb, 0xcf, 0x1e, 0x67, 0x21, 0x64, 0x93, 0xa0, + 0xea, 0xc4, 0xb9, 0x2d, 0x55, 0xc8, 0xac, 0x99, + }, + }, + { + .data_len = 48, + .digest = { + 0xd0, 0xd6, 0xdb, 0xe2, 0x45, 0x98, 0x6a, 0x7a, + 0x5f, 0xd6, 0xcd, 0x9d, 0x12, 0x26, 0x20, 0x87, + }, + }, + { + .data_len = 49, + .digest = { + 0x63, 0x25, 0x3c, 0xe2, 0x2a, 0xfa, 0xe3, 0x1e, + 0x54, 0x10, 0x18, 0x28, 0xc6, 0xb8, 0xcb, 0x58, + }, + }, + { + .data_len = 63, + .digest = { + 0x4d, 0xab, 0xae, 0x99, 0x90, 0x13, 0x3f, 0x4f, + 0x42, 0x0f, 0x19, 0x94, 0xa2, 0x52, 0xfd, 0xaf, + }, + }, + { + .data_len = 64, + .digest = { + 0xf7, 0x49, 0xb9, 0xa7, 0xf9, 0x3e, 0xa0, 0xca, + 0xb2, 0x6c, 0xd7, 0x87, 0x7d, 0x1e, 0xd2, 0xcb, + }, + }, + { + .data_len = 65, + .digest = { + 0x27, 0x2c, 0xb7, 0xc8, 0xdd, 0x26, 0xa9, 0xfe, + 0x37, 0x64, 0x84, 0x38, 0xa5, 0x7e, 0xbc, 0x04, + }, + }, + { + .data_len = 127, + .digest = { + 0xfd, 0x1f, 0x01, 0xa4, 0xea, 0x9b, 0xbd, 0xef, + 0x09, 0x97, 0x57, 0x60, 0x95, 0x23, 0xcc, 0x71, + }, + }, + { + .data_len = 128, + .digest = { + 0x28, 0x21, 0xee, 0x56, 0x9f, 0x38, 0xd6, 0x0e, + 0xe3, 0x22, 0x06, 0x20, 0xad, 0xd8, 0x33, 0x74, + }, + }, + { + .data_len = 129, + .digest = { + 0x07, 0x28, 0x4a, 0x2a, 0xd3, 0x85, 0xa6, 0x87, + 0x5c, 0x01, 0x8c, 0xb9, 0xd3, 0x4b, 0xce, 0x20, + }, + }, + { + .data_len = 256, + .digest = { + 0xe6, 0x12, 0x25, 0x6b, 0xf9, 0x69, 0x4d, 0x5a, + 0x1a, 0xb0, 0xe6, 0x11, 0x46, 0x24, 0x08, 0xdf, + }, + }, + { + .data_len = 511, + .digest = { + 0xce, 0x28, 0x1f, 0x14, 0xb9, 0xcc, 0x7e, 0x1f, + 0xb5, 0x13, 0x2b, 0x45, 0x04, 0x54, 0xe9, 0x5f, + }, + }, + { + .data_len = 513, + .digest = { + 0x63, 0x12, 0xbd, 0x85, 0x60, 0x1b, 0x99, 0x7e, + 0x0a, 0xf7, 0x0f, 0xc1, 0xb5, 0x66, 0xf8, 0x9a, + }, + }, + { + .data_len = 1000, + .digest = { + 0xbd, 0x49, 0x5e, 0x21, 0xc6, 0x58, 0x74, 0x6b, + 0x21, 0xc2, 0x62, 0x6a, 0x15, 0xca, 0x1d, 0x8a, + }, + }, + { + .data_len = 3333, + .digest = { + 0xfe, 0x6b, 0xfa, 0xfc, 0x4c, 0x0b, 0x63, 0x0d, + 0x41, 0x7f, 0xa9, 0xd8, 0xba, 0xe3, 0xce, 0xce, + }, + }, + { + .data_len = 4096, + .digest = { + 0x41, 0x7c, 0xbc, 0x2e, 0x2f, 0xff, 0xdf, 0x09, + 0x31, 0xc5, 0x79, 0x0a, 0x1d, 0x6e, 0x46, 0xec, + }, + }, + { + .data_len = 4128, + .digest = { + 0x6a, 0x9d, 0x86, 0xa8, 0xab, 0xa5, 0xc1, 0xc5, + 0x0d, 0x54, 0xf3, 0x51, 0x10, 0x46, 0x25, 0x5a, + }, + }, + { + .data_len = 4160, + .digest = { + 0x4c, 0x50, 0xff, 0x2a, 0xe9, 0xca, 0x9e, 0x07, + 0x8a, 0x86, 0x67, 0x5e, 0xe5, 0x0a, 0xfd, 0x69, + }, + }, + { + .data_len = 4224, + .digest = { + 0x3a, 0xfa, 0x80, 0x9d, 0x80, 0xe3, 0x1e, 0x95, + 0x53, 0x93, 0x39, 0x17, 0xd3, 0xda, 0x49, 0x15, + }, + }, + { + .data_len = 16384, + .digest = { + 0x48, 0xf4, 0x4f, 0x2d, 0x5d, 0xf2, 0x02, 0xcf, + 0x51, 0x3c, 0x1b, 0x12, 0x80, 0x8f, 0xb0, 0xd6, + }, + }, +}; + +static const u8 hash_testvec_consolidated[AES_BLOCK_SIZE] = { + 0x41, 0xad, 0x25, 0xa1, 0xeb, 0xce, 0x6b, 0x9c, + 0x06, 0xdf, 0x47, 0xc4, 0x3a, 0x59, 0x50, 0x07, +}; diff --git a/lib/crypto/tests/aes_cbc_macs_kunit.c b/lib/crypto/tests/aes_cbc_macs_kunit.c new file mode 100644 index 000000000000..ae3745212f03 --- /dev/null +++ b/lib/crypto/tests/aes_cbc_macs_kunit.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include +#include "aes-cmac-testvecs.h" + +/* + * A fixed key used when presenting AES-CMAC as an unkeyed hash function in + * order to reuse hash-test-template.h. At the beginning of the test suite, + * this is initialized to a key prepared from bytes generated from a fixed seed. + */ +static struct aes_cmac_key test_key; + +static void aes_cmac_init_withtestkey(struct aes_cmac_ctx *ctx) +{ + aes_cmac_init(ctx, &test_key); +} + +static void aes_cmac_withtestkey(const u8 *data, size_t data_len, + u8 out[AES_BLOCK_SIZE]) +{ + aes_cmac(&test_key, data, data_len, out); +} + +#define HASH aes_cmac_withtestkey +#define HASH_CTX aes_cmac_ctx +#define HASH_SIZE AES_BLOCK_SIZE +#define HASH_INIT aes_cmac_init_withtestkey +#define HASH_UPDATE aes_cmac_update +#define HASH_FINAL aes_cmac_final +#include "hash-test-template.h" + +static int aes_cbc_macs_suite_init(struct kunit_suite *suite) +{ + u8 raw_key[AES_KEYSIZE_256]; + int err; + + rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); + err = aes_cmac_preparekey(&test_key, raw_key, sizeof(raw_key)); + if (err) + return err; + return hash_suite_init(suite); +} + +static void aes_cbc_macs_suite_exit(struct kunit_suite *suite) +{ + hash_suite_exit(suite); +} + +/* Verify compatibility of the AES-CMAC implementation with RFC 4493. */ +static void test_aes_cmac_rfc4493(struct kunit *test) +{ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c, + }; + static const struct { + size_t data_len; + const u8 data[40]; + const u8 mac[AES_BLOCK_SIZE]; + } testvecs[] = { + { + /* Example 1 from RFC 4493 */ + .data_len = 0, + .mac = { + 0xbb, 0x1d, 0x69, 0x29, 0xe9, 0x59, 0x37, 0x28, + 0x7f, 0xa3, 0x7d, 0x12, 0x9b, 0x75, 0x67, 0x46, + }, + + }, + { + /* Example 2 from RFC 4493 */ + .data = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + }, + .data_len = 16, + .mac = { + 0x07, 0x0a, 0x16, 0xb4, 0x6b, 0x4d, 0x41, 0x44, + 0xf7, 0x9b, 0xdd, 0x9d, 0xd0, 0x4a, 0x28, 0x7c, + }, + }, + { + /* Example 3 from RFC 4493 */ + .data = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + }, + .data_len = 40, + .mac = { + 0xdf, 0xa6, 0x67, 0x47, 0xde, 0x9a, 0xe6, 0x30, + 0x30, 0xca, 0x32, 0x61, 0x14, 0x97, 0xc8, 0x27, + }, + }, + }; + struct aes_cmac_key key; + int err; + + err = aes_cmac_preparekey(&key, raw_key, sizeof(raw_key)); + KUNIT_ASSERT_EQ(test, err, 0); + + for (size_t i = 0; i < ARRAY_SIZE(testvecs); i++) { + u8 mac[AES_BLOCK_SIZE]; + + aes_cmac(&key, testvecs[i].data, testvecs[i].data_len, mac); + KUNIT_ASSERT_MEMEQ(test, mac, testvecs[i].mac, AES_BLOCK_SIZE); + } +} + +/* + * Verify compatibility of the AES-XCBC-MAC implementation with RFC 3566. + * + * Additional AES-XCBC-MAC tests are not necessary, since the AES-XCBC-MAC + * implementation is well covered by the AES-CMAC tests already. Only the key + * preparation function differs; the rest of the code is shared. + */ +static void test_aes_xcbcmac_rfc3566(struct kunit *test) +{ + struct aes_cmac_key key; + /* AES-XCBC-MAC Test Case #4 from RFC 3566 */ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + }; + static const u8 message[20] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, + 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + }; + static const u8 expected_mac[AES_BLOCK_SIZE] = { + 0x47, 0xf5, 0x1b, 0x45, 0x64, 0x96, 0x62, 0x15, + 0xb8, 0x98, 0x5c, 0x63, 0x05, 0x5e, 0xd3, 0x08, + }; + u8 actual_mac[AES_BLOCK_SIZE]; + + aes_xcbcmac_preparekey(&key, raw_key); + aes_cmac(&key, message, sizeof(message), actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, AES_BLOCK_SIZE); +} + +static void test_aes_cbcmac_rfc3610(struct kunit *test) +{ + /* + * The following AES-CBC-MAC test vector is extracted from RFC 3610 + * Packet Vector #11. It required some rearrangement to get the actual + * input to AES-CBC-MAC from the values given. + */ + static const u8 raw_key[AES_KEYSIZE_128] = { + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + }; + const size_t unpadded_data_len = 52; + static const u8 data[64] = { + /* clang-format off */ + /* CCM header */ + 0x61, 0x00, 0x00, 0x00, 0x0d, 0x0c, 0x0b, 0x0a, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0x00, 0x14, + /* CCM additional authentication blocks */ + 0x00, 0x0c, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x00, 0x00, + /* CCM message blocks */ + 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, + 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, + 0x1c, 0x1d, 0x1e, 0x1f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* clang-format on */ + }; + static const u8 expected_mac[AES_BLOCK_SIZE] = { + 0x6b, 0x5e, 0x24, 0x34, 0x12, 0xcc, 0xc2, 0xad, + 0x6f, 0x1b, 0x11, 0xc3, 0xa1, 0xa9, 0xd8, 0xbc, + }; + struct aes_enckey key; + struct aes_cbcmac_ctx ctx; + u8 actual_mac[AES_BLOCK_SIZE]; + int err; + + err = aes_prepareenckey(&key, raw_key, sizeof(raw_key)); + KUNIT_ASSERT_EQ(test, err, 0); + + /* + * Trailing zeroes should not affect the CBC-MAC value, up to the next + * AES block boundary. + */ + for (size_t data_len = unpadded_data_len; data_len <= sizeof(data); + data_len++) { + aes_cbcmac_init(&ctx, &key); + aes_cbcmac_update(&ctx, data, data_len); + aes_cbcmac_final(&ctx, actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, + AES_BLOCK_SIZE); + + /* Incremental computations should produce the same result. */ + for (size_t part1_len = 0; part1_len <= data_len; part1_len++) { + aes_cbcmac_init(&ctx, &key); + aes_cbcmac_update(&ctx, data, part1_len); + aes_cbcmac_update(&ctx, &data[part1_len], + data_len - part1_len); + aes_cbcmac_final(&ctx, actual_mac); + KUNIT_ASSERT_MEMEQ(test, actual_mac, expected_mac, + AES_BLOCK_SIZE); + } + } +} + +static struct kunit_case aes_cbc_macs_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(test_aes_cmac_rfc4493), + KUNIT_CASE(test_aes_xcbcmac_rfc3566), + KUNIT_CASE(test_aes_cbcmac_rfc3610), + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite aes_cbc_macs_test_suite = { + .name = "aes_cbc_macs", + .test_cases = aes_cbc_macs_test_cases, + .suite_init = aes_cbc_macs_suite_init, + .suite_exit = aes_cbc_macs_suite_exit, +}; +kunit_test_suite(aes_cbc_macs_test_suite); + +MODULE_DESCRIPTION( + "KUnit tests and benchmark for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); +MODULE_LICENSE("GPL"); diff --git a/scripts/crypto/gen-hash-testvecs.py b/scripts/crypto/gen-hash-testvecs.py index 8eeb650fcada..34b7c48f3456 100755 --- a/scripts/crypto/gen-hash-testvecs.py +++ b/scripts/crypto/gen-hash-testvecs.py @@ -3,8 +3,12 @@ # # Script that generates test vectors for the given hash function. # +# Requires that python-cryptography be installed. +# # Copyright 2025 Google LLC +import cryptography.hazmat.primitives.ciphers +import cryptography.hazmat.primitives.cmac import hashlib import hmac import sys @@ -24,6 +28,20 @@ def rand_bytes(length): out.append((seed >> 16) % 256) return bytes(out) +AES_256_KEY_SIZE = 32 + +# AES-CMAC. Just wraps the implementation from python-cryptography. +class AesCmac: + def __init__(self, key): + aes = cryptography.hazmat.primitives.ciphers.algorithms.AES(key) + self.cmac = cryptography.hazmat.primitives.cmac.CMAC(aes) + + def update(self, data): + self.cmac.update(data) + + def digest(self): + return self.cmac.finalize() + POLY1305_KEY_SIZE = 32 # A straightforward, unoptimized implementation of Poly1305. @@ -80,9 +98,12 @@ class Polyval: return self.acc.to_bytes(16, byteorder='little') def hash_init(alg): + # The keyed hash functions are assigned a fixed random key here, to present + # them as unkeyed hash functions. This allows all the test cases for + # unkeyed hash functions to work on them. + if alg == 'aes-cmac': + return AesCmac(rand_bytes(AES_256_KEY_SIZE)) if alg == 'poly1305': - # Use a fixed random key here, to present Poly1305 as an unkeyed hash. - # This allows all the test cases for unkeyed hashes to work on Poly1305. return Poly1305(rand_bytes(POLY1305_KEY_SIZE)) if alg == 'polyval': return Polyval(rand_bytes(POLYVAL_BLOCK_SIZE)) @@ -116,6 +137,8 @@ def print_c_struct_u8_array_field(name, value): print('\t\t},') def alg_digest_size_const(alg): + if alg == 'aes-cmac': + return 'AES_BLOCK_SIZE' if alg.startswith('blake2'): return f'{alg.upper()}_HASH_SIZE' return f"{alg.upper().replace('-', '_')}_DIGEST_SIZE" @@ -252,7 +275,9 @@ if len(sys.argv) != 2: alg = sys.argv[1] print('/* SPDX-License-Identifier: GPL-2.0-or-later */') print(f'/* This file was generated by: {sys.argv[0]} {" ".join(sys.argv[1:])} */') -if alg.startswith('blake2'): +if alg == 'aes-cmac': + gen_unkeyed_testvecs(alg) +elif alg.startswith('blake2'): gen_unkeyed_testvecs(alg) gen_additional_blake2_testvecs(alg) elif alg == 'nh': From 2505f9157ebf2bbdb7b1c0ff1cb7274e651ab028 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Feb 2026 13:34:53 -0800 Subject: [PATCH 07/66] lib/crypto: aes: Add FIPS self-test for CMAC Add a FIPS cryptographic algorithm self-test for AES-CMAC to fulfill the self-test requirement when this code is built into a FIPS 140 cryptographic module. This provides parity with the traditional crypto API, which uses crypto/testmgr.c to meet the FIPS self-test requirement. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260218213501.136844-8-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/aes.c | 35 ++++++++++++++++++++++++++--- lib/crypto/fips.h | 5 +++++ scripts/crypto/gen-fips-testvecs.py | 10 +++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c index 39deae6105c0..ca733f15b2a8 100644 --- a/lib/crypto/aes.c +++ b/lib/crypto/aes.c @@ -12,6 +12,7 @@ #include #include #include +#include "fips.h" static const u8 ____cacheline_aligned aes_sbox[] = { 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, @@ -708,12 +709,41 @@ void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE]) memzero_explicit(ctx, sizeof(*ctx)); } EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL"); -#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ -#ifdef aes_mod_init_arch +/* + * FIPS cryptographic algorithm self-test for AES-CMAC. As per the FIPS 140-3 + * Implementation Guidance, a cryptographic algorithm self-test for at least one + * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes + * is implemented. This fulfills that requirement via AES-CMAC. + * + * This is just for FIPS. The full tests are in the KUnit test suite. + */ +static void __init aes_cmac_fips_test(void) +{ + struct aes_cmac_key key; + u8 mac[AES_BLOCK_SIZE]; + + if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != + 0) + panic("aes: CMAC FIPS self-test failed (preparekey)\n"); + aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac); + if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0) + panic("aes: CMAC FIPS self-test failed (wrong MAC)\n"); + memzero_explicit(&key, sizeof(key)); +} +#else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ +static inline void aes_cmac_fips_test(void) +{ +} +#endif /* !CONFIG_CRYPTO_LIB_AES_CBC_MACS */ + static int __init aes_mod_init(void) { +#ifdef aes_mod_init_arch aes_mod_init_arch(); +#endif + if (fips_enabled) + aes_cmac_fips_test(); return 0; } subsys_initcall(aes_mod_init); @@ -722,7 +752,6 @@ static void __exit aes_mod_exit(void) { } module_exit(aes_mod_exit); -#endif MODULE_DESCRIPTION("AES block cipher"); MODULE_AUTHOR("Ard Biesheuvel "); diff --git a/lib/crypto/fips.h b/lib/crypto/fips.h index 023410c2e0db..9fc49747db64 100644 --- a/lib/crypto/fips.h +++ b/lib/crypto/fips.h @@ -43,3 +43,8 @@ static const u8 fips_test_sha3_256_value[] __initconst __maybe_unused = { 0xba, 0x9b, 0xb6, 0xaa, 0x32, 0xa7, 0x97, 0x00, 0x98, 0xdb, 0xff, 0xe7, 0xc6, 0xde, 0xb5, 0x82, }; + +static const u8 fips_test_aes_cmac_value[] __initconst __maybe_unused = { + 0xc5, 0x88, 0x28, 0x55, 0xd7, 0x2c, 0x00, 0xb6, + 0x6a, 0xa7, 0xfc, 0x82, 0x90, 0x81, 0xcf, 0x18, +}; diff --git a/scripts/crypto/gen-fips-testvecs.py b/scripts/crypto/gen-fips-testvecs.py index db873f88619a..9f18bcb97412 100755 --- a/scripts/crypto/gen-fips-testvecs.py +++ b/scripts/crypto/gen-fips-testvecs.py @@ -3,8 +3,12 @@ # # Script that generates lib/crypto/fips.h # +# Requires that python-cryptography be installed. +# # Copyright 2025 Google LLC +import cryptography.hazmat.primitives.ciphers +import cryptography.hazmat.primitives.cmac import hashlib import hmac @@ -34,3 +38,9 @@ for alg in 'sha1', 'sha256', 'sha512': print_static_u8_array_definition(f'fips_test_sha3_256_value', hashlib.sha3_256(fips_test_data).digest()) + +aes = cryptography.hazmat.primitives.ciphers.algorithms.AES(fips_test_key) +aes_cmac = cryptography.hazmat.primitives.cmac.CMAC(aes) +aes_cmac.update(fips_test_data) +print_static_u8_array_definition('fips_test_aes_cmac_value', + aes_cmac.finalize()) From c9de7246d5422bbea5374a402e761cf7725a263e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Feb 2026 13:35:00 -0800 Subject: [PATCH 08/66] wifi: mac80211: Use AES-CMAC library in ieee80211_aes_cmac() Now that AES-CMAC has a library API, convert the mac80211 AES-CMAC packet authentication code to use it instead of a "cmac(aes)" crypto_shash. This has multiple benefits, such as: - It's faster. The AES-CMAC code is now called directly, without unnecessary overhead such as indirect calls. - MAC calculation can no longer fail. - The AES-CMAC key struct is now a fixed size, allowing it to be embedded directly into 'struct ieee80211_key' rather than using a separate allocation. Note that although this increases the size of the 'u.cmac' field of 'struct ieee80211_key', it doesn't cause it to exceed the size of the largest variant of the union 'u'. Therefore, the size of 'struct ieee80211_key' itself is unchanged. Acked-by: Johannes Berg Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260218213501.136844-15-ebiggers@kernel.org Signed-off-by: Eric Biggers --- net/mac80211/Kconfig | 1 + net/mac80211/aes_cmac.c | 65 ++++++++--------------------------------- net/mac80211/aes_cmac.h | 12 +++----- net/mac80211/key.c | 11 ++----- net/mac80211/key.h | 3 +- net/mac80211/wpa.c | 13 +++------ 6 files changed, 26 insertions(+), 79 deletions(-) diff --git a/net/mac80211/Kconfig b/net/mac80211/Kconfig index cf0f7780fb10..0afbe4f4f976 100644 --- a/net/mac80211/Kconfig +++ b/net/mac80211/Kconfig @@ -3,6 +3,7 @@ config MAC80211 tristate "Generic IEEE 802.11 Networking Stack (mac80211)" depends on CFG80211 select CRYPTO + select CRYPTO_LIB_AES_CBC_MACS select CRYPTO_LIB_ARC4 select CRYPTO_AES select CRYPTO_CCM diff --git a/net/mac80211/aes_cmac.c b/net/mac80211/aes_cmac.c index 0827965455dc..55b674ad7d7a 100644 --- a/net/mac80211/aes_cmac.c +++ b/net/mac80211/aes_cmac.c @@ -7,10 +7,9 @@ #include #include -#include #include #include -#include +#include #include #include "key.h" @@ -20,65 +19,25 @@ static const u8 zero[IEEE80211_CMAC_256_MIC_LEN]; -int ieee80211_aes_cmac(struct crypto_shash *tfm, const u8 *aad, - const u8 *data, size_t data_len, u8 *mic, - unsigned int mic_len) +void ieee80211_aes_cmac(const struct aes_cmac_key *key, const u8 *aad, + const u8 *data, size_t data_len, u8 *mic, + unsigned int mic_len) { - int err; - SHASH_DESC_ON_STACK(desc, tfm); + struct aes_cmac_ctx ctx; u8 out[AES_BLOCK_SIZE]; const __le16 *fc; - desc->tfm = tfm; - - err = crypto_shash_init(desc); - if (err) - return err; - err = crypto_shash_update(desc, aad, AAD_LEN); - if (err) - return err; + aes_cmac_init(&ctx, key); + aes_cmac_update(&ctx, aad, AAD_LEN); fc = (const __le16 *)aad; if (ieee80211_is_beacon(*fc)) { /* mask Timestamp field to zero */ - err = crypto_shash_update(desc, zero, 8); - if (err) - return err; - err = crypto_shash_update(desc, data + 8, - data_len - 8 - mic_len); - if (err) - return err; + aes_cmac_update(&ctx, zero, 8); + aes_cmac_update(&ctx, data + 8, data_len - 8 - mic_len); } else { - err = crypto_shash_update(desc, data, data_len - mic_len); - if (err) - return err; + aes_cmac_update(&ctx, data, data_len - mic_len); } - err = crypto_shash_finup(desc, zero, mic_len, out); - if (err) - return err; + aes_cmac_update(&ctx, zero, mic_len); + aes_cmac_final(&ctx, out); memcpy(mic, out, mic_len); - - return 0; -} - -struct crypto_shash *ieee80211_aes_cmac_key_setup(const u8 key[], - size_t key_len) -{ - struct crypto_shash *tfm; - - tfm = crypto_alloc_shash("cmac(aes)", 0, 0); - if (!IS_ERR(tfm)) { - int err = crypto_shash_setkey(tfm, key, key_len); - - if (err) { - crypto_free_shash(tfm); - return ERR_PTR(err); - } - } - - return tfm; -} - -void ieee80211_aes_cmac_key_free(struct crypto_shash *tfm) -{ - crypto_free_shash(tfm); } diff --git a/net/mac80211/aes_cmac.h b/net/mac80211/aes_cmac.h index 5f971a8298cb..c7a6df47b327 100644 --- a/net/mac80211/aes_cmac.h +++ b/net/mac80211/aes_cmac.h @@ -6,14 +6,10 @@ #ifndef AES_CMAC_H #define AES_CMAC_H -#include -#include +#include -struct crypto_shash *ieee80211_aes_cmac_key_setup(const u8 key[], - size_t key_len); -int ieee80211_aes_cmac(struct crypto_shash *tfm, const u8 *aad, - const u8 *data, size_t data_len, u8 *mic, - unsigned int mic_len); -void ieee80211_aes_cmac_key_free(struct crypto_shash *tfm); +void ieee80211_aes_cmac(const struct aes_cmac_key *key, const u8 *aad, + const u8 *data, size_t data_len, u8 *mic, + unsigned int mic_len); #endif /* AES_CMAC_H */ diff --git a/net/mac80211/key.c b/net/mac80211/key.c index 04c8809173d7..4b8965633df3 100644 --- a/net/mac80211/key.c +++ b/net/mac80211/key.c @@ -690,10 +690,9 @@ ieee80211_key_alloc(u32 cipher, int idx, size_t key_len, * Initialize AES key state here as an optimization so that * it does not need to be initialized for every packet. */ - key->u.aes_cmac.tfm = - ieee80211_aes_cmac_key_setup(key_data, key_len); - if (IS_ERR(key->u.aes_cmac.tfm)) { - err = PTR_ERR(key->u.aes_cmac.tfm); + err = aes_cmac_preparekey(&key->u.aes_cmac.key, key_data, + key_len); + if (err) { kfree(key); return ERR_PTR(err); } @@ -750,10 +749,6 @@ static void ieee80211_key_free_common(struct ieee80211_key *key) case WLAN_CIPHER_SUITE_CCMP_256: ieee80211_aes_key_free(key->u.ccmp.tfm); break; - case WLAN_CIPHER_SUITE_AES_CMAC: - case WLAN_CIPHER_SUITE_BIP_CMAC_256: - ieee80211_aes_cmac_key_free(key->u.aes_cmac.tfm); - break; case WLAN_CIPHER_SUITE_BIP_GMAC_128: case WLAN_CIPHER_SUITE_BIP_GMAC_256: ieee80211_aes_gmac_key_free(key->u.aes_gmac.tfm); diff --git a/net/mac80211/key.h b/net/mac80211/key.h index 1fa0f4f78962..826e4e9387c5 100644 --- a/net/mac80211/key.h +++ b/net/mac80211/key.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -93,7 +94,7 @@ struct ieee80211_key { } ccmp; struct { u8 rx_pn[IEEE80211_CMAC_PN_LEN]; - struct crypto_shash *tfm; + struct aes_cmac_key key; u32 replays; /* dot11RSNAStatsCMACReplays */ u32 icverrors; /* dot11RSNAStatsCMACICVErrors */ } aes_cmac; diff --git a/net/mac80211/wpa.c b/net/mac80211/wpa.c index fdf98c21d32c..59324b367bdd 100644 --- a/net/mac80211/wpa.c +++ b/net/mac80211/wpa.c @@ -872,11 +872,8 @@ ieee80211_crypto_aes_cmac_encrypt(struct ieee80211_tx_data *tx, bip_aad(skb, aad); - if (ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad, - skb->data + 24, skb->len - 24, - mmie->mic, mic_len)) - return TX_DROP; - + ieee80211_aes_cmac(&key->u.aes_cmac.key, aad, skb->data + 24, + skb->len - 24, mmie->mic, mic_len); return TX_CONTINUE; } @@ -918,10 +915,8 @@ ieee80211_crypto_aes_cmac_decrypt(struct ieee80211_rx_data *rx, if (!(status->flag & RX_FLAG_DECRYPTED)) { /* hardware didn't decrypt/verify MIC */ bip_aad(skb, aad); - if (ieee80211_aes_cmac(key->u.aes_cmac.tfm, aad, - skb->data + 24, skb->len - 24, - mic, mic_len)) - return RX_DROP_U_DECRYPT_FAIL; + ieee80211_aes_cmac(&key->u.aes_cmac.key, aad, skb->data + 24, + skb->len - 24, mic, mic_len); if (crypto_memneq(mic, mmie->mic, mic_len)) { key->u.aes_cmac.icverrors++; return RX_DROP_U_MIC_FAIL; From 5e07ce466356d9833757b2d6c0380b6ee7b11a77 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Feb 2026 13:35:01 -0800 Subject: [PATCH 09/66] wifi: mac80211: Use AES-CMAC library in aes_s2v() Now that AES-CMAC has a library API, convert aes_s2v() to use it instead of a "cmac(aes)" crypto_shash. The result is faster and simpler code. It's also more reliable, since with the library the only step that can fail is preparing the key. In contrast, crypto_shash_digest(), crypto_shash_init(), crypto_shash_update(), and crypto_shash_final() could all fail and return an errno value. aes_s2v() ignored these errors, which was a bug. So that bug is fixed as well. As part of this, change the prototype of aes_s2v() to take the raw key directly instead of a prepared key. Its only two callers prepare a key for each call, so it might as well be done directly in aes_s2v(). Since this removes the last dependency on the "cmac(aes)" crypto_shash from mac80211, also remove the 'select CRYPTO_CMAC'. Acked-by: Johannes Berg Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260218213501.136844-16-ebiggers@kernel.org Signed-off-by: Eric Biggers --- net/mac80211/Kconfig | 1 - net/mac80211/fils_aead.c | 48 ++++++++++++++-------------------------- 2 files changed, 17 insertions(+), 32 deletions(-) diff --git a/net/mac80211/Kconfig b/net/mac80211/Kconfig index 0afbe4f4f976..d6bc295e23a1 100644 --- a/net/mac80211/Kconfig +++ b/net/mac80211/Kconfig @@ -8,7 +8,6 @@ config MAC80211 select CRYPTO_AES select CRYPTO_CCM select CRYPTO_GCM - select CRYPTO_CMAC select CRC32 help This option enables the hardware independent IEEE 802.11 diff --git a/net/mac80211/fils_aead.c b/net/mac80211/fils_aead.c index 912c46f74d24..d2f4a17eab99 100644 --- a/net/mac80211/fils_aead.c +++ b/net/mac80211/fils_aead.c @@ -4,13 +4,11 @@ * Copyright 2016, Qualcomm Atheros, Inc. */ -#include -#include +#include #include #include #include "ieee80211_i.h" -#include "aes_cmac.h" #include "fils_aead.h" static void gf_mulx(u8 *pad) @@ -22,31 +20,35 @@ static void gf_mulx(u8 *pad) put_unaligned_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0), pad + 8); } -static int aes_s2v(struct crypto_shash *tfm, +static int aes_s2v(const u8 *in_key, size_t key_len, size_t num_elem, const u8 *addr[], size_t len[], u8 *v) { u8 d[AES_BLOCK_SIZE], tmp[AES_BLOCK_SIZE] = {}; - SHASH_DESC_ON_STACK(desc, tfm); + struct aes_cmac_key key; + struct aes_cmac_ctx ctx; size_t i; + int res; - desc->tfm = tfm; + res = aes_cmac_preparekey(&key, in_key, key_len); + if (res) + return res; /* D = AES-CMAC(K, ) */ - crypto_shash_digest(desc, tmp, AES_BLOCK_SIZE, d); + aes_cmac(&key, tmp, AES_BLOCK_SIZE, d); for (i = 0; i < num_elem - 1; i++) { /* D = dbl(D) xor AES_CMAC(K, Si) */ gf_mulx(d); /* dbl */ - crypto_shash_digest(desc, addr[i], len[i], tmp); + aes_cmac(&key, addr[i], len[i], tmp); crypto_xor(d, tmp, AES_BLOCK_SIZE); } - crypto_shash_init(desc); + aes_cmac_init(&ctx, &key); if (len[i] >= AES_BLOCK_SIZE) { /* len(Sn) >= 128 */ /* T = Sn xorend D */ - crypto_shash_update(desc, addr[i], len[i] - AES_BLOCK_SIZE); + aes_cmac_update(&ctx, addr[i], len[i] - AES_BLOCK_SIZE); crypto_xor(d, addr[i] + len[i] - AES_BLOCK_SIZE, AES_BLOCK_SIZE); } else { @@ -57,8 +59,10 @@ static int aes_s2v(struct crypto_shash *tfm, d[len[i]] ^= 0x80; } /* V = AES-CMAC(K, T) */ - crypto_shash_finup(desc, d, AES_BLOCK_SIZE, v); + aes_cmac_update(&ctx, d, AES_BLOCK_SIZE); + aes_cmac_final(&ctx, v); + memzero_explicit(&key, sizeof(key)); return 0; } @@ -69,7 +73,6 @@ static int aes_siv_encrypt(const u8 *key, size_t key_len, size_t len[], u8 *out) { u8 v[AES_BLOCK_SIZE]; - struct crypto_shash *tfm; struct crypto_skcipher *tfm2; struct skcipher_request *req; int res; @@ -83,15 +86,7 @@ static int aes_siv_encrypt(const u8 *key, size_t key_len, num_elem++; /* S2V */ - - tfm = crypto_alloc_shash("cmac(aes)", 0, 0); - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - /* K1 for S2V */ - res = crypto_shash_setkey(tfm, key, key_len); - if (!res) - res = aes_s2v(tfm, num_elem, addr, len, v); - crypto_free_shash(tfm); + res = aes_s2v(key /* K1 */, key_len, num_elem, addr, len, v); if (res) return res; @@ -146,7 +141,6 @@ static int aes_siv_decrypt(const u8 *key, size_t key_len, size_t num_elem, const u8 *addr[], size_t len[], u8 *out) { - struct crypto_shash *tfm; struct crypto_skcipher *tfm2; struct skcipher_request *req; struct scatterlist src[1], dst[1]; @@ -198,15 +192,7 @@ static int aes_siv_decrypt(const u8 *key, size_t key_len, return res; /* S2V */ - - tfm = crypto_alloc_shash("cmac(aes)", 0, 0); - if (IS_ERR(tfm)) - return PTR_ERR(tfm); - /* K1 for S2V */ - res = crypto_shash_setkey(tfm, key, key_len); - if (!res) - res = aes_s2v(tfm, num_elem, addr, len, check); - crypto_free_shash(tfm); + res = aes_s2v(key /* K1 */, key_len, num_elem, addr, len, check); if (res) return res; if (memcmp(check, frame_iv, AES_BLOCK_SIZE) != 0) From ce260754bb435aea18e6a1a1ce3759249013f5a4 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 25 Feb 2026 17:00:05 -0800 Subject: [PATCH 10/66] crypto: jitterentropy - Use SHA-3 library Make the jitterentropy RNG use the SHA-3 library API instead of crypto_shash. This ends up being quite a bit simpler, as various dynamic allocations and error checks become unnecessary. Signed-off-by: David Howells Co-developed-by: Eric Biggers Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260226010005.43528-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 2 +- crypto/jitterentropy-kcapi.c | 116 +++++++++-------------------------- crypto/jitterentropy.c | 25 ++++---- crypto/jitterentropy.h | 19 +++--- 4 files changed, 53 insertions(+), 109 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index c977d40f906d..b8608ef6823b 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1177,8 +1177,8 @@ endif # if CRYPTO_DRBG_MENU config CRYPTO_JITTERENTROPY tristate "CPU Jitter Non-Deterministic RNG (Random Number Generator)" + select CRYPTO_LIB_SHA3 select CRYPTO_RNG - select CRYPTO_SHA3 help CPU Jitter RNG (Random Number Generator) from the Jitterentropy library diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 7c880cf34c52..4ad729357441 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -37,7 +37,6 @@ * DAMAGE. */ -#include #include #include #include @@ -48,8 +47,6 @@ #include "jitterentropy.h" -#define JENT_CONDITIONING_HASH "sha3-256" - /*************************************************************************** * Helper function ***************************************************************************/ @@ -101,22 +98,14 @@ void jent_get_nstime(__u64 *out) jent_raw_hires_entropy_store(tmp); } -int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, - unsigned int addtl_len, __u64 hash_loop_cnt, - unsigned int stuck) +void jent_hash_time(struct sha3_ctx *hash_state, __u64 time, u8 *addtl, + unsigned int addtl_len, __u64 hash_loop_cnt, + unsigned int stuck) { - struct shash_desc *hash_state_desc = (struct shash_desc *)hash_state; - SHASH_DESC_ON_STACK(desc, hash_state_desc->tfm); + struct sha3_ctx tmp_state; /* zeroized by sha3_final() */ u8 intermediary[SHA3_256_DIGEST_SIZE]; __u64 j = 0; - int ret; - desc->tfm = hash_state_desc->tfm; - - if (sizeof(intermediary) != crypto_shash_digestsize(desc->tfm)) { - pr_warn_ratelimited("Unexpected digest size\n"); - return -EINVAL; - } kmsan_unpoison_memory(intermediary, sizeof(intermediary)); /* @@ -130,24 +119,20 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, * * Note, it does not matter which or how much data you inject, we are * interested in one Keccack1600 compression operation performed with - * the crypto_shash_final. + * the sha3_final. */ for (j = 0; j < hash_loop_cnt; j++) { - ret = crypto_shash_init(desc) ?: - crypto_shash_update(desc, intermediary, - sizeof(intermediary)) ?: - crypto_shash_finup(desc, addtl, addtl_len, intermediary); - if (ret) - goto err; + sha3_256_init(&tmp_state); + sha3_update(&tmp_state, intermediary, sizeof(intermediary)); + sha3_update(&tmp_state, addtl, addtl_len); + sha3_final(&tmp_state, intermediary); } /* * Inject the data from the previous loop into the pool. This data is * not considered to contain any entropy, but it stirs the pool a bit. */ - ret = crypto_shash_update(hash_state_desc, intermediary, sizeof(intermediary)); - if (ret) - goto err; + sha3_update(hash_state, intermediary, sizeof(intermediary)); /* * Insert the time stamp into the hash context representing the pool. @@ -162,30 +147,24 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, time = 0; } - ret = crypto_shash_update(hash_state_desc, (u8 *)&time, sizeof(__u64)); - -err: - shash_desc_zero(desc); + sha3_update(hash_state, (u8 *)&time, sizeof(__u64)); memzero_explicit(intermediary, sizeof(intermediary)); - - return ret; } -int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len) +void jent_read_random_block(struct sha3_ctx *hash_state, char *dst, + unsigned int dst_len) { - struct shash_desc *hash_state_desc = (struct shash_desc *)hash_state; u8 jent_block[SHA3_256_DIGEST_SIZE]; - /* Obtain data from entropy pool and re-initialize it */ - int ret = crypto_shash_final(hash_state_desc, jent_block) ?: - crypto_shash_init(hash_state_desc) ?: - crypto_shash_update(hash_state_desc, jent_block, - sizeof(jent_block)); - if (!ret && dst_len) + /* Obtain data from entropy pool and re-initialize it */ + sha3_final(hash_state, jent_block); + sha3_256_init(hash_state); + sha3_update(hash_state, jent_block, sizeof(jent_block)); + + if (dst_len) memcpy(dst, jent_block, dst_len); memzero_explicit(jent_block, sizeof(jent_block)); - return ret; } /*************************************************************************** @@ -195,8 +174,7 @@ int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len) struct jitterentropy { spinlock_t jent_lock; struct rand_data *entropy_collector; - struct crypto_shash *tfm; - struct shash_desc *sdesc; + struct sha3_ctx hash_state; }; static void jent_kcapi_cleanup(struct crypto_tfm *tfm) @@ -205,15 +183,7 @@ static void jent_kcapi_cleanup(struct crypto_tfm *tfm) spin_lock(&rng->jent_lock); - if (rng->sdesc) { - shash_desc_zero(rng->sdesc); - kfree(rng->sdesc); - } - rng->sdesc = NULL; - - if (rng->tfm) - crypto_free_shash(rng->tfm); - rng->tfm = NULL; + memzero_explicit(&rng->hash_state, sizeof(rng->hash_state)); if (rng->entropy_collector) jent_entropy_collector_free(rng->entropy_collector); @@ -224,34 +194,15 @@ static void jent_kcapi_cleanup(struct crypto_tfm *tfm) static int jent_kcapi_init(struct crypto_tfm *tfm) { struct jitterentropy *rng = crypto_tfm_ctx(tfm); - struct crypto_shash *hash; - struct shash_desc *sdesc; - int size, ret = 0; + int ret = 0; spin_lock_init(&rng->jent_lock); /* Use SHA3-256 as conditioner */ - hash = crypto_alloc_shash(JENT_CONDITIONING_HASH, 0, 0); - if (IS_ERR(hash)) { - pr_err("Cannot allocate conditioning digest\n"); - return PTR_ERR(hash); - } - rng->tfm = hash; + sha3_256_init(&rng->hash_state); - size = sizeof(struct shash_desc) + crypto_shash_descsize(hash); - sdesc = kmalloc(size, GFP_KERNEL); - if (!sdesc) { - ret = -ENOMEM; - goto err; - } - - sdesc->tfm = hash; - crypto_shash_init(sdesc); - rng->sdesc = sdesc; - - rng->entropy_collector = - jent_entropy_collector_alloc(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, - sdesc); + rng->entropy_collector = jent_entropy_collector_alloc( + CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, &rng->hash_state); if (!rng->entropy_collector) { ret = -ENOMEM; goto err; @@ -326,23 +277,16 @@ static struct rng_alg jent_alg = { static int __init jent_mod_init(void) { - SHASH_DESC_ON_STACK(desc, tfm); - struct crypto_shash *tfm; + struct sha3_ctx hash_state; int ret = 0; jent_testing_init(); - tfm = crypto_alloc_shash(JENT_CONDITIONING_HASH, 0, 0); - if (IS_ERR(tfm)) { - jent_testing_exit(); - return PTR_ERR(tfm); - } + sha3_256_init(&hash_state); - desc->tfm = tfm; - crypto_shash_init(desc); - ret = jent_entropy_init(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, desc, NULL); - shash_desc_zero(desc); - crypto_free_shash(tfm); + ret = jent_entropy_init(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, &hash_state, + NULL); + memzero_explicit(&hash_state, sizeof(hash_state)); if (ret) { /* Handle permanent health test error */ if (fips_enabled) diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index 3f93cdc9a7af..d5832caa8ab3 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -68,7 +68,7 @@ struct rand_data { * of the RNG are marked as SENSITIVE. A user must not * access that information while the RNG executes its loops to * calculate the next random value. */ - void *hash_state; /* SENSITIVE hash state entropy pool */ + struct sha3_ctx *hash_state; /* SENSITIVE hash state entropy pool */ __u64 prev_time; /* SENSITIVE Previous time stamp */ __u64 last_delta; /* SENSITIVE stuck test */ __s64 last_delta2; /* SENSITIVE stuck test */ @@ -417,10 +417,9 @@ static __u64 jent_loop_shuffle(unsigned int bits, unsigned int min) * time [in] time stamp to be injected * stuck [in] Is the time stamp identified as stuck? * - * Output: - * updated hash context in the entropy collector or error code + * Output: updated hash context in the entropy collector */ -static int jent_condition_data(struct rand_data *ec, __u64 time, int stuck) +static void jent_condition_data(struct rand_data *ec, __u64 time, int stuck) { #define SHA3_HASH_LOOP (1<<3) struct { @@ -435,8 +434,8 @@ static int jent_condition_data(struct rand_data *ec, __u64 time, int stuck) ec->apt_base }; - return jent_hash_time(ec->hash_state, time, (u8 *)&addtl, sizeof(addtl), - SHA3_HASH_LOOP, stuck); + jent_hash_time(ec->hash_state, time, (u8 *)&addtl, sizeof(addtl), + SHA3_HASH_LOOP, stuck); } /* @@ -538,8 +537,7 @@ static int jent_measure_jitter(struct rand_data *ec, __u64 *ret_current_delta) stuck = jent_stuck(ec, current_delta); /* Now call the next noise sources which also injects the data */ - if (jent_condition_data(ec, current_delta, stuck)) - stuck = 1; + jent_condition_data(ec, current_delta, stuck); /* return the raw entropy value */ if (ret_current_delta) @@ -597,7 +595,7 @@ static void jent_gen_entropy(struct rand_data *ec) * @return 0 when request is fulfilled or an error * * The following error codes can occur: - * -1 entropy_collector is NULL or the generation failed + * -1 entropy_collector is NULL * -2 Intermittent health failure * -3 Permanent health failure */ @@ -640,8 +638,7 @@ int jent_read_entropy(struct rand_data *ec, unsigned char *data, } tocopy = min(DATA_SIZE_BITS / 8, len); - if (jent_read_random_block(ec->hash_state, p, tocopy)) - return -1; + jent_read_random_block(ec->hash_state, p, tocopy); len -= tocopy; p += tocopy; @@ -656,7 +653,7 @@ int jent_read_entropy(struct rand_data *ec, unsigned char *data, struct rand_data *jent_entropy_collector_alloc(unsigned int osr, unsigned int flags, - void *hash_state) + struct sha3_ctx *hash_state) { struct rand_data *entropy_collector; @@ -704,8 +701,8 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector) jent_zfree(entropy_collector); } -int jent_entropy_init(unsigned int osr, unsigned int flags, void *hash_state, - struct rand_data *p_ec) +int jent_entropy_init(unsigned int osr, unsigned int flags, + struct sha3_ctx *hash_state, struct rand_data *p_ec) { /* * If caller provides an allocated ec, reuse it which implies that the diff --git a/crypto/jitterentropy.h b/crypto/jitterentropy.h index 4c5dbf2a8d8f..5bb15cb33000 100644 --- a/crypto/jitterentropy.h +++ b/crypto/jitterentropy.h @@ -1,24 +1,27 @@ // SPDX-License-Identifier: GPL-2.0-or-later +struct sha3_ctx; extern void *jent_kvzalloc(unsigned int len); extern void jent_kvzfree(void *ptr, unsigned int len); extern void *jent_zalloc(unsigned int len); extern void jent_zfree(void *ptr); extern void jent_get_nstime(__u64 *out); -extern int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, - unsigned int addtl_len, __u64 hash_loop_cnt, - unsigned int stuck); -int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len); +void jent_hash_time(struct sha3_ctx *hash_state, __u64 time, u8 *addtl, + unsigned int addtl_len, __u64 hash_loop_cnt, + unsigned int stuck); +void jent_read_random_block(struct sha3_ctx *hash_state, char *dst, + unsigned int dst_len); struct rand_data; extern int jent_entropy_init(unsigned int osr, unsigned int flags, - void *hash_state, struct rand_data *p_ec); + struct sha3_ctx *hash_state, + struct rand_data *p_ec); extern int jent_read_entropy(struct rand_data *ec, unsigned char *data, unsigned int len); -extern struct rand_data *jent_entropy_collector_alloc(unsigned int osr, - unsigned int flags, - void *hash_state); +extern struct rand_data * +jent_entropy_collector_alloc(unsigned int osr, unsigned int flags, + struct sha3_ctx *hash_state); extern void jent_entropy_collector_free(struct rand_data *entropy_collector); #ifdef CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE From 44b02a14d993d91ae36409a54941ac5a5ad20b44 Mon Sep 17 00:00:00 2001 From: AlanSong-oc Date: Fri, 13 Mar 2026 16:01:50 +0800 Subject: [PATCH 11/66] lib/crypto: x86/sha256: PHE Extensions optimized SHA256 transform function Zhaoxin CPUs have implemented the SHA(Secure Hash Algorithm) as its CPU instructions by PHE(Padlock Hash Engine) Extensions, including XSHA1, XSHA256, XSHA384 and XSHA512 instructions. The instruction specification is available at the following link. (https://gitee.com/openzhaoxin/zhaoxin_specifications/blob/20260227/ZX_Padlock_Reference.pdf) With the help of implementation of SHA in hardware instead of software, can develop applications with higher performance, more security and more flexibility. This patch includes the XSHA256 instruction optimized implementation of SHA-256 transform function. The table below shows the benchmark results before and after applying this patch by using CRYPTO_LIB_BENCHMARK on Zhaoxin KX-7000 platform, highlighting the achieved speedups. +---------+--------------------------+ | | SHA256 | +---------+--------+-----------------+ | Len | Before | After | +---------+--------+-----------------+ | 1* | 2 | 7 (3.50x) | | 16 | 35 | 119 (3.40x) | | 64 | 74 | 280 (3.78x) | | 127 | 99 | 387 (3.91x) | | 128 | 103 | 427 (4.15x) | | 200 | 123 | 537 (4.37x) | | 256 | 128 | 582 (4.55x) | | 511 | 144 | 679 (4.72x) | | 512 | 146 | 714 (4.89x) | | 1024 | 157 | 796 (5.07x) | | 3173 | 167 | 883 (5.28x) | | 4096 | 166 | 876 (5.28x) | | 16384 | 169 | 899 (5.32x) | +---------+--------+-----------------+ *: The length of each data block to be processed by one complete SHA sequence. **: The throughput of processing data blocks, unit is Mb/s. After applying this patch, the SHA256 KUnit test suite passes on Zhaoxin platforms. Detailed test logs are shown below. [ 7.767257] # Subtest: sha256 [ 7.770542] # module: sha256_kunit [ 7.770544] 1..15 [ 7.777383] ok 1 test_hash_test_vectors [ 7.788563] ok 2 test_hash_all_lens_up_to_4096 [ 7.806090] ok 3 test_hash_incremental_updates [ 7.813553] ok 4 test_hash_buffer_overruns [ 7.822384] ok 5 test_hash_overlaps [ 7.829388] ok 6 test_hash_alignment_consistency [ 7.833843] ok 7 test_hash_ctx_zeroization [ 7.915191] ok 8 test_hash_interrupt_context_1 [ 8.362312] ok 9 test_hash_interrupt_context_2 [ 8.401607] ok 10 test_hmac [ 8.415458] ok 11 test_sha256_finup_2x [ 8.419397] ok 12 test_sha256_finup_2x_defaultctx [ 8.424107] ok 13 test_sha256_finup_2x_hugelen [ 8.451289] # benchmark_hash: len=1: 7 MB/s [ 8.465372] # benchmark_hash: len=16: 119 MB/s [ 8.481760] # benchmark_hash: len=64: 280 MB/s [ 8.499344] # benchmark_hash: len=127: 387 MB/s [ 8.515800] # benchmark_hash: len=128: 427 MB/s [ 8.531970] # benchmark_hash: len=200: 537 MB/s [ 8.548241] # benchmark_hash: len=256: 582 MB/s [ 8.564838] # benchmark_hash: len=511: 679 MB/s [ 8.580872] # benchmark_hash: len=512: 714 MB/s [ 8.596858] # benchmark_hash: len=1024: 796 MB/s [ 8.612567] # benchmark_hash: len=3173: 883 MB/s [ 8.628546] # benchmark_hash: len=4096: 876 MB/s [ 8.644482] # benchmark_hash: len=16384: 899 MB/s [ 8.649773] ok 14 benchmark_hash [ 8.655505] ok 15 benchmark_sha256_finup_2x # SKIP not relevant [ 8.659065] # sha256: pass:14 fail:0 skip:1 total:15 [ 8.665276] # Totals: pass:14 fail:0 skip:1 total:15 [ 8.670195] ok 7 sha256 Signed-off-by: AlanSong-oc Link: https://lore.kernel.org/r/20260313080150.9393-3-AlanSong-oc@zhaoxin.com Signed-off-by: Eric Biggers --- lib/crypto/x86/sha256.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib/crypto/x86/sha256.h b/lib/crypto/x86/sha256.h index 38e33b22a092..0ee69d8e39fe 100644 --- a/lib/crypto/x86/sha256.h +++ b/lib/crypto/x86/sha256.h @@ -31,6 +31,27 @@ DEFINE_X86_SHA256_FN(sha256_blocks_avx, sha256_transform_avx); DEFINE_X86_SHA256_FN(sha256_blocks_avx2, sha256_transform_rorx); DEFINE_X86_SHA256_FN(sha256_blocks_ni, sha256_ni_transform); +#define PHE_ALIGNMENT 16 +static void sha256_blocks_phe(struct sha256_block_state *state, + const u8 *data, size_t nblocks) +{ + /* + * On Zhaoxin processors, XSHA256 requires the %rdi register + * in 64-bit mode (or %edi in 32-bit mode) to point to + * a 32-byte, 16-byte-aligned buffer. + */ + u8 buf[32 + PHE_ALIGNMENT - 1]; + u8 *dst = PTR_ALIGN(&buf[0], PHE_ALIGNMENT); + size_t padding = -1; + + memcpy(dst, state, SHA256_DIGEST_SIZE); + asm volatile(".byte 0xf3,0x0f,0xa6,0xd0" /* REP XSHA256 */ + : "+a"(padding), "+c"(nblocks), "+S"(data) + : "D"(dst) + : "memory"); + memcpy(state, dst, SHA256_DIGEST_SIZE); +} + static void sha256_blocks(struct sha256_block_state *state, const u8 *data, size_t nblocks) { @@ -79,6 +100,10 @@ static void sha256_mod_init_arch(void) if (boot_cpu_has(X86_FEATURE_SHA_NI)) { static_call_update(sha256_blocks_x86, sha256_blocks_ni); static_branch_enable(&have_sha_ni); + } else if (IS_ENABLED(CONFIG_CPU_SUP_ZHAOXIN) && + boot_cpu_has(X86_FEATURE_PHE_EN) && + boot_cpu_data.x86 >= 0x07) { + static_call_update(sha256_blocks_x86, sha256_blocks_phe); } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL) && boot_cpu_has(X86_FEATURE_AVX)) { From ed1767442d919f57aaf83d69c33853da2644d902 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 13 Mar 2026 20:59:26 -0700 Subject: [PATCH 12/66] lib/crypto: tests: Introduce CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT For kunit.py to run all the crypto library tests when passed the --alltests option, tools/testing/kunit/configs/all_tests.config needs to enable options that satisfy the test dependencies. This is the same as what lib/crypto/.kunitconfig already does. However, the strategy that lib/crypto/.kunitconfig currently uses to select all the hidden library options isn't going to scale up well when it needs to be repeated in two places. Instead let's go ahead and introduce an option CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT that depends on KUNIT and selects all the crypto library options that have corresponding KUnit tests. Update lib/crypto/.kunitconfig to use this option. Link: https://lore.kernel.org/r/20260314035927.51351-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/.kunitconfig | 22 +--------------------- lib/crypto/tests/Kconfig | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig index 8cfd213bded9..63a592731d1d 100644 --- a/lib/crypto/.kunitconfig +++ b/lib/crypto/.kunitconfig @@ -1,26 +1,6 @@ CONFIG_KUNIT=y -# These kconfig options select all the CONFIG_CRYPTO_LIB_* symbols that have a -# corresponding KUnit test. Those symbols cannot be directly enabled here, -# since they are hidden symbols. -CONFIG_CRYPTO=y -CONFIG_CRYPTO_ADIANTUM=y -CONFIG_CRYPTO_AES=y -CONFIG_CRYPTO_BLAKE2B=y -CONFIG_CRYPTO_CHACHA20POLY1305=y -CONFIG_CRYPTO_CMAC=y -CONFIG_CRYPTO_HCTR2=y -CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_MLDSA=y -CONFIG_CRYPTO_SHA1=y -CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_SHA512=y -CONFIG_CRYPTO_SHA3=y -CONFIG_INET=y -CONFIG_IPV6=y -CONFIG_NET=y -CONFIG_NETDEVICES=y -CONFIG_WIREGUARD=y +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 0d71de3da15d..caab7fdbdfde 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -116,6 +116,30 @@ config CRYPTO_LIB_SHA3_KUNIT_TEST including SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128 and SHAKE256. +config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT + tristate "Enable all crypto library code for KUnit tests" + depends on KUNIT + select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_BLAKE2B + select CRYPTO_LIB_CURVE25519 + select CRYPTO_LIB_MD5 + select CRYPTO_LIB_MLDSA + select CRYPTO_LIB_NH + select CRYPTO_LIB_POLY1305 + select CRYPTO_LIB_POLYVAL + select CRYPTO_LIB_SHA1 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + select CRYPTO_LIB_SHA3 + help + Enable all the crypto library code that has KUnit tests. + + Enable this only if you'd like to test all the crypto library code, + even code that wouldn't otherwise need to be built. + + You'll still need to enable the tests themselves, either individually + or using KUNIT_ALL_TESTS. + config CRYPTO_LIB_BENCHMARK_VISIBLE bool From 8d547482231fef30d0d6440629b73560ad3e937c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 13 Mar 2026 20:59:27 -0700 Subject: [PATCH 13/66] kunit: configs: Enable all crypto library tests in all_tests.config The new option CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT enables all the crypto library code that has KUnit tests, causing CONFIG_KUNIT_ALL_TESTS to enable all these tests. Add this option to all_tests.config so that kunit.py will run them when passed the --alltests option. Link: https://lore.kernel.org/r/20260314035927.51351-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- tools/testing/kunit/configs/all_tests.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/kunit/configs/all_tests.config b/tools/testing/kunit/configs/all_tests.config index 422e186cf3cf..6910b07082da 100644 --- a/tools/testing/kunit/configs/all_tests.config +++ b/tools/testing/kunit/configs/all_tests.config @@ -44,6 +44,8 @@ CONFIG_REGMAP_BUILD=y CONFIG_AUDIT=y +CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y + CONFIG_PRIME_NUMBERS=y CONFIG_SECURITY=y From 6d80749becf8fc5ffa004194e578f79b558235ef Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 16 Mar 2026 21:06:26 -0700 Subject: [PATCH 14/66] lib/crypto: tests: Drop the default to CRYPTO_SELFTESTS Defaulting the crypto KUnit tests to KUNIT_ALL_TESTS || CRYPTO_SELFTESTS instead of simply KUNIT_ALL_TESTS was originally intended to make it easy to enable all the crypto KUnit tests. This additional default is nonstandard for KUnit tests, though, and it can cause all the KUnit tests to be built-in unexpectedly if CRYPTO_SELFTESTS is set. It also constitutes a back-reference to crypto/ from lib/crypto/, which is something that we should be avoiding in order to get clean layering. Now that we provide a lib/crypto/.kunitconfig file that enables all crypto KUnit tests, let's consider that to be the supported way to enable all these tests, and drop the default of CRYPTO_SELFTESTS. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260317040626.5697-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/tests/Kconfig | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index caab7fdbdfde..42e1770e1883 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -3,7 +3,7 @@ config CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST tristate "KUnit tests for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_AES_CBC_MACS - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC message @@ -12,7 +12,7 @@ config CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST config CRYPTO_LIB_BLAKE2B_KUNIT_TEST tristate "KUnit tests for BLAKE2b" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_BLAKE2B - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the BLAKE2b cryptographic hash function. @@ -20,7 +20,7 @@ config CRYPTO_LIB_BLAKE2B_KUNIT_TEST config CRYPTO_LIB_BLAKE2S_KUNIT_TEST tristate "KUnit tests for BLAKE2s" if !KUNIT_ALL_TESTS depends on KUNIT - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE # No need to depend on CRYPTO_LIB_BLAKE2S here, as that option doesn't # exist; the BLAKE2s code is always built-in for the /dev/random driver. @@ -30,7 +30,7 @@ config CRYPTO_LIB_BLAKE2S_KUNIT_TEST config CRYPTO_LIB_CURVE25519_KUNIT_TEST tristate "KUnit tests for Curve25519" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_CURVE25519 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the Curve25519 Diffie-Hellman function. @@ -38,7 +38,7 @@ config CRYPTO_LIB_CURVE25519_KUNIT_TEST config CRYPTO_LIB_MD5_KUNIT_TEST tristate "KUnit tests for MD5" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_MD5 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the MD5 cryptographic hash function and its @@ -47,7 +47,7 @@ config CRYPTO_LIB_MD5_KUNIT_TEST config CRYPTO_LIB_MLDSA_KUNIT_TEST tristate "KUnit tests for ML-DSA" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_MLDSA - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the ML-DSA digital signature algorithm. @@ -55,14 +55,14 @@ config CRYPTO_LIB_MLDSA_KUNIT_TEST config CRYPTO_LIB_NH_KUNIT_TEST tristate "KUnit tests for NH" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_NH - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS help KUnit tests for the NH almost-universal hash function. config CRYPTO_LIB_POLY1305_KUNIT_TEST tristate "KUnit tests for Poly1305" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_POLY1305 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the Poly1305 library functions. @@ -70,7 +70,7 @@ config CRYPTO_LIB_POLY1305_KUNIT_TEST config CRYPTO_LIB_POLYVAL_KUNIT_TEST tristate "KUnit tests for POLYVAL" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_POLYVAL - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the POLYVAL library functions. @@ -78,7 +78,7 @@ config CRYPTO_LIB_POLYVAL_KUNIT_TEST config CRYPTO_LIB_SHA1_KUNIT_TEST tristate "KUnit tests for SHA-1" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA1 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-1 cryptographic hash function and its @@ -89,7 +89,7 @@ config CRYPTO_LIB_SHA1_KUNIT_TEST config CRYPTO_LIB_SHA256_KUNIT_TEST tristate "KUnit tests for SHA-224 and SHA-256" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA256 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-224 and SHA-256 cryptographic hash functions @@ -100,7 +100,7 @@ config CRYPTO_LIB_SHA256_KUNIT_TEST config CRYPTO_LIB_SHA512_KUNIT_TEST tristate "KUnit tests for SHA-384 and SHA-512" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA512 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA-384 and SHA-512 cryptographic hash functions @@ -109,7 +109,7 @@ config CRYPTO_LIB_SHA512_KUNIT_TEST config CRYPTO_LIB_SHA3_KUNIT_TEST tristate "KUnit tests for SHA-3" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_SHA3 - default KUNIT_ALL_TESTS || CRYPTO_SELFTESTS + default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help KUnit tests for the SHA3 cryptographic hash and XOF functions, From d76a943039a772fb2afd9c92bd25412d09bdf5c8 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 14 Mar 2026 10:35:26 -0700 Subject: [PATCH 15/66] lib/crypto: Remove unused file blockhash.h For a short time this file was used by the SHA-256 and Poly1305 library code, but they are no longer using it. Remove this unused file. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260314173526.17349-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/internal/blockhash.h | 52 ----------------------------- 1 file changed, 52 deletions(-) delete mode 100644 include/crypto/internal/blockhash.h diff --git a/include/crypto/internal/blockhash.h b/include/crypto/internal/blockhash.h deleted file mode 100644 index 52d9d4c82493..000000000000 --- a/include/crypto/internal/blockhash.h +++ /dev/null @@ -1,52 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Handle partial blocks for block hash. - * - * Copyright (c) 2015 Linaro Ltd - * Copyright (c) 2025 Herbert Xu - */ - -#ifndef _CRYPTO_INTERNAL_BLOCKHASH_H -#define _CRYPTO_INTERNAL_BLOCKHASH_H - -#include -#include - -#define BLOCK_HASH_UPDATE_BASE(block_fn, state, src, nbytes, bs, dv, \ - buf, buflen) \ - ({ \ - typeof(block_fn) *_block_fn = &(block_fn); \ - typeof(state + 0) _state = (state); \ - unsigned int _buflen = (buflen); \ - size_t _nbytes = (nbytes); \ - unsigned int _bs = (bs); \ - const u8 *_src = (src); \ - u8 *_buf = (buf); \ - while ((_buflen + _nbytes) >= _bs) { \ - const u8 *data = _src; \ - size_t len = _nbytes; \ - size_t blocks; \ - int remain; \ - if (_buflen) { \ - remain = _bs - _buflen; \ - memcpy(_buf + _buflen, _src, remain); \ - data = _buf; \ - len = _bs; \ - } \ - remain = len % bs; \ - blocks = (len - remain) / (dv); \ - (*_block_fn)(_state, data, blocks); \ - _src += len - remain - _buflen; \ - _nbytes -= len - remain - _buflen; \ - _buflen = 0; \ - } \ - memcpy(_buf + _buflen, _src, _nbytes); \ - _buflen += _nbytes; \ - }) - -#define BLOCK_HASH_UPDATE(block, state, src, nbytes, bs, buf, buflen) \ - BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, 1, buf, buflen) -#define BLOCK_HASH_UPDATE_BLOCKS(block, state, src, nbytes, bs, buf, buflen) \ - BLOCK_HASH_UPDATE_BASE(block, state, src, nbytes, bs, bs, buf, buflen) - -#endif /* _CRYPTO_INTERNAL_BLOCKHASH_H */ From c2db2288b8c3e2878cc37962375419cca8dfe3b6 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 14 Mar 2026 10:50:49 -0700 Subject: [PATCH 16/66] lib/crypto: arm64: Drop checks for CONFIG_KERNEL_MODE_NEON CONFIG_KERNEL_MODE_NEON is always enabled on arm64, and it always has been since its introduction in 2013. Given that and the fact that the usefulness of kernel-mode NEON has only been increasing over time, checking for this option in arm64-specific code is unnecessary. Remove these checks from lib/crypto/ to simplify the code and prevent any future bugs where e.g. code gets disabled due to a typo in this logic. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260314175049.26931-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/Kconfig | 12 ++++++------ lib/crypto/Makefile | 17 ++++++----------- lib/crypto/arm64/aes.h | 16 ++++------------ lib/crypto/arm64/sha256.h | 8 ++------ lib/crypto/arm64/sha512.h | 5 +---- 5 files changed, 19 insertions(+), 39 deletions(-) diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 42ec51645915..4910fe20e42a 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -80,7 +80,7 @@ config CRYPTO_LIB_CHACHA_ARCH bool depends on CRYPTO_LIB_CHACHA && !UML && !KMSAN default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS && CPU_MIPS32_R2 default y if PPC64 && CPU_LITTLE_ENDIAN && VSX default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ @@ -140,7 +140,7 @@ config CRYPTO_LIB_NH_ARCH bool depends on CRYPTO_LIB_NH && !UML && !KMSAN default y if ARM && KERNEL_MODE_NEON - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if X86_64 config CRYPTO_LIB_POLY1305 @@ -153,7 +153,7 @@ config CRYPTO_LIB_POLY1305_ARCH bool depends on CRYPTO_LIB_POLY1305 && !UML && !KMSAN default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS # The PPC64 code needs to be fixed to work in softirq context. default y if PPC64 && CPU_LITTLE_ENDIAN && VSX && BROKEN @@ -187,7 +187,7 @@ config CRYPTO_LIB_POLYVAL config CRYPTO_LIB_POLYVAL_ARCH bool depends on CRYPTO_LIB_POLYVAL && !UML - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if X86_64 config CRYPTO_LIB_CHACHA20POLY1305 @@ -206,7 +206,7 @@ config CRYPTO_LIB_SHA1_ARCH bool depends on CRYPTO_LIB_SHA1 && !UML default y if ARM - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if MIPS && CPU_CAVIUM_OCTEON default y if PPC default y if S390 @@ -262,7 +262,7 @@ config CRYPTO_LIB_SHA3 config CRYPTO_LIB_SHA3_ARCH bool depends on CRYPTO_LIB_SHA3 && !UML - default y if ARM64 && KERNEL_MODE_NEON + default y if ARM64 default y if S390 config CRYPTO_LIB_SM3 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index c05d4b4e8e82..a961615c8c7f 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -23,13 +23,10 @@ ifeq ($(CONFIG_CRYPTO_LIB_AES_ARCH),y) CFLAGS_aes.o += -I$(src)/$(SRCARCH) libaes-$(CONFIG_ARM) += arm/aes-cipher-core.o - -ifeq ($(CONFIG_ARM64),y) -libaes-y += arm64/aes-cipher-core.o -libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o \ - arm64/aes-ce.o \ - arm64/aes-neon.o -endif +libaes-$(CONFIG_ARM64) += arm64/aes-cipher-core.o \ + arm64/aes-ce-core.o \ + arm64/aes-ce.o \ + arm64/aes-neon.o ifeq ($(CONFIG_PPC),y) ifeq ($(CONFIG_SPE),y) @@ -299,10 +296,9 @@ AFLAGS_arm/sha256-core.o += $(aflags-thumb2-y) endif ifeq ($(CONFIG_ARM64),y) -libsha256-y += arm64/sha256-core.o +libsha256-y += arm64/sha256-ce.o arm64/sha256-core.o $(obj)/arm64/sha256-core.S: $(src)/arm64/sha2-armv8.pl $(call cmd,perlasm_with_args) -libsha256-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha256-ce.o endif libsha256-$(CONFIG_PPC) += powerpc/sha256-spe-asm.o @@ -329,10 +325,9 @@ AFLAGS_arm/sha512-core.o += $(aflags-thumb2-y) endif ifeq ($(CONFIG_ARM64),y) -libsha512-y += arm64/sha512-core.o +libsha512-y += arm64/sha512-ce-core.o arm64/sha512-core.o $(obj)/arm64/sha512-core.S: $(src)/arm64/sha2-armv8.pl $(call cmd,perlasm_with_args) -libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o endif libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h index 78e7b4e5f120..135d3324a30a 100644 --- a/lib/crypto/arm64/aes.h +++ b/lib/crypto/arm64/aes.h @@ -52,8 +52,7 @@ static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys, struct aes_block *key_enc, *key_dec; int i, j; - if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || - !static_branch_likely(&have_aes) || unlikely(!may_use_simd())) { + if (!static_branch_likely(&have_aes) || unlikely(!may_use_simd())) { aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len); return; } @@ -130,7 +129,6 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, } EXPORT_SYMBOL(ce_aes_expandkey); -#if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_encrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_decrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_encrypt, "CRYPTO_INTERNAL"); @@ -156,7 +154,6 @@ EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); -#endif #if IS_MODULE(CONFIG_CRYPTO_AES_ARM64_CE_CCM) EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL"); #endif @@ -165,8 +162,7 @@ static void aes_encrypt_arch(const struct aes_enckey *key, u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_aes) && likely(may_use_simd())) { + if (static_branch_likely(&have_aes) && likely(may_use_simd())) { scoped_ksimd() __aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds); } else { @@ -178,8 +174,7 @@ static void aes_decrypt_arch(const struct aes_key *key, u8 out[AES_BLOCK_SIZE], const u8 in[AES_BLOCK_SIZE]) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_aes) && likely(may_use_simd())) { + if (static_branch_likely(&have_aes) && likely(may_use_simd())) { scoped_ksimd() __aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in, key->nrounds); @@ -196,8 +191,7 @@ static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], size_t nblocks, bool enc_before, bool enc_after) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon) && likely(may_use_simd())) { + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { do { size_t rem; @@ -223,7 +217,6 @@ static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], } #endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ -#ifdef CONFIG_KERNEL_MODE_NEON #define aes_mod_init_arch aes_mod_init_arch static void aes_mod_init_arch(void) { @@ -233,4 +226,3 @@ static void aes_mod_init_arch(void) static_branch_enable(&have_aes); } } -#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/lib/crypto/arm64/sha256.h b/lib/crypto/arm64/sha256.h index 568dff0f276a..1fad3d7baa9a 100644 --- a/lib/crypto/arm64/sha256.h +++ b/lib/crypto/arm64/sha256.h @@ -20,8 +20,7 @@ asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state, static void sha256_blocks(struct sha256_block_state *state, const u8 *data, size_t nblocks) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_neon) && likely(may_use_simd())) { + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { if (static_branch_likely(&have_ce)) { do { size_t rem; @@ -61,8 +60,7 @@ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx, * Further limit len to 65536 to avoid spending too long with preemption * disabled. (Of course, in practice len is nearly always 4096 anyway.) */ - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE && + if (static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE && len <= 65536 && likely(may_use_simd())) { scoped_ksimd() sha256_ce_finup2x(ctx, data1, data2, len, out1, out2); @@ -78,7 +76,6 @@ static bool sha256_finup_2x_is_optimized_arch(void) return static_key_enabled(&have_ce); } -#ifdef CONFIG_KERNEL_MODE_NEON #define sha256_mod_init_arch sha256_mod_init_arch static void sha256_mod_init_arch(void) { @@ -88,4 +85,3 @@ static void sha256_mod_init_arch(void) static_branch_enable(&have_ce); } } -#endif /* CONFIG_KERNEL_MODE_NEON */ diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h index 7eb7ef04d268..d978c4d07e90 100644 --- a/lib/crypto/arm64/sha512.h +++ b/lib/crypto/arm64/sha512.h @@ -18,8 +18,7 @@ asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, static void sha512_blocks(struct sha512_block_state *state, const u8 *data, size_t nblocks) { - if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && - static_branch_likely(&have_sha512_insns) && + if (static_branch_likely(&have_sha512_insns) && likely(may_use_simd())) { do { size_t rem; @@ -35,11 +34,9 @@ static void sha512_blocks(struct sha512_block_state *state, } } -#ifdef CONFIG_KERNEL_MODE_NEON #define sha512_mod_init_arch sha512_mod_init_arch static void sha512_mod_init_arch(void) { if (cpu_have_named_feature(SHA512)) static_branch_enable(&have_sha512_insns); } -#endif /* CONFIG_KERNEL_MODE_NEON */ From 645e64136b93e431a48f3b178fe7cf13737b10f6 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 09:42:33 -0700 Subject: [PATCH 17/66] sample/tsm-mr: Use SHA-2 library APIs Given that tsm_mr_sample has a particular set of algorithms that it wants, just use the library APIs for those algorithms rather than crypto_shash. This is more straightforward and more efficient. This also fixes a bug where this module failed to build if it was enabled without CRYPTO_HASH happening to be set elsewhere in the kconfig. (With the concurrent change to make TSM_MEASUREMENTS stop selecting CRYPTO, this existing build error would have become easier to encounter, as well.) Also, even if it built, crypto_alloc_shash() could fail at runtime due to the needed algorithms not being available. The library functions simply use direct linking. So if it builds, which it will due to the kconfig options being selected, they are available. Fixes: f6953f1f9ec4 ("tsm-mr: Add tsm-mr sample code") Acked-by: Arnd Bergmann Acked-by: Dan Williams Link: https://lore.kernel.org/r/20260318164233.19800-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- samples/Kconfig | 2 ++ samples/tsm-mr/tsm_mr_sample.c | 66 +++++++++++++++++----------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/samples/Kconfig b/samples/Kconfig index 5bc7c9e5a59e..a75e8e78330d 100644 --- a/samples/Kconfig +++ b/samples/Kconfig @@ -186,6 +186,8 @@ config SAMPLE_TIMER config SAMPLE_TSM_MR tristate "TSM measurement sample" + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 select TSM_MEASUREMENTS select VIRT_DRIVERS help diff --git a/samples/tsm-mr/tsm_mr_sample.c b/samples/tsm-mr/tsm_mr_sample.c index a2c652148639..c79dbc1e0456 100644 --- a/samples/tsm-mr/tsm_mr_sample.c +++ b/samples/tsm-mr/tsm_mr_sample.c @@ -6,7 +6,7 @@ #include #include #include -#include +#include static struct { u8 static_mr[SHA384_DIGEST_SIZE]; @@ -23,47 +23,45 @@ static struct { static int sample_report_refresh(const struct tsm_measurements *tm) { - struct crypto_shash *tfm; - int rc; - - tfm = crypto_alloc_shash(hash_algo_name[HASH_ALGO_SHA512], 0, 0); - if (IS_ERR(tfm)) { - pr_err("crypto_alloc_shash failed: %ld\n", PTR_ERR(tfm)); - return PTR_ERR(tfm); - } - - rc = crypto_shash_tfm_digest(tfm, (u8 *)&sample_report, - offsetof(typeof(sample_report), - report_digest), - sample_report.report_digest); - crypto_free_shash(tfm); - if (rc) - pr_err("crypto_shash_tfm_digest failed: %d\n", rc); - return rc; + sha512((const u8 *)&sample_report, + offsetof(typeof(sample_report), report_digest), + sample_report.report_digest); + return 0; } static int sample_report_extend_mr(const struct tsm_measurements *tm, const struct tsm_measurement_register *mr, const u8 *data) { - SHASH_DESC_ON_STACK(desc, 0); - int rc; + union { + struct sha256_ctx sha256; + struct sha384_ctx sha384; + struct sha512_ctx sha512; + } ctx; - desc->tfm = crypto_alloc_shash(hash_algo_name[mr->mr_hash], 0, 0); - if (IS_ERR(desc->tfm)) { - pr_err("crypto_alloc_shash failed: %ld\n", PTR_ERR(desc->tfm)); - return PTR_ERR(desc->tfm); + switch (mr->mr_hash) { + case HASH_ALGO_SHA256: + sha256_init(&ctx.sha256); + sha256_update(&ctx.sha256, mr->mr_value, mr->mr_size); + sha256_update(&ctx.sha256, data, mr->mr_size); + sha256_final(&ctx.sha256, mr->mr_value); + return 0; + case HASH_ALGO_SHA384: + sha384_init(&ctx.sha384); + sha384_update(&ctx.sha384, mr->mr_value, mr->mr_size); + sha384_update(&ctx.sha384, data, mr->mr_size); + sha384_final(&ctx.sha384, mr->mr_value); + return 0; + case HASH_ALGO_SHA512: + sha512_init(&ctx.sha512); + sha512_update(&ctx.sha512, mr->mr_value, mr->mr_size); + sha512_update(&ctx.sha512, data, mr->mr_size); + sha512_final(&ctx.sha512, mr->mr_value); + return 0; + default: + pr_err("Unsupported hash algorithm: %d\n", mr->mr_hash); + return -EOPNOTSUPP; } - - rc = crypto_shash_init(desc); - if (!rc) - rc = crypto_shash_update(desc, mr->mr_value, mr->mr_size); - if (!rc) - rc = crypto_shash_finup(desc, data, mr->mr_size, mr->mr_value); - crypto_free_shash(desc->tfm); - if (rc) - pr_err("SHA calculation failed: %d\n", rc); - return rc; } #define MR_(mr, hash) .mr_value = &sample_report.mr, TSM_MR_(mr, hash) From 6bc9effb4cbf9b6eba0f51aba1c8893dfd4c8100 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 3 Dec 2025 21:55:12 -0800 Subject: [PATCH 18/66] coco/guest: Remove unneeded selection of CRYPTO All that's needed here is CRYPTO_HASH_INFO. It used to be the case that CRYPTO_HASH_INFO was visible only when CRYPTO, but that was fixed by commit aacb37f597d0 ("lib/crypto: hash_info: Move hash_info.c into lib/crypto/"). Now CRYPTO_HASH_INFO can be selected directly. Acked-by: Dan Williams Link: https://lore.kernel.org/r/20251204055512.494013-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- drivers/virt/coco/guest/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/virt/coco/guest/Kconfig b/drivers/virt/coco/guest/Kconfig index 3d5e1d05bf34..da570dc4bd48 100644 --- a/drivers/virt/coco/guest/Kconfig +++ b/drivers/virt/coco/guest/Kconfig @@ -13,5 +13,4 @@ config TSM_REPORTS config TSM_MEASUREMENTS select TSM_GUEST select CRYPTO_HASH_INFO - select CRYPTO bool From 61f66c5216a961784b12307be60a25204525605c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:02 -0700 Subject: [PATCH 19/66] lib/crypto: gf128hash: Rename polyval module to gf128hash Currently, the standalone GHASH code is coupled with crypto_shash. This has resulted in unnecessary complexity and overhead, as well as the code being unavailable to library code such as the AES-GCM library. Like was done with POLYVAL, it needs to find a new home in lib/crypto/. GHASH and POLYVAL are closely related and can each be implemented in terms of each other. Optimized code for one can be reused with the other. But also since GHASH tends to be difficult to implement directly due to its unnatural bit order, most modern GHASH implementations (including the existing arm, arm64, powerpc, and x86 optimized GHASH code, and the new generic GHASH code I'll be adding) actually reinterpret the GHASH computation as an equivalent POLYVAL computation, pre and post-processing the inputs and outputs to map to/from POLYVAL. Given this close relationship, it makes sense to group the GHASH and POLYVAL code together in the same module. This gives us a wide range of options for implementing them, reusing code between the two and properly utilizing whatever instructions each architecture provides. Thus, GHASH support will be added to the library module that is currently called "polyval". Rename it to an appropriate name: "gf128hash". Rename files, options, functions, etc. where appropriate to reflect the upcoming sharing with GHASH. (Note: polyval_kunit is not renamed, as ghash_kunit will be added alongside it instead.) Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 2 +- crypto/hctr2.c | 2 +- include/crypto/{polyval.h => gf128hash.h} | 16 ++++++------- lib/crypto/Kconfig | 24 +++++++++---------- lib/crypto/Makefile | 20 ++++++++-------- lib/crypto/arm64/{polyval.h => gf128hash.h} | 4 ++-- lib/crypto/{polyval.c => gf128hash.c} | 26 ++++++++++----------- lib/crypto/tests/Kconfig | 4 ++-- lib/crypto/tests/polyval_kunit.c | 2 +- lib/crypto/x86/{polyval.h => gf128hash.h} | 4 ++-- 10 files changed, 52 insertions(+), 52 deletions(-) rename include/crypto/{polyval.h => gf128hash.h} (94%) rename lib/crypto/arm64/{polyval.h => gf128hash.h} (95%) rename lib/crypto/{polyval.c => gf128hash.c} (94%) rename lib/crypto/x86/{polyval.h => gf128hash.h} (95%) diff --git a/crypto/Kconfig b/crypto/Kconfig index b8608ef6823b..5627b3691561 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -686,7 +686,7 @@ config CRYPTO_ECB config CRYPTO_HCTR2 tristate "HCTR2" select CRYPTO_XCTR - select CRYPTO_LIB_POLYVAL + select CRYPTO_LIB_GF128HASH select CRYPTO_MANAGER help HCTR2 length-preserving encryption mode diff --git a/crypto/hctr2.c b/crypto/hctr2.c index f4cd6c29b4d3..ad5edf9366ac 100644 --- a/crypto/hctr2.c +++ b/crypto/hctr2.c @@ -16,9 +16,9 @@ * (https://eprint.iacr.org/2021/1441.pdf) */ +#include #include #include -#include #include #include diff --git a/include/crypto/polyval.h b/include/crypto/gf128hash.h similarity index 94% rename from include/crypto/polyval.h rename to include/crypto/gf128hash.h index b28b8ef11353..5ffa86f5c13f 100644 --- a/include/crypto/polyval.h +++ b/include/crypto/gf128hash.h @@ -1,12 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * POLYVAL library API + * GF(2^128) polynomial hashing: GHASH and POLYVAL * * Copyright 2025 Google LLC */ -#ifndef _CRYPTO_POLYVAL_H -#define _CRYPTO_POLYVAL_H +#ifndef _CRYPTO_GF128HASH_H +#define _CRYPTO_GF128HASH_H #include #include @@ -44,7 +44,7 @@ struct polyval_elem { * exponentiation repeats the POLYVAL dot operation, with its "extra" x^-128. */ struct polyval_key { -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH #ifdef CONFIG_ARM64 /** @h_powers: Powers of the hash key H^8 through H^1 */ struct polyval_elem h_powers[8]; @@ -54,10 +54,10 @@ struct polyval_key { #else #error "Unhandled arch" #endif -#else /* CONFIG_CRYPTO_LIB_POLYVAL_ARCH */ +#else /* CONFIG_CRYPTO_LIB_GF128HASH_ARCH */ /** @h: The hash key H */ struct polyval_elem h; -#endif /* !CONFIG_CRYPTO_LIB_POLYVAL_ARCH */ +#endif /* !CONFIG_CRYPTO_LIB_GF128HASH_ARCH */ }; /** @@ -84,7 +84,7 @@ struct polyval_ctx { * * Context: Any context. */ -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]); @@ -187,4 +187,4 @@ static inline void polyval(const struct polyval_key *key, polyval_final(&ctx, out); } -#endif /* _CRYPTO_POLYVAL_H */ +#endif /* _CRYPTO_GF128HASH_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 4910fe20e42a..98cedd95c2a5 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -110,6 +110,18 @@ config CRYPTO_LIB_CURVE25519_GENERIC config CRYPTO_LIB_DES tristate +config CRYPTO_LIB_GF128HASH + tristate + help + The GHASH and POLYVAL library functions. Select this if your module + uses any of the functions from . + +config CRYPTO_LIB_GF128HASH_ARCH + bool + depends on CRYPTO_LIB_GF128HASH && !UML + default y if ARM64 + default y if X86_64 + config CRYPTO_LIB_MD5 tristate help @@ -178,18 +190,6 @@ config CRYPTO_LIB_POLY1305_RSIZE default 9 if ARM || ARM64 default 1 -config CRYPTO_LIB_POLYVAL - tristate - help - The POLYVAL library functions. Select this if your module uses any of - the functions from . - -config CRYPTO_LIB_POLYVAL_ARCH - bool - depends on CRYPTO_LIB_POLYVAL && !UML - default y if ARM64 - default y if X86_64 - config CRYPTO_LIB_CHACHA20POLY1305 tristate select CRYPTO_LIB_CHACHA diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index a961615c8c7f..fc30622123d2 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -154,6 +154,16 @@ libdes-y := des.o ################################################################################ +obj-$(CONFIG_CRYPTO_LIB_GF128HASH) += libgf128hash.o +libgf128hash-y := gf128hash.o +ifeq ($(CONFIG_CRYPTO_LIB_GF128HASH_ARCH),y) +CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH) +libgf128hash-$(CONFIG_ARM64) += arm64/polyval-ce-core.o +libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o +endif + +################################################################################ + obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o libmd5-y := md5.o ifeq ($(CONFIG_CRYPTO_LIB_MD5_ARCH),y) @@ -251,16 +261,6 @@ clean-files += arm/poly1305-core.S \ ################################################################################ -obj-$(CONFIG_CRYPTO_LIB_POLYVAL) += libpolyval.o -libpolyval-y := polyval.o -ifeq ($(CONFIG_CRYPTO_LIB_POLYVAL_ARCH),y) -CFLAGS_polyval.o += -I$(src)/$(SRCARCH) -libpolyval-$(CONFIG_ARM64) += arm64/polyval-ce-core.o -libpolyval-$(CONFIG_X86) += x86/polyval-pclmul-avx.o -endif - -################################################################################ - obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o libsha1-y := sha1.o ifeq ($(CONFIG_CRYPTO_LIB_SHA1_ARCH),y) diff --git a/lib/crypto/arm64/polyval.h b/lib/crypto/arm64/gf128hash.h similarity index 95% rename from lib/crypto/arm64/polyval.h rename to lib/crypto/arm64/gf128hash.h index a39763395e9b..c1012007adcf 100644 --- a/lib/crypto/arm64/polyval.h +++ b/lib/crypto/arm64/gf128hash.h @@ -72,8 +72,8 @@ static void polyval_blocks_arch(struct polyval_elem *acc, } } -#define polyval_mod_init_arch polyval_mod_init_arch -static void polyval_mod_init_arch(void) +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) { if (cpu_have_named_feature(PMULL)) static_branch_enable(&have_pmull); diff --git a/lib/crypto/polyval.c b/lib/crypto/gf128hash.c similarity index 94% rename from lib/crypto/polyval.c rename to lib/crypto/gf128hash.c index 5796275f574a..8bb848bf26b7 100644 --- a/lib/crypto/polyval.c +++ b/lib/crypto/gf128hash.c @@ -1,11 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * POLYVAL library functions + * GF(2^128) polynomial hashing: GHASH and POLYVAL * * Copyright 2025 Google LLC */ -#include +#include #include #include #include @@ -218,8 +218,8 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, } /* Include the arch-optimized implementation of POLYVAL, if one is available. */ -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH -#include "polyval.h" /* $(SRCARCH)/polyval.h */ +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH +#include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */ void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { @@ -238,7 +238,7 @@ EXPORT_SYMBOL_GPL(polyval_preparekey); static void polyval_mul(struct polyval_ctx *ctx) { -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH polyval_mul_arch(&ctx->acc, ctx->key); #else polyval_mul_generic(&ctx->acc, &ctx->key->h); @@ -248,7 +248,7 @@ static void polyval_mul(struct polyval_ctx *ctx) static void polyval_blocks(struct polyval_ctx *ctx, const u8 *data, size_t nblocks) { -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks); #else polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); @@ -289,19 +289,19 @@ void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]) } EXPORT_SYMBOL_GPL(polyval_final); -#ifdef polyval_mod_init_arch -static int __init polyval_mod_init(void) +#ifdef gf128hash_mod_init_arch +static int __init gf128hash_mod_init(void) { - polyval_mod_init_arch(); + gf128hash_mod_init_arch(); return 0; } -subsys_initcall(polyval_mod_init); +subsys_initcall(gf128hash_mod_init); -static void __exit polyval_mod_exit(void) +static void __exit gf128hash_mod_exit(void) { } -module_exit(polyval_mod_exit); +module_exit(gf128hash_mod_exit); #endif -MODULE_DESCRIPTION("POLYVAL almost-XOR-universal hash function"); +MODULE_DESCRIPTION("GF(2^128) polynomial hashing: GHASH and POLYVAL"); MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 42e1770e1883..aa627b6b9855 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -69,7 +69,7 @@ config CRYPTO_LIB_POLY1305_KUNIT_TEST config CRYPTO_LIB_POLYVAL_KUNIT_TEST tristate "KUnit tests for POLYVAL" if !KUNIT_ALL_TESTS - depends on KUNIT && CRYPTO_LIB_POLYVAL + depends on KUNIT && CRYPTO_LIB_GF128HASH default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help @@ -122,11 +122,11 @@ config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT select CRYPTO_LIB_AES_CBC_MACS select CRYPTO_LIB_BLAKE2B select CRYPTO_LIB_CURVE25519 + select CRYPTO_LIB_GF128HASH select CRYPTO_LIB_MD5 select CRYPTO_LIB_MLDSA select CRYPTO_LIB_NH select CRYPTO_LIB_POLY1305 - select CRYPTO_LIB_POLYVAL select CRYPTO_LIB_SHA1 select CRYPTO_LIB_SHA256 select CRYPTO_LIB_SHA512 diff --git a/lib/crypto/tests/polyval_kunit.c b/lib/crypto/tests/polyval_kunit.c index f47f41a39a41..d1f53a690ab8 100644 --- a/lib/crypto/tests/polyval_kunit.c +++ b/lib/crypto/tests/polyval_kunit.c @@ -2,7 +2,7 @@ /* * Copyright 2025 Google LLC */ -#include +#include #include "polyval-testvecs.h" /* diff --git a/lib/crypto/x86/polyval.h b/lib/crypto/x86/gf128hash.h similarity index 95% rename from lib/crypto/x86/polyval.h rename to lib/crypto/x86/gf128hash.h index ef8797521420..fe506cf6431b 100644 --- a/lib/crypto/x86/polyval.h +++ b/lib/crypto/x86/gf128hash.h @@ -74,8 +74,8 @@ static void polyval_blocks_arch(struct polyval_elem *acc, } } -#define polyval_mod_init_arch polyval_mod_init_arch -static void polyval_mod_init_arch(void) +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) { if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) && boot_cpu_has(X86_FEATURE_AVX)) From b3b6e8f9b38911e9b30a5abe845541ade0797327 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:03 -0700 Subject: [PATCH 20/66] lib/crypto: gf128hash: Support GF128HASH_ARCH without all POLYVAL functions Currently, some architectures (arm64 and x86) have optimized code for both GHASH and POLYVAL. Others (arm, powerpc, riscv, and s390) have optimized code only for GHASH. While POLYVAL support could be implemented on these other architectures, until then we need to support the case where arch-optimized functions are present only for GHASH. Therefore, update the support for arch-optimized POLYVAL functions to allow architectures to opt into supporting these functions individually. The new meaning of CONFIG_CRYPTO_LIB_GF128HASH_ARCH is that some level of GHASH and/or POLYVAL acceleration is provided. Also provide an implementation of polyval_mul() based on polyval_blocks_arch(), for when polyval_mul_arch() isn't implemented. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/gf128hash.h | 22 +++------------------- lib/crypto/arm64/gf128hash.h | 3 +++ lib/crypto/gf128hash.c | 16 ++++++++++++---- lib/crypto/x86/gf128hash.h | 3 +++ 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h index 5ffa86f5c13f..1052041e3499 100644 --- a/include/crypto/gf128hash.h +++ b/include/crypto/gf128hash.h @@ -44,20 +44,14 @@ struct polyval_elem { * exponentiation repeats the POLYVAL dot operation, with its "extra" x^-128. */ struct polyval_key { -#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH -#ifdef CONFIG_ARM64 - /** @h_powers: Powers of the hash key H^8 through H^1 */ - struct polyval_elem h_powers[8]; -#elif defined(CONFIG_X86) +#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \ + (defined(CONFIG_ARM64) || defined(CONFIG_X86)) /** @h_powers: Powers of the hash key H^8 through H^1 */ struct polyval_elem h_powers[8]; #else -#error "Unhandled arch" -#endif -#else /* CONFIG_CRYPTO_LIB_GF128HASH_ARCH */ /** @h: The hash key H */ struct polyval_elem h; -#endif /* !CONFIG_CRYPTO_LIB_GF128HASH_ARCH */ +#endif }; /** @@ -84,19 +78,9 @@ struct polyval_ctx { * * Context: Any context. */ -#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]); -#else -static inline void polyval_preparekey(struct polyval_key *key, - const u8 raw_key[POLYVAL_BLOCK_SIZE]) -{ - /* Just a simple copy, so inline it. */ - memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE); -} -#endif - /** * polyval_init() - Initialize a POLYVAL context for a new message * @ctx: The context to initialize diff --git a/lib/crypto/arm64/gf128hash.h b/lib/crypto/arm64/gf128hash.h index c1012007adcf..796c36804dda 100644 --- a/lib/crypto/arm64/gf128hash.h +++ b/lib/crypto/arm64/gf128hash.h @@ -17,6 +17,7 @@ asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks); +#define polyval_preparekey_arch polyval_preparekey_arch static void polyval_preparekey_arch(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { @@ -40,6 +41,7 @@ static void polyval_preparekey_arch(struct polyval_key *key, } } +#define polyval_mul_arch polyval_mul_arch static void polyval_mul_arch(struct polyval_elem *acc, const struct polyval_key *key) { @@ -51,6 +53,7 @@ static void polyval_mul_arch(struct polyval_elem *acc, } } +#define polyval_blocks_arch polyval_blocks_arch static void polyval_blocks_arch(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks) diff --git a/lib/crypto/gf128hash.c b/lib/crypto/gf128hash.c index 8bb848bf26b7..05f44a9193f7 100644 --- a/lib/crypto/gf128hash.c +++ b/lib/crypto/gf128hash.c @@ -217,16 +217,20 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, } while (--nblocks); } -/* Include the arch-optimized implementation of POLYVAL, if one is available. */ #ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH #include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */ +#endif + void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { +#ifdef polyval_preparekey_arch polyval_preparekey_arch(key, raw_key); +#else + memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE); +#endif } EXPORT_SYMBOL_GPL(polyval_preparekey); -#endif /* Else, polyval_preparekey() is an inline function. */ /* * polyval_mul_generic() and polyval_blocks_generic() take the key as a @@ -238,8 +242,12 @@ EXPORT_SYMBOL_GPL(polyval_preparekey); static void polyval_mul(struct polyval_ctx *ctx) { -#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH +#ifdef polyval_mul_arch polyval_mul_arch(&ctx->acc, ctx->key); +#elif defined(polyval_blocks_arch) + static const u8 zeroes[POLYVAL_BLOCK_SIZE]; + + polyval_blocks_arch(&ctx->acc, ctx->key, zeroes, 1); #else polyval_mul_generic(&ctx->acc, &ctx->key->h); #endif @@ -248,7 +256,7 @@ static void polyval_mul(struct polyval_ctx *ctx) static void polyval_blocks(struct polyval_ctx *ctx, const u8 *data, size_t nblocks) { -#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH +#ifdef polyval_blocks_arch polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks); #else polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); diff --git a/lib/crypto/x86/gf128hash.h b/lib/crypto/x86/gf128hash.h index fe506cf6431b..adf6147ea677 100644 --- a/lib/crypto/x86/gf128hash.h +++ b/lib/crypto/x86/gf128hash.h @@ -17,6 +17,7 @@ asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks); +#define polyval_preparekey_arch polyval_preparekey_arch static void polyval_preparekey_arch(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { @@ -40,6 +41,7 @@ static void polyval_preparekey_arch(struct polyval_key *key, } } +#define polyval_mul_arch polyval_mul_arch static void polyval_mul_arch(struct polyval_elem *acc, const struct polyval_key *key) { @@ -52,6 +54,7 @@ static void polyval_mul_arch(struct polyval_elem *acc, } } +#define polyval_blocks_arch polyval_blocks_arch static void polyval_blocks_arch(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks) From c417e7045b70345f59643fb2db67b0e7fbd7fbd0 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:04 -0700 Subject: [PATCH 21/66] lib/crypto: gf128hash: Add GHASH support Add GHASH support to the gf128hash module. This will replace the GHASH support in the crypto_shash API. It will be used by the "gcm" template and by the AES-GCM library (when an arch-optimized implementation of the full AES-GCM is unavailable). This consists of a simple API that mirrors the existing POLYVAL API, a generic implementation of that API based on the existing efficient and side-channel-resistant polyval_mul_generic(), and the framework for architecture-optimized implementations of the GHASH functions. The GHASH accumulator is stored in POLYVAL format rather than GHASH format, since this is what most modern GHASH implementations actually need. The few implementations that expect the accumulator in GHASH format will just convert the accumulator to/from GHASH format temporarily. (Supporting architecture-specific accumulator formats would be possible, but doesn't seem worth the complexity.) However, architecture-specific formats of struct ghash_key will be supported, since a variety of formats will be needed there anyway. The default format is just the key in POLYVAL format. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/gf128hash.h | 95 ++++++++++++++++++++++++ lib/crypto/gf128hash.c | 145 +++++++++++++++++++++++++++++++++---- 2 files changed, 227 insertions(+), 13 deletions(-) diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h index 1052041e3499..5090fbaa87f8 100644 --- a/include/crypto/gf128hash.h +++ b/include/crypto/gf128hash.h @@ -11,6 +11,8 @@ #include #include +#define GHASH_BLOCK_SIZE 16 +#define GHASH_DIGEST_SIZE 16 #define POLYVAL_BLOCK_SIZE 16 #define POLYVAL_DIGEST_SIZE 16 @@ -33,6 +35,16 @@ struct polyval_elem { }; }; +/** + * struct ghash_key - Prepared key for GHASH + * + * Use ghash_preparekey() to initialize this. + */ +struct ghash_key { + /** @h: The hash key H, in POLYVAL format */ + struct polyval_elem h; +}; + /** * struct polyval_key - Prepared key for POLYVAL * @@ -54,6 +66,20 @@ struct polyval_key { #endif }; +/** + * struct ghash_ctx - Context for computing a GHASH value + * @key: Pointer to the prepared GHASH key. The user of the API is + * responsible for ensuring that the key lives as long as the context. + * @acc: The accumulator. It is stored in POLYVAL format rather than GHASH + * format, since most implementations want it in POLYVAL format. + * @partial: Number of data bytes processed so far modulo GHASH_BLOCK_SIZE + */ +struct ghash_ctx { + const struct ghash_key *key; + struct polyval_elem acc; + size_t partial; +}; + /** * struct polyval_ctx - Context for computing a POLYVAL value * @key: Pointer to the prepared POLYVAL key. The user of the API is @@ -67,6 +93,18 @@ struct polyval_ctx { size_t partial; }; +/** + * ghash_preparekey() - Prepare a GHASH key + * @key: (output) The key structure to initialize + * @raw_key: The raw hash key + * + * Initialize a GHASH key structure from a raw key. + * + * Context: Any context. + */ +void ghash_preparekey(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]); + /** * polyval_preparekey() - Prepare a POLYVAL key * @key: (output) The key structure to initialize @@ -81,6 +119,18 @@ struct polyval_ctx { void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]); +/** + * ghash_init() - Initialize a GHASH context for a new message + * @ctx: The context to initialize + * @key: The key to use. Note that a pointer to the key is saved in the + * context, so the key must live at least as long as the context. + */ +static inline void ghash_init(struct ghash_ctx *ctx, + const struct ghash_key *key) +{ + *ctx = (struct ghash_ctx){ .key = key }; +} + /** * polyval_init() - Initialize a POLYVAL context for a new message * @ctx: The context to initialize @@ -125,6 +175,18 @@ static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx, *acc = ctx->acc; } +/** + * ghash_update() - Update a GHASH context with message data + * @ctx: The context to update; must have been initialized + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * + * This can be called any number of times. + * + * Context: Any context. + */ +void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len); + /** * polyval_update() - Update a POLYVAL context with message data * @ctx: The context to update; must have been initialized @@ -137,6 +199,20 @@ static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx, */ void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len); +/** + * ghash_final() - Finish computing a GHASH value + * @ctx: The context to finalize + * @out: The output value + * + * If the total data length isn't a multiple of GHASH_BLOCK_SIZE, then the + * final block is automatically zero-padded. + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE]); + /** * polyval_final() - Finish computing a POLYVAL value * @ctx: The context to finalize @@ -151,6 +227,25 @@ void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len); */ void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]); +/** + * ghash() - Compute a GHASH value + * @key: The prepared key + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * @out: The output value + * + * Context: Any context. + */ +static inline void ghash(const struct ghash_key *key, const u8 *data, + size_t len, u8 out[GHASH_BLOCK_SIZE]) +{ + struct ghash_ctx ctx; + + ghash_init(&ctx, key); + ghash_update(&ctx, data, len); + ghash_final(&ctx, out); +} + /** * polyval() - Compute a POLYVAL value * @key: The prepared key diff --git a/lib/crypto/gf128hash.c b/lib/crypto/gf128hash.c index 05f44a9193f7..2650603d8ba8 100644 --- a/lib/crypto/gf128hash.c +++ b/lib/crypto/gf128hash.c @@ -12,23 +12,30 @@ #include /* - * POLYVAL is an almost-XOR-universal hash function. Similar to GHASH, POLYVAL - * interprets the message as the coefficients of a polynomial in GF(2^128) and - * evaluates that polynomial at a secret point. POLYVAL has a simple - * mathematical relationship with GHASH, but it uses a better field convention - * which makes it easier and faster to implement. + * GHASH and POLYVAL are almost-XOR-universal hash functions. They interpret + * the message as the coefficients of a polynomial in the finite field GF(2^128) + * and evaluate that polynomial at a secret point. * - * POLYVAL is not a cryptographic hash function, and it should be used only by - * algorithms that are specifically designed to use it. + * Neither GHASH nor POLYVAL is a cryptographic hash function. They should be + * used only by algorithms that are specifically designed to use them. * - * POLYVAL is specified by "AES-GCM-SIV: Nonce Misuse-Resistant Authenticated - * Encryption" (https://datatracker.ietf.org/doc/html/rfc8452) + * GHASH is the older variant, defined as part of GCM in NIST SP 800-38D + * (https://nvlpubs.nist.gov/nistpubs/legacy/sp/nistspecialpublication800-38d.pdf). + * GHASH is hard to implement directly, due to its backwards mapping between + * bits and polynomial coefficients. GHASH implementations typically pre and + * post-process the inputs and outputs (mainly by byte-swapping) to convert the + * GHASH computation into an equivalent computation over a different, + * easier-to-use representation of GF(2^128). * - * POLYVAL is also used by HCTR2. See "Length-preserving encryption with HCTR2" - * (https://eprint.iacr.org/2021/1441.pdf). + * POLYVAL is a newer GF(2^128) polynomial hash, originally defined as part of + * AES-GCM-SIV (https://datatracker.ietf.org/doc/html/rfc8452) and also used by + * HCTR2 (https://eprint.iacr.org/2021/1441.pdf). It uses that easier-to-use + * field representation directly, eliminating the data conversion steps. * - * This file provides a library API for POLYVAL. This API can delegate to - * either a generic implementation or an architecture-optimized implementation. + * This file provides library APIs for GHASH and POLYVAL. These APIs can + * delegate to either a generic implementation or an architecture-optimized + * implementation. Due to the mathematical relationship between GHASH and + * POLYVAL, in some cases code for one is reused with the other. * * For the generic implementation, we don't use the traditional table approach * to GF(2^128) multiplication. That approach is not constant-time and requires @@ -205,6 +212,19 @@ polyval_mul_generic(struct polyval_elem *a, const struct polyval_elem *b) a->hi = cpu_to_le64(c3); } +static void __maybe_unused ghash_blocks_generic(struct polyval_elem *acc, + const struct polyval_elem *key, + const u8 *data, size_t nblocks) +{ + do { + acc->lo ^= + cpu_to_le64(get_unaligned_be64((__be64 *)(data + 8))); + acc->hi ^= cpu_to_le64(get_unaligned_be64((__be64 *)data)); + polyval_mul_generic(acc, key); + data += GHASH_BLOCK_SIZE; + } while (--nblocks); +} + static void __maybe_unused polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, const u8 *data, size_t nblocks) @@ -217,10 +237,108 @@ polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, } while (--nblocks); } +/* Convert the key from GHASH format to POLYVAL format. */ +static void __maybe_unused ghash_key_to_polyval(const u8 in[GHASH_BLOCK_SIZE], + struct polyval_elem *out) +{ + u64 hi = get_unaligned_be64(&in[0]); + u64 lo = get_unaligned_be64(&in[8]); + u64 mask = (s64)hi >> 63; + + hi = (hi << 1) ^ (lo >> 63) ^ (mask & ((u64)0xc2 << 56)); + lo = (lo << 1) ^ (mask & 1); + out->lo = cpu_to_le64(lo); + out->hi = cpu_to_le64(hi); +} + +/* Convert the accumulator from POLYVAL format to GHASH format. */ +static void polyval_acc_to_ghash(const struct polyval_elem *in, + u8 out[GHASH_BLOCK_SIZE]) +{ + put_unaligned_be64(le64_to_cpu(in->hi), &out[0]); + put_unaligned_be64(le64_to_cpu(in->lo), &out[8]); +} + +/* Convert the accumulator from GHASH format to POLYVAL format. */ +static void __maybe_unused ghash_acc_to_polyval(const u8 in[GHASH_BLOCK_SIZE], + struct polyval_elem *out) +{ + out->lo = cpu_to_le64(get_unaligned_be64(&in[8])); + out->hi = cpu_to_le64(get_unaligned_be64(&in[0])); +} + #ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH #include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */ #endif +void ghash_preparekey(struct ghash_key *key, const u8 raw_key[GHASH_BLOCK_SIZE]) +{ +#ifdef ghash_preparekey_arch + ghash_preparekey_arch(key, raw_key); +#else + ghash_key_to_polyval(raw_key, &key->h); +#endif +} +EXPORT_SYMBOL_GPL(ghash_preparekey); + +static void ghash_mul(struct ghash_ctx *ctx) +{ +#ifdef ghash_mul_arch + ghash_mul_arch(&ctx->acc, ctx->key); +#elif defined(ghash_blocks_arch) + static const u8 zeroes[GHASH_BLOCK_SIZE]; + + ghash_blocks_arch(&ctx->acc, ctx->key, zeroes, 1); +#else + polyval_mul_generic(&ctx->acc, &ctx->key->h); +#endif +} + +/* nblocks is always >= 1. */ +static void ghash_blocks(struct ghash_ctx *ctx, const u8 *data, size_t nblocks) +{ +#ifdef ghash_blocks_arch + ghash_blocks_arch(&ctx->acc, ctx->key, data, nblocks); +#else + ghash_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); +#endif +} + +void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len) +{ + if (unlikely(ctx->partial)) { + size_t n = min(len, GHASH_BLOCK_SIZE - ctx->partial); + + len -= n; + while (n--) + ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - ctx->partial++] ^= + *data++; + if (ctx->partial < GHASH_BLOCK_SIZE) + return; + ghash_mul(ctx); + } + if (len >= GHASH_BLOCK_SIZE) { + size_t nblocks = len / GHASH_BLOCK_SIZE; + + ghash_blocks(ctx, data, nblocks); + data += len & ~(GHASH_BLOCK_SIZE - 1); + len &= GHASH_BLOCK_SIZE - 1; + } + for (size_t i = 0; i < len; i++) + ctx->acc.bytes[GHASH_BLOCK_SIZE - 1 - i] ^= data[i]; + ctx->partial = len; +} +EXPORT_SYMBOL_GPL(ghash_update); + +void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE]) +{ + if (unlikely(ctx->partial)) + ghash_mul(ctx); + polyval_acc_to_ghash(&ctx->acc, out); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(ghash_final); + void polyval_preparekey(struct polyval_key *key, const u8 raw_key[POLYVAL_BLOCK_SIZE]) { @@ -253,6 +371,7 @@ static void polyval_mul(struct polyval_ctx *ctx) #endif } +/* nblocks is always >= 1. */ static void polyval_blocks(struct polyval_ctx *ctx, const u8 *data, size_t nblocks) { From 75e34bef53251744d95fd242b0345122fa462c7b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:05 -0700 Subject: [PATCH 22/66] lib/crypto: tests: Add KUnit tests for GHASH Add a KUnit test suite for the GHASH library functions. It closely mirrors the POLYVAL test suite. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-5-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/.kunitconfig | 1 + lib/crypto/tests/Kconfig | 8 ++ lib/crypto/tests/Makefile | 1 + lib/crypto/tests/ghash-testvecs.h | 186 ++++++++++++++++++++++++++ lib/crypto/tests/ghash_kunit.c | 194 ++++++++++++++++++++++++++++ scripts/crypto/gen-hash-testvecs.py | 63 ++++++++- 6 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 lib/crypto/tests/ghash-testvecs.h create mode 100644 lib/crypto/tests/ghash_kunit.c diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig index 63a592731d1d..391836511c8b 100644 --- a/lib/crypto/.kunitconfig +++ b/lib/crypto/.kunitconfig @@ -6,6 +6,7 @@ CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST=y CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST=y CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST=y CONFIG_CRYPTO_LIB_NH_KUNIT_TEST=y diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index aa627b6b9855..5b60d5c3644b 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -35,6 +35,14 @@ config CRYPTO_LIB_CURVE25519_KUNIT_TEST help KUnit tests for the Curve25519 Diffie-Hellman function. +config CRYPTO_LIB_GHASH_KUNIT_TEST + tristate "KUnit tests for GHASH" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_GF128HASH + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the GHASH library functions. + config CRYPTO_LIB_MD5_KUNIT_TEST tristate "KUnit tests for MD5" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_MD5 diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile index f864e0ffbee4..751ae507fdd0 100644 --- a/lib/crypto/tests/Makefile +++ b/lib/crypto/tests/Makefile @@ -4,6 +4,7 @@ obj-$(CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST) += aes_cbc_macs_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o obj-$(CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST) += curve25519_kunit.o +obj-$(CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST) += ghash_kunit.o obj-$(CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST) += md5_kunit.o obj-$(CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST) += mldsa_kunit.o obj-$(CONFIG_CRYPTO_LIB_NH_KUNIT_TEST) += nh_kunit.o diff --git a/lib/crypto/tests/ghash-testvecs.h b/lib/crypto/tests/ghash-testvecs.h new file mode 100644 index 000000000000..759eb4072336 --- /dev/null +++ b/lib/crypto/tests/ghash-testvecs.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py ghash */ + +static const struct { + size_t data_len; + u8 digest[GHASH_DIGEST_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + }, + { + .data_len = 1, + .digest = { + 0x13, 0x91, 0xa1, 0x11, 0x08, 0xc3, 0x7e, 0xeb, + 0x21, 0x42, 0x4a, 0xd6, 0x45, 0x0f, 0x41, 0xa7, + }, + }, + { + .data_len = 2, + .digest = { + 0xde, 0x00, 0x63, 0x3f, 0x71, 0x0f, 0xc6, 0x29, + 0x53, 0x2e, 0x49, 0xd9, 0xc2, 0xb7, 0x73, 0xce, + }, + }, + { + .data_len = 3, + .digest = { + 0xcf, 0xc7, 0xa8, 0x20, 0x24, 0xe9, 0x7a, 0x6c, + 0x2c, 0x2a, 0x34, 0x70, 0x26, 0xba, 0xd5, 0x9a, + }, + }, + { + .data_len = 16, + .digest = { + 0xaa, 0xe0, 0xdc, 0x7f, 0xcf, 0x8b, 0xe6, 0x0c, + 0x2e, 0x93, 0x89, 0x7d, 0x68, 0x4e, 0xc2, 0x63, + }, + }, + { + .data_len = 32, + .digest = { + 0x4b, 0x8b, 0x93, 0x5c, 0x79, 0xad, 0x85, 0x08, + 0xd3, 0x8a, 0xcd, 0xdd, 0x4c, 0x6e, 0x0e, 0x6f, + }, + }, + { + .data_len = 48, + .digest = { + 0xfa, 0xa0, 0x25, 0xdd, 0x61, 0x9a, 0x52, 0x9a, + 0xea, 0xee, 0xc6, 0x62, 0xb2, 0xba, 0x11, 0x49, + }, + }, + { + .data_len = 49, + .digest = { + 0x23, 0xf1, 0x05, 0xeb, 0x30, 0x40, 0xb9, 0x1d, + 0xe6, 0x35, 0x51, 0x4e, 0x0f, 0xc0, 0x1b, 0x9e, + }, + }, + { + .data_len = 63, + .digest = { + 0x8d, 0xcf, 0xa0, 0xc8, 0x83, 0x21, 0x06, 0x81, + 0xc6, 0x36, 0xd5, 0x62, 0xbf, 0xa0, 0xcd, 0x9c, + }, + }, + { + .data_len = 64, + .digest = { + 0xe7, 0xca, 0xbe, 0xe7, 0x66, 0xc8, 0x85, 0xad, + 0xbc, 0xaf, 0x58, 0x21, 0xd7, 0x67, 0x82, 0x15, + }, + }, + { + .data_len = 65, + .digest = { + 0x9f, 0x48, 0x10, 0xd9, 0xa2, 0x6b, 0x9d, 0xe0, + 0xb1, 0x87, 0xe1, 0x39, 0xc3, 0xd7, 0xee, 0x09, + }, + }, + { + .data_len = 127, + .digest = { + 0xa4, 0x36, 0xb7, 0x82, 0xd2, 0x67, 0x7e, 0xaf, + 0x5d, 0xfd, 0x67, 0x9c, 0x1d, 0x9f, 0xe4, 0xf7, + }, + }, + { + .data_len = 128, + .digest = { + 0x57, 0xe7, 0x1d, 0x78, 0xf0, 0x8e, 0xc7, 0x0c, + 0x15, 0xee, 0x18, 0xc4, 0xd1, 0x75, 0x90, 0xaa, + }, + }, + { + .data_len = 129, + .digest = { + 0x9b, 0xad, 0x81, 0xa9, 0x22, 0xb2, 0x19, 0x53, + 0x60, 0x30, 0xe7, 0xa0, 0x4f, 0xd6, 0x72, 0x42, + }, + }, + { + .data_len = 256, + .digest = { + 0xf7, 0x33, 0x42, 0xbf, 0x58, 0xde, 0x88, 0x0f, + 0x8d, 0x3d, 0xa6, 0x11, 0x14, 0xc3, 0xf1, 0xdc, + }, + }, + { + .data_len = 511, + .digest = { + 0x59, 0xdc, 0xa9, 0xc0, 0x4e, 0xd6, 0x97, 0xb3, + 0x60, 0xaf, 0xa8, 0xa0, 0xea, 0x54, 0x8e, 0xc3, + }, + }, + { + .data_len = 513, + .digest = { + 0xa2, 0x23, 0x37, 0xcc, 0x97, 0xec, 0xea, 0xbe, + 0xd6, 0xc7, 0x13, 0xf7, 0x93, 0x73, 0xc0, 0x64, + }, + }, + { + .data_len = 1000, + .digest = { + 0x46, 0x8b, 0x43, 0x77, 0x9b, 0xc2, 0xfc, 0xa4, + 0x68, 0x6a, 0x6c, 0x07, 0xa4, 0x6f, 0x47, 0x65, + }, + }, + { + .data_len = 3333, + .digest = { + 0x69, 0x7f, 0x19, 0xc3, 0xb9, 0xa4, 0xff, 0x40, + 0xe3, 0x03, 0x71, 0xa3, 0x88, 0x8a, 0xf1, 0xbd, + }, + }, + { + .data_len = 4096, + .digest = { + 0x4d, 0x65, 0xe6, 0x9c, 0xeb, 0x6a, 0x46, 0x8d, + 0xe9, 0x32, 0x96, 0x72, 0xb3, 0x0d, 0x08, 0xa9, + }, + }, + { + .data_len = 4128, + .digest = { + 0xfc, 0xa1, 0x74, 0x46, 0x21, 0x64, 0xa7, 0x64, + 0xbe, 0x47, 0x03, 0x1e, 0x05, 0xf7, 0xd8, 0x37, + }, + }, + { + .data_len = 4160, + .digest = { + 0x70, 0x5b, 0xe9, 0x17, 0xab, 0xd5, 0xa2, 0xee, + 0xcb, 0x39, 0xa4, 0x81, 0x2f, 0x41, 0x70, 0xae, + }, + }, + { + .data_len = 4224, + .digest = { + 0x07, 0xbd, 0xb6, 0x52, 0xe2, 0x75, 0x2c, 0x33, + 0x6d, 0x1b, 0x63, 0x56, 0x58, 0xda, 0x98, 0x55, + }, + }, + { + .data_len = 16384, + .digest = { + 0x9c, 0xb5, 0xf4, 0x14, 0xe8, 0xa8, 0x4a, 0xde, + 0xee, 0x7b, 0xbb, 0xd6, 0x21, 0x6d, 0x6a, 0x69, + }, + }, +}; + +static const u8 hash_testvec_consolidated[GHASH_DIGEST_SIZE] = { + 0x08, 0xef, 0xf5, 0x27, 0xb1, 0xca, 0xd4, 0x1d, + 0xad, 0x38, 0x69, 0x88, 0x6b, 0x16, 0xdf, 0xa8, +}; + +static const u8 ghash_allones_hashofhashes[GHASH_DIGEST_SIZE] = { + 0xef, 0x85, 0x58, 0xf8, 0x54, 0x9c, 0x5e, 0x54, + 0xd9, 0xbe, 0x04, 0x1f, 0xff, 0xff, 0xff, 0xff, +}; diff --git a/lib/crypto/tests/ghash_kunit.c b/lib/crypto/tests/ghash_kunit.c new file mode 100644 index 000000000000..68b3837a3607 --- /dev/null +++ b/lib/crypto/tests/ghash_kunit.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include +#include "ghash-testvecs.h" + +/* + * A fixed key used when presenting GHASH as an unkeyed hash function in order + * to reuse hash-test-template.h. At the beginning of the test suite, this is + * initialized to a key prepared from bytes generated from a fixed seed. + */ +static struct ghash_key test_key; + +static void ghash_init_withtestkey(struct ghash_ctx *ctx) +{ + ghash_init(ctx, &test_key); +} + +static void ghash_withtestkey(const u8 *data, size_t len, + u8 out[GHASH_BLOCK_SIZE]) +{ + ghash(&test_key, data, len, out); +} + +/* Generate the HASH_KUNIT_CASES using hash-test-template.h. */ +#define HASH ghash_withtestkey +#define HASH_CTX ghash_ctx +#define HASH_SIZE GHASH_BLOCK_SIZE +#define HASH_INIT ghash_init_withtestkey +#define HASH_UPDATE ghash_update +#define HASH_FINAL ghash_final +#include "hash-test-template.h" + +/* + * Test a key and messages containing all one bits. This is useful to detect + * overflow bugs in implementations that emulate carryless multiplication using + * a series of standard multiplications with the bits spread out. + */ +static void test_ghash_allones_key_and_message(struct kunit *test) +{ + struct ghash_key key; + struct ghash_ctx hashofhashes_ctx; + u8 hash[GHASH_BLOCK_SIZE]; + + static_assert(TEST_BUF_LEN >= 4096); + memset(test_buf, 0xff, 4096); + + ghash_preparekey(&key, test_buf); + ghash_init(&hashofhashes_ctx, &key); + for (size_t len = 0; len <= 4096; len += 16) { + ghash(&key, test_buf, len, hash); + ghash_update(&hashofhashes_ctx, hash, sizeof(hash)); + } + ghash_final(&hashofhashes_ctx, hash); + KUNIT_ASSERT_MEMEQ(test, hash, ghash_allones_hashofhashes, + sizeof(hash)); +} + +#define MAX_LEN_FOR_KEY_CHECK 1024 + +/* + * Given two prepared keys which should be identical (but may differ in + * alignment and/or whether they are followed by a guard page or not), verify + * that they produce consistent results on various data lengths. + */ +static void check_key_consistency(struct kunit *test, + const struct ghash_key *key1, + const struct ghash_key *key2) +{ + u8 *data = test_buf; + u8 hash1[GHASH_BLOCK_SIZE]; + u8 hash2[GHASH_BLOCK_SIZE]; + + rand_bytes(data, MAX_LEN_FOR_KEY_CHECK); + KUNIT_ASSERT_MEMEQ(test, key1, key2, sizeof(*key1)); + + for (int i = 0; i < 100; i++) { + size_t len = rand_length(MAX_LEN_FOR_KEY_CHECK); + + ghash(key1, data, len, hash1); + ghash(key2, data, len, hash2); + KUNIT_ASSERT_MEMEQ(test, hash1, hash2, sizeof(hash1)); + } +} + +/* Test that no buffer overreads occur on either raw_key or ghash_key. */ +static void test_ghash_with_guarded_key(struct kunit *test) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + u8 *guarded_raw_key = &test_buf[TEST_BUF_LEN - sizeof(raw_key)]; + struct ghash_key key1, key2; + struct ghash_key *guarded_key = + (struct ghash_key *)&test_buf[TEST_BUF_LEN - sizeof(key1)]; + + /* Prepare with regular buffers. */ + rand_bytes(raw_key, sizeof(raw_key)); + ghash_preparekey(&key1, raw_key); + + /* Prepare with guarded raw_key, then check that it works. */ + memcpy(guarded_raw_key, raw_key, sizeof(raw_key)); + ghash_preparekey(&key2, guarded_raw_key); + check_key_consistency(test, &key1, &key2); + + /* Prepare guarded ghash_key, then check that it works. */ + ghash_preparekey(guarded_key, raw_key); + check_key_consistency(test, &key1, guarded_key); +} + +/* + * Test that ghash_key only needs to be aligned to + * __alignof__(struct ghash_key), i.e. 8 bytes. The assembly code may prefer + * 16-byte or higher alignment, but it mustn't require it. + */ +static void test_ghash_with_minimally_aligned_key(struct kunit *test) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + struct ghash_key key; + struct ghash_key *minaligned_key = + (struct ghash_key *)&test_buf[MAX_LEN_FOR_KEY_CHECK + + __alignof__(struct ghash_key)]; + + KUNIT_ASSERT_TRUE(test, IS_ALIGNED((uintptr_t)minaligned_key, + __alignof__(struct ghash_key))); + KUNIT_ASSERT_TRUE(test, !IS_ALIGNED((uintptr_t)minaligned_key, + 2 * __alignof__(struct ghash_key))); + + rand_bytes(raw_key, sizeof(raw_key)); + ghash_preparekey(&key, raw_key); + ghash_preparekey(minaligned_key, raw_key); + check_key_consistency(test, &key, minaligned_key); +} + +struct ghash_irq_test_state { + struct ghash_key expected_key; + u8 raw_key[GHASH_BLOCK_SIZE]; +}; + +static bool ghash_irq_test_func(void *state_) +{ + struct ghash_irq_test_state *state = state_; + struct ghash_key key; + + ghash_preparekey(&key, state->raw_key); + return memcmp(&key, &state->expected_key, sizeof(key)) == 0; +} + +/* + * Test that ghash_preparekey() produces the same output regardless of whether + * FPU or vector registers are usable when it is called. + */ +static void test_ghash_preparekey_in_irqs(struct kunit *test) +{ + struct ghash_irq_test_state state; + + rand_bytes(state.raw_key, sizeof(state.raw_key)); + ghash_preparekey(&state.expected_key, state.raw_key); + kunit_run_irq_test(test, ghash_irq_test_func, 200000, &state); +} + +static int ghash_suite_init(struct kunit_suite *suite) +{ + u8 raw_key[GHASH_BLOCK_SIZE]; + + rand_bytes_seeded_from_len(raw_key, sizeof(raw_key)); + ghash_preparekey(&test_key, raw_key); + return hash_suite_init(suite); +} + +static void ghash_suite_exit(struct kunit_suite *suite) +{ + hash_suite_exit(suite); +} + +static struct kunit_case ghash_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(test_ghash_allones_key_and_message), + KUNIT_CASE(test_ghash_with_guarded_key), + KUNIT_CASE(test_ghash_with_minimally_aligned_key), + KUNIT_CASE(test_ghash_preparekey_in_irqs), + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite ghash_test_suite = { + .name = "ghash", + .test_cases = ghash_test_cases, + .suite_init = ghash_suite_init, + .suite_exit = ghash_suite_exit, +}; +kunit_test_suite(ghash_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for GHASH"); +MODULE_LICENSE("GPL"); diff --git a/scripts/crypto/gen-hash-testvecs.py b/scripts/crypto/gen-hash-testvecs.py index 34b7c48f3456..e69ce213fb33 100755 --- a/scripts/crypto/gen-hash-testvecs.py +++ b/scripts/crypto/gen-hash-testvecs.py @@ -68,6 +68,52 @@ class Poly1305: m = (self.h + self.s) % 2**128 return m.to_bytes(16, byteorder='little') +GHASH_POLY = sum((1 << i) for i in [128, 7, 2, 1, 0]) +GHASH_BLOCK_SIZE = 16 + +# A straightforward, unoptimized implementation of GHASH. +class Ghash: + + @staticmethod + def reflect_bits_in_bytes(v): + res = 0 + for offs in range(0, 128, 8): + for bit in range(8): + if (v & (1 << (offs + bit))) != 0: + res ^= 1 << (offs + 7 - bit) + return res + + @staticmethod + def bytes_to_poly(data): + return Ghash.reflect_bits_in_bytes(int.from_bytes(data, byteorder='little')) + + @staticmethod + def poly_to_bytes(poly): + return Ghash.reflect_bits_in_bytes(poly).to_bytes(16, byteorder='little') + + def __init__(self, key): + assert len(key) == 16 + self.h = Ghash.bytes_to_poly(key) + self.acc = 0 + + # Note: this supports partial blocks only at the end. + def update(self, data): + for i in range(0, len(data), 16): + # acc += block + self.acc ^= Ghash.bytes_to_poly(data[i:i+16]) + # acc = (acc * h) mod GHASH_POLY + product = 0 + for j in range(127, -1, -1): + if (self.h & (1 << j)) != 0: + product ^= self.acc << j + if (product & (1 << (128 + j))) != 0: + product ^= GHASH_POLY << j + self.acc = product + return self + + def digest(self): + return Ghash.poly_to_bytes(self.acc) + POLYVAL_POLY = sum((1 << i) for i in [128, 127, 126, 121, 0]) POLYVAL_BLOCK_SIZE = 16 @@ -103,6 +149,8 @@ def hash_init(alg): # unkeyed hash functions to work on them. if alg == 'aes-cmac': return AesCmac(rand_bytes(AES_256_KEY_SIZE)) + if alg == 'ghash': + return Ghash(rand_bytes(GHASH_BLOCK_SIZE)) if alg == 'poly1305': return Poly1305(rand_bytes(POLY1305_KEY_SIZE)) if alg == 'polyval': @@ -257,6 +305,15 @@ def gen_additional_poly1305_testvecs(): 'poly1305_allones_macofmacs[POLY1305_DIGEST_SIZE]', Poly1305(key).update(data).digest()) +def gen_additional_ghash_testvecs(): + key = b'\xff' * GHASH_BLOCK_SIZE + hashes = b'' + for data_len in range(0, 4097, 16): + hashes += Ghash(key).update(b'\xff' * data_len).digest() + print_static_u8_array_definition( + 'ghash_allones_hashofhashes[GHASH_DIGEST_SIZE]', + Ghash(key).update(hashes).digest()) + def gen_additional_polyval_testvecs(): key = b'\xff' * POLYVAL_BLOCK_SIZE hashes = b'' @@ -268,7 +325,8 @@ def gen_additional_polyval_testvecs(): if len(sys.argv) != 2: sys.stderr.write('Usage: gen-hash-testvecs.py ALGORITHM\n') - sys.stderr.write('ALGORITHM may be any supported by Python hashlib; or poly1305, polyval, or sha3.\n') + sys.stderr.write('ALGORITHM may be any supported by Python hashlib;\n') + sys.stderr.write(' or aes-cmac, ghash, nh, poly1305, polyval, or sha3.\n') sys.stderr.write('Example: gen-hash-testvecs.py sha512\n') sys.exit(1) @@ -280,6 +338,9 @@ if alg == 'aes-cmac': elif alg.startswith('blake2'): gen_unkeyed_testvecs(alg) gen_additional_blake2_testvecs(alg) +elif alg == 'ghash': + gen_unkeyed_testvecs(alg) + gen_additional_ghash_testvecs() elif alg == 'nh': gen_nh_testvecs() elif alg == 'poly1305': From 39afaff983fe64ea749aafb3ebd63d44212d2860 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:06 -0700 Subject: [PATCH 23/66] crypto: arm/ghash - Make the "ghash" crypto_shash NEON-only arch/arm/crypto/ghash-ce-glue.c originally provided only a "ghash" crypto_shash algorithm using PMULL if available, else NEON. Significantly later, it was updated to also provide a full AES-GCM implementation using PMULL. This made the PMULL support in the "ghash" crypto_shash largely obsolete. Indeed, the arm64 equivalent of this file unconditionally uses only ASIMD in its "ghash" crypto_shash. Given that inconsistency and the fact that the NEON-only code is more easily separable into the GHASH library than the PMULL based code is, let's align with arm64 and just support NEON-only for the pure GHASH. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-6-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm/crypto/ghash-ce-glue.c | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index 454adcc62cc6..d7d787de7dd3 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -36,7 +36,7 @@ MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); struct ghash_key { be128 k; - u64 h[][2]; + u64 h[1][2]; }; struct gcm_key { @@ -51,12 +51,10 @@ struct arm_ghash_desc_ctx { }; asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, - u64 const h[][2], const char *head); + u64 const h[4][2], const char *head); asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, - u64 const h[][2], const char *head); - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64); + u64 const h[1][2], const char *head); static int ghash_init(struct shash_desc *desc) { @@ -70,10 +68,7 @@ static void ghash_do_update(int blocks, u64 dg[], const char *src, struct ghash_key *key, const char *head) { kernel_neon_begin(); - if (static_branch_likely(&use_p64)) - pmull_ghash_update_p64(blocks, dg, src, key->h, head); - else - pmull_ghash_update_p8(blocks, dg, src, key->h, head); + pmull_ghash_update_p8(blocks, dg, src, key->h, head); kernel_neon_end(); } @@ -147,19 +142,6 @@ static int ghash_setkey(struct crypto_shash *tfm, /* needed for the fallback */ memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); ghash_reflect(key->h[0], &key->k); - - if (static_branch_likely(&use_p64)) { - be128 h = key->k; - - gf128mul_lle(&h, &key->k); - ghash_reflect(key->h[1], &h); - - gf128mul_lle(&h, &key->k); - ghash_reflect(key->h[2], &h); - - gf128mul_lle(&h, &key->k); - ghash_reflect(key->h[3], &h); - } return 0; } @@ -175,11 +157,11 @@ static struct shash_alg ghash_alg = { .statesize = sizeof(struct ghash_desc_ctx), .base.cra_name = "ghash", - .base.cra_driver_name = "ghash-ce", + .base.cra_driver_name = "ghash-neon", .base.cra_priority = 300, .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .base.cra_blocksize = GHASH_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), + .base.cra_ctxsize = sizeof(struct ghash_key), .base.cra_module = THIS_MODULE, }; @@ -571,8 +553,6 @@ static int __init ghash_ce_mod_init(void) ARRAY_SIZE(gcm_aes_algs)); if (err) return err; - ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]); - static_branch_enable(&use_p64); } err = crypto_register_shash(&ghash_alg); From ca5ff14c1a70e7eeff5705105554ce8bac643937 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:07 -0700 Subject: [PATCH 24/66] crypto: arm/ghash - Move NEON GHASH assembly into its own file arch/arm/crypto/ghash-ce-core.S implements pmull_ghash_update_p8(), which is used only by a crypto_shash implementation of GHASH. It also implements other functions, including pmull_ghash_update_p64() and others, which are used only by a crypto_aead implementation of AES-GCM. While some code is shared between pmull_ghash_update_p8() and pmull_ghash_update_p64(), it's not very much. Since pmull_ghash_update_p8() will also need to be migrated into lib/crypto/ to achieve parity in the standalone GHASH support, let's move it into a separate file ghash-neon-core.S. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-7-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm/crypto/Makefile | 2 +- arch/arm/crypto/ghash-ce-core.S | 171 ++---------------------- arch/arm/crypto/ghash-neon-core.S | 207 ++++++++++++++++++++++++++++++ 3 files changed, 222 insertions(+), 158 deletions(-) create mode 100644 arch/arm/crypto/ghash-neon-core.S diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index e73099e120b3..cedce94d5ee5 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -10,4 +10,4 @@ obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o -ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o +ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o ghash-neon-core.o diff --git a/arch/arm/crypto/ghash-ce-core.S b/arch/arm/crypto/ghash-ce-core.S index 858c0d66798b..a449525d61f8 100644 --- a/arch/arm/crypto/ghash-ce-core.S +++ b/arch/arm/crypto/ghash-ce-core.S @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Accelerated GHASH implementation with NEON/ARMv8 vmull.p8/64 instructions. + * Accelerated AES-GCM implementation with ARMv8 Crypto Extensions. * * Copyright (C) 2015 - 2017 Linaro Ltd. * Copyright (C) 2023 Google LLC. @@ -29,39 +29,10 @@ XM_H .req d7 XH_L .req d8 - t0l .req d10 - t0h .req d11 - t1l .req d12 - t1h .req d13 - t2l .req d14 - t2h .req d15 - t3l .req d16 - t3h .req d17 - t4l .req d18 - t4h .req d19 - - t0q .req q5 - t1q .req q6 - t2q .req q7 - t3q .req q8 - t4q .req q9 XH2 .req q9 - s1l .req d20 - s1h .req d21 - s2l .req d22 - s2h .req d23 - s3l .req d24 - s3h .req d25 - s4l .req d26 - s4h .req d27 - MASK .req d28 - SHASH2_p8 .req d28 - k16 .req d29 - k32 .req d30 - k48 .req d31 SHASH2_p64 .req d31 HH .req q10 @@ -93,72 +64,6 @@ .text - .macro __pmull_p64, rd, rn, rm, b1, b2, b3, b4 - vmull.p64 \rd, \rn, \rm - .endm - - /* - * This implementation of 64x64 -> 128 bit polynomial multiplication - * using vmull.p8 instructions (8x8 -> 16) is taken from the paper - * "Fast Software Polynomial Multiplication on ARM Processors Using - * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and - * Ricardo Dahab (https://hal.inria.fr/hal-01506572) - * - * It has been slightly tweaked for in-order performance, and to allow - * 'rq' to overlap with 'ad' or 'bd'. - */ - .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l - vext.8 t0l, \ad, \ad, #1 @ A1 - .ifc \b1, t4l - vext.8 t4l, \bd, \bd, #1 @ B1 - .endif - vmull.p8 t0q, t0l, \bd @ F = A1*B - vext.8 t1l, \ad, \ad, #2 @ A2 - vmull.p8 t4q, \ad, \b1 @ E = A*B1 - .ifc \b2, t3l - vext.8 t3l, \bd, \bd, #2 @ B2 - .endif - vmull.p8 t1q, t1l, \bd @ H = A2*B - vext.8 t2l, \ad, \ad, #3 @ A3 - vmull.p8 t3q, \ad, \b2 @ G = A*B2 - veor t0q, t0q, t4q @ L = E + F - .ifc \b3, t4l - vext.8 t4l, \bd, \bd, #3 @ B3 - .endif - vmull.p8 t2q, t2l, \bd @ J = A3*B - veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 - veor t1q, t1q, t3q @ M = G + H - .ifc \b4, t3l - vext.8 t3l, \bd, \bd, #4 @ B4 - .endif - vmull.p8 t4q, \ad, \b3 @ I = A*B3 - veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 - vmull.p8 t3q, \ad, \b4 @ K = A*B4 - vand t0h, t0h, k48 - vand t1h, t1h, k32 - veor t2q, t2q, t4q @ N = I + J - veor t0l, t0l, t0h - veor t1l, t1l, t1h - veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 - vand t2h, t2h, k16 - veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 - vmov.i64 t3h, #0 - vext.8 t0q, t0q, t0q, #15 - veor t2l, t2l, t2h - vext.8 t1q, t1q, t1q, #14 - vmull.p8 \rq, \ad, \bd @ D = A*B - vext.8 t2q, t2q, t2q, #13 - vext.8 t3q, t3q, t3q, #12 - veor t0q, t0q, t1q - veor t2q, t2q, t3q - veor \rq, \rq, t0q - veor \rq, \rq, t2q - .endm - - // - // PMULL (64x64->128) based reduction for CPUs that can do - // it in a single instruction. - // .macro __pmull_reduce_p64 vmull.p64 T1, XL_L, MASK @@ -170,30 +75,7 @@ vmull.p64 XL, T1_H, MASK .endm - // - // Alternative reduction for CPUs that lack support for the - // 64x64->128 PMULL instruction - // - .macro __pmull_reduce_p8 - veor XL_H, XL_H, XM_L - veor XH_L, XH_L, XM_H - - vshl.i64 T1, XL, #57 - vshl.i64 T2, XL, #62 - veor T1, T1, T2 - vshl.i64 T2, XL, #63 - veor T1, T1, T2 - veor XL_H, XL_H, T1_L - veor XH_L, XH_L, T1_H - - vshr.u64 T1, XL, #1 - veor XH, XH, XL - veor XL, XL, T1 - vshr.u64 T1, T1, #6 - vshr.u64 XL, XL, #1 - .endm - - .macro ghash_update, pn, enc, aggregate=1, head=1 + .macro ghash_update, enc, aggregate=1, head=1 vld1.64 {XL}, [r1] .if \head @@ -206,8 +88,7 @@ b 3f .endif -0: .ifc \pn, p64 - .if \aggregate +0: .if \aggregate tst r0, #3 // skip until #blocks is a bne 2f // round multiple of 4 @@ -288,7 +169,6 @@ b 1b .endif - .endif 2: vld1.8 {T1}, [r2]! @@ -308,15 +188,15 @@ veor T1_L, T1_L, XL_H veor XL, XL, IN1 - __pmull_\pn XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 + vmull.p64 XH, XL_H, SHASH_H @ a1 * b1 veor T1, T1, XL - __pmull_\pn XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 - __pmull_\pn XM, T1_L, SHASH2_\pn @ (a1+a0)(b1+b0) + vmull.p64 XL, XL_L, SHASH_L @ a0 * b0 + vmull.p64 XM, T1_L, SHASH2_p64 @ (a1+a0)(b1+b0) 4: veor T1, XL, XH veor XM, XM, T1 - __pmull_reduce_\pn + __pmull_reduce_p64 veor T1, T1, XH veor XL, XL, T1 @@ -325,8 +205,8 @@ .endm /* - * void pmull_ghash_update(int blocks, u64 dg[], const char *src, - * struct ghash_key const *k, const char *head) + * void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, + * u64 const h[4][2], const char *head) */ ENTRY(pmull_ghash_update_p64) vld1.64 {SHASH}, [r3]! @@ -341,35 +221,12 @@ ENTRY(pmull_ghash_update_p64) vmov.i8 MASK, #0xe1 vshl.u64 MASK, MASK, #57 - ghash_update p64 + ghash_update vst1.64 {XL}, [r1] bx lr ENDPROC(pmull_ghash_update_p64) -ENTRY(pmull_ghash_update_p8) - vld1.64 {SHASH}, [r3] - veor SHASH2_p8, SHASH_L, SHASH_H - - vext.8 s1l, SHASH_L, SHASH_L, #1 - vext.8 s2l, SHASH_L, SHASH_L, #2 - vext.8 s3l, SHASH_L, SHASH_L, #3 - vext.8 s4l, SHASH_L, SHASH_L, #4 - vext.8 s1h, SHASH_H, SHASH_H, #1 - vext.8 s2h, SHASH_H, SHASH_H, #2 - vext.8 s3h, SHASH_H, SHASH_H, #3 - vext.8 s4h, SHASH_H, SHASH_H, #4 - - vmov.i64 k16, #0xffff - vmov.i64 k32, #0xffffffff - vmov.i64 k48, #0xffffffffffff - - ghash_update p8 - vst1.64 {XL}, [r1] - - bx lr -ENDPROC(pmull_ghash_update_p8) - e0 .req q9 e1 .req q10 e2 .req q11 @@ -536,7 +393,7 @@ ENTRY(pmull_gcm_encrypt) vld1.64 {SHASH}, [r3] - ghash_update p64, enc, head=0 + ghash_update enc, head=0 vst1.64 {XL}, [r1] pop {r4-r8, pc} @@ -554,7 +411,7 @@ ENTRY(pmull_gcm_decrypt) vld1.64 {SHASH}, [r3] - ghash_update p64, dec, head=0 + ghash_update dec, head=0 vst1.64 {XL}, [r1] pop {r4-r8, pc} @@ -603,7 +460,7 @@ ENTRY(pmull_gcm_enc_final) vshl.u64 MASK, MASK, #57 mov r0, #1 bne 3f // process head block first - ghash_update p64, aggregate=0, head=0 + ghash_update aggregate=0, head=0 vrev64.8 XL, XL vext.8 XL, XL, XL, #8 @@ -660,7 +517,7 @@ ENTRY(pmull_gcm_dec_final) vshl.u64 MASK, MASK, #57 mov r0, #1 bne 3f // process head block first - ghash_update p64, aggregate=0, head=0 + ghash_update aggregate=0, head=0 vrev64.8 XL, XL vext.8 XL, XL, XL, #8 diff --git a/arch/arm/crypto/ghash-neon-core.S b/arch/arm/crypto/ghash-neon-core.S new file mode 100644 index 000000000000..bdf6fb6d063c --- /dev/null +++ b/arch/arm/crypto/ghash-neon-core.S @@ -0,0 +1,207 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Accelerated GHASH implementation with NEON vmull.p8 instructions. + * + * Copyright (C) 2015 - 2017 Linaro Ltd. + * Copyright (C) 2023 Google LLC. + */ + +#include +#include + + .fpu neon + + SHASH .req q0 + T1 .req q1 + XL .req q2 + XM .req q3 + XH .req q4 + IN1 .req q4 + + SHASH_L .req d0 + SHASH_H .req d1 + T1_L .req d2 + T1_H .req d3 + XL_L .req d4 + XL_H .req d5 + XM_L .req d6 + XM_H .req d7 + XH_L .req d8 + + t0l .req d10 + t0h .req d11 + t1l .req d12 + t1h .req d13 + t2l .req d14 + t2h .req d15 + t3l .req d16 + t3h .req d17 + t4l .req d18 + t4h .req d19 + + t0q .req q5 + t1q .req q6 + t2q .req q7 + t3q .req q8 + t4q .req q9 + + s1l .req d20 + s1h .req d21 + s2l .req d22 + s2h .req d23 + s3l .req d24 + s3h .req d25 + s4l .req d26 + s4h .req d27 + + SHASH2_p8 .req d28 + + k16 .req d29 + k32 .req d30 + k48 .req d31 + + T2 .req q7 + + .text + + /* + * This implementation of 64x64 -> 128 bit polynomial multiplication + * using vmull.p8 instructions (8x8 -> 16) is taken from the paper + * "Fast Software Polynomial Multiplication on ARM Processors Using + * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and + * Ricardo Dahab (https://hal.inria.fr/hal-01506572) + * + * It has been slightly tweaked for in-order performance, and to allow + * 'rq' to overlap with 'ad' or 'bd'. + */ + .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l + vext.8 t0l, \ad, \ad, #1 @ A1 + .ifc \b1, t4l + vext.8 t4l, \bd, \bd, #1 @ B1 + .endif + vmull.p8 t0q, t0l, \bd @ F = A1*B + vext.8 t1l, \ad, \ad, #2 @ A2 + vmull.p8 t4q, \ad, \b1 @ E = A*B1 + .ifc \b2, t3l + vext.8 t3l, \bd, \bd, #2 @ B2 + .endif + vmull.p8 t1q, t1l, \bd @ H = A2*B + vext.8 t2l, \ad, \ad, #3 @ A3 + vmull.p8 t3q, \ad, \b2 @ G = A*B2 + veor t0q, t0q, t4q @ L = E + F + .ifc \b3, t4l + vext.8 t4l, \bd, \bd, #3 @ B3 + .endif + vmull.p8 t2q, t2l, \bd @ J = A3*B + veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8 + veor t1q, t1q, t3q @ M = G + H + .ifc \b4, t3l + vext.8 t3l, \bd, \bd, #4 @ B4 + .endif + vmull.p8 t4q, \ad, \b3 @ I = A*B3 + veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16 + vmull.p8 t3q, \ad, \b4 @ K = A*B4 + vand t0h, t0h, k48 + vand t1h, t1h, k32 + veor t2q, t2q, t4q @ N = I + J + veor t0l, t0l, t0h + veor t1l, t1l, t1h + veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24 + vand t2h, t2h, k16 + veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32 + vmov.i64 t3h, #0 + vext.8 t0q, t0q, t0q, #15 + veor t2l, t2l, t2h + vext.8 t1q, t1q, t1q, #14 + vmull.p8 \rq, \ad, \bd @ D = A*B + vext.8 t2q, t2q, t2q, #13 + vext.8 t3q, t3q, t3q, #12 + veor t0q, t0q, t1q + veor t2q, t2q, t3q + veor \rq, \rq, t0q + veor \rq, \rq, t2q + .endm + + .macro __pmull_reduce_p8 + veor XL_H, XL_H, XM_L + veor XH_L, XH_L, XM_H + + vshl.i64 T1, XL, #57 + vshl.i64 T2, XL, #62 + veor T1, T1, T2 + vshl.i64 T2, XL, #63 + veor T1, T1, T2 + veor XL_H, XL_H, T1_L + veor XH_L, XH_L, T1_H + + vshr.u64 T1, XL, #1 + veor XH, XH, XL + veor XL, XL, T1 + vshr.u64 T1, T1, #6 + vshr.u64 XL, XL, #1 + .endm + + .macro ghash_update + vld1.64 {XL}, [r1] + + /* do the head block first, if supplied */ + ldr ip, [sp] + teq ip, #0 + beq 0f + vld1.64 {T1}, [ip] + teq r0, #0 + b 3f + +0: + vld1.8 {T1}, [r2]! + subs r0, r0, #1 + +3: /* multiply XL by SHASH in GF(2^128) */ + vrev64.8 T1, T1 + + vext.8 IN1, T1, T1, #8 + veor T1_L, T1_L, XL_H + veor XL, XL, IN1 + + __pmull_p8 XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1 + veor T1, T1, XL + __pmull_p8 XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0 + __pmull_p8 XM, T1_L, SHASH2_p8 @ (a1+a0)(b1+b0) + + veor T1, XL, XH + veor XM, XM, T1 + + __pmull_reduce_p8 + + veor T1, T1, XH + veor XL, XL, T1 + + bne 0b + .endm + + /* + * void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, + * u64 const h[1][2], const char *head) + */ +ENTRY(pmull_ghash_update_p8) + vld1.64 {SHASH}, [r3] + veor SHASH2_p8, SHASH_L, SHASH_H + + vext.8 s1l, SHASH_L, SHASH_L, #1 + vext.8 s2l, SHASH_L, SHASH_L, #2 + vext.8 s3l, SHASH_L, SHASH_L, #3 + vext.8 s4l, SHASH_L, SHASH_L, #4 + vext.8 s1h, SHASH_H, SHASH_H, #1 + vext.8 s2h, SHASH_H, SHASH_H, #2 + vext.8 s3h, SHASH_H, SHASH_H, #3 + vext.8 s4h, SHASH_H, SHASH_H, #4 + + vmov.i64 k16, #0xffff + vmov.i64 k32, #0xffffffff + vmov.i64 k48, #0xffffffffffff + + ghash_update + vst1.64 {XL}, [r1] + + bx lr +ENDPROC(pmull_ghash_update_p8) From 71e59795c9f65a30416ed719b4b4da585df3903a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:08 -0700 Subject: [PATCH 25/66] lib/crypto: arm/ghash: Migrate optimized code into library Remove the "ghash-neon" crypto_shash algorithm. Move the corresponding assembly code into lib/crypto/, and wire it up to the GHASH library. This makes the GHASH library be optimized on arm (though only with NEON, not PMULL; for now the goal is just parity with crypto_shash). It greatly reduces the amount of arm-specific glue code that is needed, and it fixes the issue where this optimization was disabled by default. To integrate the assembly code correctly with the library, make the following tweaks: - Change the type of 'blocks' from int to size_t. - Change the types of 'dg' and 'h' to polyval_elem. Note that this simply reflects the format that the code was already using, at least on little endian CPUs. For big endian CPUs, add byte-swaps. - Remove the 'head' argument, which is no longer needed. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-8-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm/crypto/Kconfig | 13 +- arch/arm/crypto/Makefile | 2 +- arch/arm/crypto/ghash-ce-glue.c | 144 +----------------- lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 1 + lib/crypto/arm/gf128hash.h | 43 ++++++ .../crypto/arm}/ghash-neon-core.S | 24 +-- 7 files changed, 66 insertions(+), 162 deletions(-) create mode 100644 lib/crypto/arm/gf128hash.h rename {arch/arm/crypto => lib/crypto/arm}/ghash-neon-core.S (92%) diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index b9c28c818b7c..f884b8b2fd93 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -3,26 +3,17 @@ menu "Accelerated Cryptographic Algorithms for CPU (arm)" config CRYPTO_GHASH_ARM_CE - tristate "Hash functions: GHASH (PMULL/NEON/ARMv8 Crypto Extensions)" + tristate "AEAD cipher: AES in GCM mode (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_AEAD - select CRYPTO_HASH - select CRYPTO_CRYPTD select CRYPTO_LIB_AES select CRYPTO_LIB_GF128MUL help - GCM GHASH function (NIST SP800-38D) + AEAD cipher: AES-GCM Architecture: arm using - - PMULL (Polynomial Multiply Long) instructions - - NEON (Advanced SIMD) extensions - ARMv8 Crypto Extensions - Use an implementation of GHASH (used by the GCM AEAD chaining mode) - that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64) - that is part of the ARMv8 Crypto Extensions, or a slower variant that - uses the vmull.p8 instruction that is part of the basic NEON ISA. - config CRYPTO_AES_ARM_BS tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (bit-sliced NEON)" depends on KERNEL_MODE_NEON diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index cedce94d5ee5..e73099e120b3 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -10,4 +10,4 @@ obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o -ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o ghash-neon-core.o +ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index d7d787de7dd3..9aa0ada5b627 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions. + * AES-GCM using ARMv8 Crypto Extensions * * Copyright (C) 2015 - 2018 Linaro Ltd. * Copyright (C) 2023 Google LLC. @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -25,20 +24,14 @@ #include #include -MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions"); +MODULE_DESCRIPTION("AES-GCM using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("ghash"); MODULE_ALIAS_CRYPTO("gcm(aes)"); MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); #define RFC4106_NONCE_SIZE 4 -struct ghash_key { - be128 k; - u64 h[1][2]; -}; - struct gcm_key { u64 h[4][2]; u32 rk[AES_MAX_KEYLENGTH_U32]; @@ -46,80 +39,9 @@ struct gcm_key { u8 nonce[]; // for RFC4106 nonce }; -struct arm_ghash_desc_ctx { - u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; -}; - asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, u64 const h[4][2], const char *head); -asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, - u64 const h[1][2], const char *head); - -static int ghash_init(struct shash_desc *desc) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - - *ctx = (struct arm_ghash_desc_ctx){}; - return 0; -} - -static void ghash_do_update(int blocks, u64 dg[], const char *src, - struct ghash_key *key, const char *head) -{ - kernel_neon_begin(); - pmull_ghash_update_p8(blocks, dg, src, key->h, head); - kernel_neon_end(); -} - -static int ghash_update(struct shash_desc *desc, const u8 *src, - unsigned int len) -{ - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - int blocks; - - blocks = len / GHASH_BLOCK_SIZE; - ghash_do_update(blocks, ctx->digest, src, key, NULL); - return len - blocks * GHASH_BLOCK_SIZE; -} - -static int ghash_export(struct shash_desc *desc, void *out) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - u8 *dst = out; - - put_unaligned_be64(ctx->digest[1], dst); - put_unaligned_be64(ctx->digest[0], dst + 8); - return 0; -} - -static int ghash_import(struct shash_desc *desc, const void *in) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - const u8 *src = in; - - ctx->digest[1] = get_unaligned_be64(src); - ctx->digest[0] = get_unaligned_be64(src + 8); - return 0; -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - ghash_do_update(1, ctx->digest, buf, key, NULL); - memzero_explicit(buf, sizeof(buf)); - } - return ghash_export(desc, dst); -} - static void ghash_reflect(u64 h[], const be128 *k) { u64 carry = be64_to_cpu(k->a) >> 63; @@ -131,40 +53,6 @@ static void ghash_reflect(u64 h[], const be128 *k) h[1] ^= 0xc200000000000000UL; } -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *inkey, unsigned int keylen) -{ - struct ghash_key *key = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - /* needed for the fallback */ - memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); - ghash_reflect(key->h[0], &key->k); - return 0; -} - -static struct shash_alg ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .export = ghash_export, - .import = ghash_import, - .descsize = sizeof(struct arm_ghash_desc_ctx), - .statesize = sizeof(struct ghash_desc_ctx), - - .base.cra_name = "ghash", - .base.cra_driver_name = "ghash-neon", - .base.cra_priority = 300, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .base.cra_blocksize = GHASH_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ghash_key), - .base.cra_module = THIS_MODULE, -}; - void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src, struct gcm_key const *k, char *dst, const char *iv, int rounds, u32 counter); @@ -543,37 +431,15 @@ static struct aead_alg gcm_aes_algs[] = {{ static int __init ghash_ce_mod_init(void) { - int err; - - if (!(elf_hwcap & HWCAP_NEON)) + if (!(elf_hwcap & HWCAP_NEON) || !(elf_hwcap2 & HWCAP2_PMULL)) return -ENODEV; - if (elf_hwcap2 & HWCAP2_PMULL) { - err = crypto_register_aeads(gcm_aes_algs, - ARRAY_SIZE(gcm_aes_algs)); - if (err) - return err; - } - - err = crypto_register_shash(&ghash_alg); - if (err) - goto err_aead; - - return 0; - -err_aead: - if (elf_hwcap2 & HWCAP2_PMULL) - crypto_unregister_aeads(gcm_aes_algs, - ARRAY_SIZE(gcm_aes_algs)); - return err; + return crypto_register_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); } static void __exit ghash_ce_mod_exit(void) { - crypto_unregister_shash(&ghash_alg); - if (elf_hwcap2 & HWCAP2_PMULL) - crypto_unregister_aeads(gcm_aes_algs, - ARRAY_SIZE(gcm_aes_algs)); + crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); } module_init(ghash_ce_mod_init); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 98cedd95c2a5..4f1a79883a56 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -119,6 +119,7 @@ config CRYPTO_LIB_GF128HASH config CRYPTO_LIB_GF128HASH_ARCH bool depends on CRYPTO_LIB_GF128HASH && !UML + default y if ARM && KERNEL_MODE_NEON default y if ARM64 default y if X86_64 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index fc30622123d2..8a06dd6a43ea 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -158,6 +158,7 @@ obj-$(CONFIG_CRYPTO_LIB_GF128HASH) += libgf128hash.o libgf128hash-y := gf128hash.o ifeq ($(CONFIG_CRYPTO_LIB_GF128HASH_ARCH),y) CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH) +libgf128hash-$(CONFIG_ARM) += arm/ghash-neon-core.o libgf128hash-$(CONFIG_ARM64) += arm64/polyval-ce-core.o libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o endif diff --git a/lib/crypto/arm/gf128hash.h b/lib/crypto/arm/gf128hash.h new file mode 100644 index 000000000000..c33c8cbe51fe --- /dev/null +++ b/lib/crypto/arm/gf128hash.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GHASH, arm optimized + * + * Copyright 2026 Google LLC + */ + +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); + +void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + const u8 *src, const struct polyval_elem *h); + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_neon) && may_use_simd()) { + do { + /* Allow rescheduling every 4 KiB. */ + size_t n = + min_t(size_t, nblocks, 4096 / GHASH_BLOCK_SIZE); + + scoped_ksimd() + pmull_ghash_update_p8(n, acc, data, &key->h); + data += n * GHASH_BLOCK_SIZE; + nblocks -= n; + } while (nblocks); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (elf_hwcap & HWCAP_NEON) + static_branch_enable(&have_neon); +} diff --git a/arch/arm/crypto/ghash-neon-core.S b/lib/crypto/arm/ghash-neon-core.S similarity index 92% rename from arch/arm/crypto/ghash-neon-core.S rename to lib/crypto/arm/ghash-neon-core.S index bdf6fb6d063c..eeffd12504a9 100644 --- a/arch/arm/crypto/ghash-neon-core.S +++ b/lib/crypto/arm/ghash-neon-core.S @@ -141,22 +141,21 @@ vshr.u64 XL, XL, #1 .endm + .macro vrev64_if_be a +#ifdef CONFIG_CPU_BIG_ENDIAN + vrev64.8 \a, \a +#endif + .endm + .macro ghash_update vld1.64 {XL}, [r1] - - /* do the head block first, if supplied */ - ldr ip, [sp] - teq ip, #0 - beq 0f - vld1.64 {T1}, [ip] - teq r0, #0 - b 3f + vrev64_if_be XL 0: vld1.8 {T1}, [r2]! subs r0, r0, #1 -3: /* multiply XL by SHASH in GF(2^128) */ + /* multiply XL by SHASH in GF(2^128) */ vrev64.8 T1, T1 vext.8 IN1, T1, T1, #8 @@ -180,11 +179,13 @@ .endm /* - * void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, - * u64 const h[1][2], const char *head) + * void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + * const u8 *src, + * const struct polyval_elem *h) */ ENTRY(pmull_ghash_update_p8) vld1.64 {SHASH}, [r3] + vrev64_if_be SHASH veor SHASH2_p8, SHASH_L, SHASH_H vext.8 s1l, SHASH_L, SHASH_L, #1 @@ -201,6 +202,7 @@ ENTRY(pmull_ghash_update_p8) vmov.i64 k48, #0xffffffffffff ghash_update + vrev64_if_be XL vst1.64 {XL}, [r1] bx lr From e3f473db02dae210f91e8eb8d0423232175639d6 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:09 -0700 Subject: [PATCH 26/66] crypto: arm64/ghash - Move NEON GHASH assembly into its own file arch/arm64/crypto/ghash-ce-core.S implements pmull_ghash_update_p8(), which is used only by a crypto_shash implementation of GHASH. It also implements other functions, including pmull_ghash_update_p64() and others, which are used only by a crypto_aead implementation of AES-GCM. While some code is shared between pmull_ghash_update_p8() and pmull_ghash_update_p64(), it's not very much. Since pmull_ghash_update_p8() will also need to be migrated into lib/crypto/ to achieve parity in the standalone GHASH support, let's move it into a separate file ghash-neon-core.S. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-9-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/crypto/Makefile | 2 +- arch/arm64/crypto/ghash-ce-core.S | 207 ++----------------------- arch/arm64/crypto/ghash-neon-core.S | 226 ++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+), 196 deletions(-) create mode 100644 arch/arm64/crypto/ghash-neon-core.S diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index 8a8e3e551ed3..b7ba43ce8584 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -27,7 +27,7 @@ obj-$(CONFIG_CRYPTO_SM4_ARM64_NEON_BLK) += sm4-neon.o sm4-neon-y := sm4-neon-glue.o sm4-neon-core.o obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o -ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o +ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o ghash-neon-core.o obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index 23ee9a5eaf27..4344fe213d14 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Accelerated GHASH implementation with ARMv8 PMULL instructions. + * Accelerated AES-GCM implementation with ARMv8 Crypto Extensions. * * Copyright (C) 2014 - 2018 Linaro Ltd. */ @@ -19,31 +19,6 @@ XH .req v7 IN1 .req v7 - k00_16 .req v8 - k32_48 .req v9 - - t3 .req v10 - t4 .req v11 - t5 .req v12 - t6 .req v13 - t7 .req v14 - t8 .req v15 - t9 .req v16 - - perm1 .req v17 - perm2 .req v18 - perm3 .req v19 - - sh1 .req v20 - sh2 .req v21 - sh3 .req v22 - sh4 .req v23 - - ss1 .req v24 - ss2 .req v25 - ss3 .req v26 - ss4 .req v27 - XL2 .req v8 XM2 .req v9 XH2 .req v10 @@ -60,90 +35,6 @@ .text .arch armv8-a+crypto - .macro __pmull_p64, rd, rn, rm - pmull \rd\().1q, \rn\().1d, \rm\().1d - .endm - - .macro __pmull2_p64, rd, rn, rm - pmull2 \rd\().1q, \rn\().2d, \rm\().2d - .endm - - .macro __pmull_p8, rq, ad, bd - ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1 - ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2 - ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3 - - __pmull_p8_\bd \rq, \ad - .endm - - .macro __pmull2_p8, rq, ad, bd - tbl t3.16b, {\ad\().16b}, perm1.16b // A1 - tbl t5.16b, {\ad\().16b}, perm2.16b // A2 - tbl t7.16b, {\ad\().16b}, perm3.16b // A3 - - __pmull2_p8_\bd \rq, \ad - .endm - - .macro __pmull_p8_SHASH, rq, ad - __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4 - .endm - - .macro __pmull_p8_SHASH2, rq, ad - __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4 - .endm - - .macro __pmull2_p8_SHASH, rq, ad - __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4 - .endm - - .macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4 - pmull\t t3.8h, t3.\nb, \bd // F = A1*B - pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1 - pmull\t t5.8h, t5.\nb, \bd // H = A2*B - pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2 - pmull\t t7.8h, t7.\nb, \bd // J = A3*B - pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3 - pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4 - pmull\t \rq\().8h, \ad, \bd // D = A*B - - eor t3.16b, t3.16b, t4.16b // L = E + F - eor t5.16b, t5.16b, t6.16b // M = G + H - eor t7.16b, t7.16b, t8.16b // N = I + J - - uzp1 t4.2d, t3.2d, t5.2d - uzp2 t3.2d, t3.2d, t5.2d - uzp1 t6.2d, t7.2d, t9.2d - uzp2 t7.2d, t7.2d, t9.2d - - // t3 = (L) (P0 + P1) << 8 - // t5 = (M) (P2 + P3) << 16 - eor t4.16b, t4.16b, t3.16b - and t3.16b, t3.16b, k32_48.16b - - // t7 = (N) (P4 + P5) << 24 - // t9 = (K) (P6 + P7) << 32 - eor t6.16b, t6.16b, t7.16b - and t7.16b, t7.16b, k00_16.16b - - eor t4.16b, t4.16b, t3.16b - eor t6.16b, t6.16b, t7.16b - - zip2 t5.2d, t4.2d, t3.2d - zip1 t3.2d, t4.2d, t3.2d - zip2 t9.2d, t6.2d, t7.2d - zip1 t7.2d, t6.2d, t7.2d - - ext t3.16b, t3.16b, t3.16b, #15 - ext t5.16b, t5.16b, t5.16b, #14 - ext t7.16b, t7.16b, t7.16b, #13 - ext t9.16b, t9.16b, t9.16b, #12 - - eor t3.16b, t3.16b, t5.16b - eor t7.16b, t7.16b, t9.16b - eor \rq\().16b, \rq\().16b, t3.16b - eor \rq\().16b, \rq\().16b, t7.16b - .endm - .macro __pmull_pre_p64 add x8, x3, #16 ld1 {HH.2d-HH4.2d}, [x8] @@ -160,43 +51,6 @@ shl MASK.2d, MASK.2d, #57 .endm - .macro __pmull_pre_p8 - ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 - eor SHASH2.16b, SHASH2.16b, SHASH.16b - - // k00_16 := 0x0000000000000000_000000000000ffff - // k32_48 := 0x00000000ffffffff_0000ffffffffffff - movi k32_48.2d, #0xffffffff - mov k32_48.h[2], k32_48.h[0] - ushr k00_16.2d, k32_48.2d, #32 - - // prepare the permutation vectors - mov_q x5, 0x080f0e0d0c0b0a09 - movi T1.8b, #8 - dup perm1.2d, x5 - eor perm1.16b, perm1.16b, T1.16b - ushr perm2.2d, perm1.2d, #8 - ushr perm3.2d, perm1.2d, #16 - ushr T1.2d, perm1.2d, #24 - sli perm2.2d, perm1.2d, #56 - sli perm3.2d, perm1.2d, #48 - sli T1.2d, perm1.2d, #40 - - // precompute loop invariants - tbl sh1.16b, {SHASH.16b}, perm1.16b - tbl sh2.16b, {SHASH.16b}, perm2.16b - tbl sh3.16b, {SHASH.16b}, perm3.16b - tbl sh4.16b, {SHASH.16b}, T1.16b - ext ss1.8b, SHASH2.8b, SHASH2.8b, #1 - ext ss2.8b, SHASH2.8b, SHASH2.8b, #2 - ext ss3.8b, SHASH2.8b, SHASH2.8b, #3 - ext ss4.8b, SHASH2.8b, SHASH2.8b, #4 - .endm - - // - // PMULL (64x64->128) based reduction for CPUs that can do - // it in a single instruction. - // .macro __pmull_reduce_p64 pmull T2.1q, XL.1d, MASK.1d eor XM.16b, XM.16b, T1.16b @@ -209,39 +63,15 @@ pmull XL.1q, XL.1d, MASK.1d .endm - // - // Alternative reduction for CPUs that lack support for the - // 64x64->128 PMULL instruction - // - .macro __pmull_reduce_p8 - eor XM.16b, XM.16b, T1.16b - - mov XL.d[1], XM.d[0] - mov XH.d[0], XM.d[1] - - shl T1.2d, XL.2d, #57 - shl T2.2d, XL.2d, #62 - eor T2.16b, T2.16b, T1.16b - shl T1.2d, XL.2d, #63 - eor T2.16b, T2.16b, T1.16b - ext T1.16b, XL.16b, XH.16b, #8 - eor T2.16b, T2.16b, T1.16b - - mov XL.d[1], T2.d[0] - mov XH.d[0], T2.d[1] - - ushr T2.2d, XL.2d, #1 - eor XH.16b, XH.16b, XL.16b - eor XL.16b, XL.16b, T2.16b - ushr T2.2d, T2.2d, #6 - ushr XL.2d, XL.2d, #1 - .endm - - .macro __pmull_ghash, pn + /* + * void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, + * u64 const h[][2], const char *head) + */ +SYM_TYPED_FUNC_START(pmull_ghash_update_p64) ld1 {SHASH.2d}, [x3] ld1 {XL.2d}, [x1] - __pmull_pre_\pn + __pmull_pre_p64 /* do the head block first, if supplied */ cbz x4, 0f @@ -249,7 +79,7 @@ mov x4, xzr b 3f -0: .ifc \pn, p64 +0: tbnz w0, #0, 2f // skip until #blocks is a tbnz w0, #1, 2f // round multiple of 4 @@ -314,7 +144,6 @@ cbz w0, 5f b 1b - .endif 2: ld1 {T1.2d}, [x2], #16 sub w0, w0, #1 @@ -327,16 +156,16 @@ CPU_LE( rev64 T1.16b, T1.16b ) eor T1.16b, T1.16b, T2.16b eor XL.16b, XL.16b, IN1.16b - __pmull2_\pn XH, XL, SHASH // a1 * b1 + pmull2 XH.1q, XL.2d, SHASH.2d // a1 * b1 eor T1.16b, T1.16b, XL.16b - __pmull_\pn XL, XL, SHASH // a0 * b0 - __pmull_\pn XM, T1, SHASH2 // (a1 + a0)(b1 + b0) + pmull XL.1q, XL.1d, SHASH.1d // a0 * b0 + pmull XM.1q, T1.1d, SHASH2.1d // (a1 + a0)(b1 + b0) 4: eor T2.16b, XL.16b, XH.16b ext T1.16b, XL.16b, XH.16b, #8 eor XM.16b, XM.16b, T2.16b - __pmull_reduce_\pn + __pmull_reduce_p64 eor T2.16b, T2.16b, XH.16b eor XL.16b, XL.16b, T2.16b @@ -345,20 +174,8 @@ CPU_LE( rev64 T1.16b, T1.16b ) 5: st1 {XL.2d}, [x1] ret - .endm - - /* - * void pmull_ghash_update(int blocks, u64 dg[], const char *src, - * struct ghash_key const *k, const char *head) - */ -SYM_TYPED_FUNC_START(pmull_ghash_update_p64) - __pmull_ghash p64 SYM_FUNC_END(pmull_ghash_update_p64) -SYM_TYPED_FUNC_START(pmull_ghash_update_p8) - __pmull_ghash p8 -SYM_FUNC_END(pmull_ghash_update_p8) - KS0 .req v8 KS1 .req v9 KS2 .req v10 diff --git a/arch/arm64/crypto/ghash-neon-core.S b/arch/arm64/crypto/ghash-neon-core.S new file mode 100644 index 000000000000..6157135ad566 --- /dev/null +++ b/arch/arm64/crypto/ghash-neon-core.S @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Accelerated GHASH implementation with ARMv8 ASIMD instructions. + * + * Copyright (C) 2014 - 2018 Linaro Ltd. + */ + +#include +#include +#include + + SHASH .req v0 + SHASH2 .req v1 + T1 .req v2 + T2 .req v3 + XM .req v5 + XL .req v6 + XH .req v7 + IN1 .req v7 + + k00_16 .req v8 + k32_48 .req v9 + + t3 .req v10 + t4 .req v11 + t5 .req v12 + t6 .req v13 + t7 .req v14 + t8 .req v15 + t9 .req v16 + + perm1 .req v17 + perm2 .req v18 + perm3 .req v19 + + sh1 .req v20 + sh2 .req v21 + sh3 .req v22 + sh4 .req v23 + + ss1 .req v24 + ss2 .req v25 + ss3 .req v26 + ss4 .req v27 + + .text + + .macro __pmull_p8, rq, ad, bd + ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1 + ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2 + ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3 + + __pmull_p8_\bd \rq, \ad + .endm + + .macro __pmull2_p8, rq, ad, bd + tbl t3.16b, {\ad\().16b}, perm1.16b // A1 + tbl t5.16b, {\ad\().16b}, perm2.16b // A2 + tbl t7.16b, {\ad\().16b}, perm3.16b // A3 + + __pmull2_p8_\bd \rq, \ad + .endm + + .macro __pmull_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_SHASH2, rq, ad + __pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4 + .endm + + .macro __pmull2_p8_SHASH, rq, ad + __pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4 + .endm + + .macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4 + pmull\t t3.8h, t3.\nb, \bd // F = A1*B + pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1 + pmull\t t5.8h, t5.\nb, \bd // H = A2*B + pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2 + pmull\t t7.8h, t7.\nb, \bd // J = A3*B + pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3 + pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4 + pmull\t \rq\().8h, \ad, \bd // D = A*B + + eor t3.16b, t3.16b, t4.16b // L = E + F + eor t5.16b, t5.16b, t6.16b // M = G + H + eor t7.16b, t7.16b, t8.16b // N = I + J + + uzp1 t4.2d, t3.2d, t5.2d + uzp2 t3.2d, t3.2d, t5.2d + uzp1 t6.2d, t7.2d, t9.2d + uzp2 t7.2d, t7.2d, t9.2d + + // t3 = (L) (P0 + P1) << 8 + // t5 = (M) (P2 + P3) << 16 + eor t4.16b, t4.16b, t3.16b + and t3.16b, t3.16b, k32_48.16b + + // t7 = (N) (P4 + P5) << 24 + // t9 = (K) (P6 + P7) << 32 + eor t6.16b, t6.16b, t7.16b + and t7.16b, t7.16b, k00_16.16b + + eor t4.16b, t4.16b, t3.16b + eor t6.16b, t6.16b, t7.16b + + zip2 t5.2d, t4.2d, t3.2d + zip1 t3.2d, t4.2d, t3.2d + zip2 t9.2d, t6.2d, t7.2d + zip1 t7.2d, t6.2d, t7.2d + + ext t3.16b, t3.16b, t3.16b, #15 + ext t5.16b, t5.16b, t5.16b, #14 + ext t7.16b, t7.16b, t7.16b, #13 + ext t9.16b, t9.16b, t9.16b, #12 + + eor t3.16b, t3.16b, t5.16b + eor t7.16b, t7.16b, t9.16b + eor \rq\().16b, \rq\().16b, t3.16b + eor \rq\().16b, \rq\().16b, t7.16b + .endm + + .macro __pmull_pre_p8 + ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 + eor SHASH2.16b, SHASH2.16b, SHASH.16b + + // k00_16 := 0x0000000000000000_000000000000ffff + // k32_48 := 0x00000000ffffffff_0000ffffffffffff + movi k32_48.2d, #0xffffffff + mov k32_48.h[2], k32_48.h[0] + ushr k00_16.2d, k32_48.2d, #32 + + // prepare the permutation vectors + mov_q x5, 0x080f0e0d0c0b0a09 + movi T1.8b, #8 + dup perm1.2d, x5 + eor perm1.16b, perm1.16b, T1.16b + ushr perm2.2d, perm1.2d, #8 + ushr perm3.2d, perm1.2d, #16 + ushr T1.2d, perm1.2d, #24 + sli perm2.2d, perm1.2d, #56 + sli perm3.2d, perm1.2d, #48 + sli T1.2d, perm1.2d, #40 + + // precompute loop invariants + tbl sh1.16b, {SHASH.16b}, perm1.16b + tbl sh2.16b, {SHASH.16b}, perm2.16b + tbl sh3.16b, {SHASH.16b}, perm3.16b + tbl sh4.16b, {SHASH.16b}, T1.16b + ext ss1.8b, SHASH2.8b, SHASH2.8b, #1 + ext ss2.8b, SHASH2.8b, SHASH2.8b, #2 + ext ss3.8b, SHASH2.8b, SHASH2.8b, #3 + ext ss4.8b, SHASH2.8b, SHASH2.8b, #4 + .endm + + .macro __pmull_reduce_p8 + eor XM.16b, XM.16b, T1.16b + + mov XL.d[1], XM.d[0] + mov XH.d[0], XM.d[1] + + shl T1.2d, XL.2d, #57 + shl T2.2d, XL.2d, #62 + eor T2.16b, T2.16b, T1.16b + shl T1.2d, XL.2d, #63 + eor T2.16b, T2.16b, T1.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor T2.16b, T2.16b, T1.16b + + mov XL.d[1], T2.d[0] + mov XH.d[0], T2.d[1] + + ushr T2.2d, XL.2d, #1 + eor XH.16b, XH.16b, XL.16b + eor XL.16b, XL.16b, T2.16b + ushr T2.2d, T2.2d, #6 + ushr XL.2d, XL.2d, #1 + .endm + + /* + * void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, + * u64 const h[][2], const char *head) + */ +SYM_TYPED_FUNC_START(pmull_ghash_update_p8) + ld1 {SHASH.2d}, [x3] + ld1 {XL.2d}, [x1] + + __pmull_pre_p8 + + /* do the head block first, if supplied */ + cbz x4, 0f + ld1 {T1.2d}, [x4] + mov x4, xzr + b 3f + +0: ld1 {T1.2d}, [x2], #16 + sub w0, w0, #1 + +3: /* multiply XL by SHASH in GF(2^128) */ +CPU_LE( rev64 T1.16b, T1.16b ) + + ext T2.16b, XL.16b, XL.16b, #8 + ext IN1.16b, T1.16b, T1.16b, #8 + eor T1.16b, T1.16b, T2.16b + eor XL.16b, XL.16b, IN1.16b + + __pmull2_p8 XH, XL, SHASH // a1 * b1 + eor T1.16b, T1.16b, XL.16b + __pmull_p8 XL, XL, SHASH // a0 * b0 + __pmull_p8 XM, T1, SHASH2 // (a1 + a0)(b1 + b0) + + eor T2.16b, XL.16b, XH.16b + ext T1.16b, XL.16b, XH.16b, #8 + eor XM.16b, XM.16b, T2.16b + + __pmull_reduce_p8 + + eor T2.16b, T2.16b, XH.16b + eor XL.16b, XL.16b, T2.16b + + cbnz w0, 0b + + st1 {XL.2d}, [x1] + ret +SYM_FUNC_END(pmull_ghash_update_p8) From a336c01f5b11cc158150d051d612a2f7ad9fd6a8 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:10 -0700 Subject: [PATCH 27/66] lib/crypto: arm64/ghash: Migrate optimized code into library Remove the "ghash-neon" crypto_shash algorithm. Move the corresponding assembly code into lib/crypto/, and wire it up to the GHASH library. This makes the GHASH library be optimized on arm64 (though only with NEON, not PMULL; for now the goal is just parity with crypto_shash). It greatly reduces the amount of arm64-specific glue code that is needed, and it fixes the issue where this optimization was disabled by default. To integrate the assembly code correctly with the library, make the following tweaks: - Change the type of 'blocks' from int to size_t - Change the types of 'dg' and 'h' to polyval_elem. Note that this simply reflects the format that the code was already using. - Remove the 'head' argument, which is no longer needed. - Remove the CFI stubs, as indirect calls are no longer used. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-10-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/crypto/Kconfig | 5 +- arch/arm64/crypto/Makefile | 2 +- arch/arm64/crypto/ghash-ce-core.S | 3 +- arch/arm64/crypto/ghash-ce-glue.c | 150 ++---------------- lib/crypto/Makefile | 3 +- lib/crypto/arm64/gf128hash.h | 68 +++++++- .../crypto/arm64}/ghash-neon-core.S | 20 +-- 7 files changed, 86 insertions(+), 165 deletions(-) rename {arch/arm64/crypto => lib/crypto/arm64}/ghash-neon-core.S (93%) diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 82794afaffc9..1a0c553fbfd7 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -3,14 +3,13 @@ menu "Accelerated Cryptographic Algorithms for CPU (arm64)" config CRYPTO_GHASH_ARM64_CE - tristate "Hash functions: GHASH (ARMv8 Crypto Extensions)" + tristate "AEAD cipher: AES in GCM mode (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON - select CRYPTO_HASH select CRYPTO_LIB_AES select CRYPTO_LIB_GF128MUL select CRYPTO_AEAD help - GCM GHASH function (NIST SP800-38D) + AEAD cipher: AES-GCM Architecture: arm64 using: - ARMv8 Crypto Extensions diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index b7ba43ce8584..8a8e3e551ed3 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -27,7 +27,7 @@ obj-$(CONFIG_CRYPTO_SM4_ARM64_NEON_BLK) += sm4-neon.o sm4-neon-y := sm4-neon-glue.o sm4-neon-core.o obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o -ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o ghash-neon-core.o +ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index 4344fe213d14..a01f136f4fb2 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -6,7 +6,6 @@ */ #include -#include #include SHASH .req v0 @@ -67,7 +66,7 @@ * void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, * u64 const h[][2], const char *head) */ -SYM_TYPED_FUNC_START(pmull_ghash_update_p64) +SYM_FUNC_START(pmull_ghash_update_p64) ld1 {SHASH.2d}, [x3] ld1 {XL.2d}, [x1] diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 63bb9e062251..590054c3260d 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Accelerated GHASH implementation with ARMv8 PMULL instructions. + * AES-GCM using ARMv8 Crypto Extensions * * Copyright (C) 2014 - 2018 Linaro Ltd. */ @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -23,10 +22,11 @@ #include -MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions"); +MODULE_DESCRIPTION("AES-GCM using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("ghash"); +MODULE_ALIAS_CRYPTO("gcm(aes)"); +MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); #define RFC4106_NONCE_SIZE 4 @@ -35,10 +35,6 @@ struct ghash_key { u64 h[][2]; }; -struct arm_ghash_desc_ctx { - u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; -}; - struct gcm_aes_ctx { struct aes_enckey aes_key; u8 nonce[RFC4106_NONCE_SIZE]; @@ -48,9 +44,6 @@ struct gcm_aes_ctx { asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, u64 const h[][2], const char *head); -asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, - u64 const h[][2], const char *head); - asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[], u64 const h[][2], u64 dg[], u8 ctr[], u32 const rk[], int rounds, u8 tag[]); @@ -59,85 +52,11 @@ asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], u32 const rk[], int rounds, const u8 l[], const u8 tag[], u64 authsize); -static int ghash_init(struct shash_desc *desc) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - - *ctx = (struct arm_ghash_desc_ctx){}; - return 0; -} - -static __always_inline -void ghash_do_simd_update(int blocks, u64 dg[], const char *src, - struct ghash_key *key, const char *head, - void (*simd_update)(int blocks, u64 dg[], - const char *src, - u64 const h[][2], - const char *head)) +static void ghash_do_simd_update(int blocks, u64 dg[], const char *src, + struct ghash_key *key, const char *head) { scoped_ksimd() - simd_update(blocks, dg, src, key->h, head); -} - -/* avoid hogging the CPU for too long */ -#define MAX_BLOCKS (SZ_64K / GHASH_BLOCK_SIZE) - -static int ghash_update(struct shash_desc *desc, const u8 *src, - unsigned int len) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - int blocks; - - blocks = len / GHASH_BLOCK_SIZE; - len -= blocks * GHASH_BLOCK_SIZE; - - do { - int chunk = min(blocks, MAX_BLOCKS); - - ghash_do_simd_update(chunk, ctx->digest, src, key, NULL, - pmull_ghash_update_p8); - blocks -= chunk; - src += chunk * GHASH_BLOCK_SIZE; - } while (unlikely(blocks > 0)); - return len; -} - -static int ghash_export(struct shash_desc *desc, void *out) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - u8 *dst = out; - - put_unaligned_be64(ctx->digest[1], dst); - put_unaligned_be64(ctx->digest[0], dst + 8); - return 0; -} - -static int ghash_import(struct shash_desc *desc, const void *in) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - const u8 *src = in; - - ctx->digest[1] = get_unaligned_be64(src); - ctx->digest[0] = get_unaligned_be64(src + 8); - return 0; -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct arm_ghash_desc_ctx *ctx = shash_desc_ctx(desc); - struct ghash_key *key = crypto_shash_ctx(desc->tfm); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - ghash_do_simd_update(1, ctx->digest, buf, key, NULL, - pmull_ghash_update_p8); - memzero_explicit(buf, sizeof(buf)); - } - return ghash_export(desc, dst); + pmull_ghash_update_p64(blocks, dg, src, key->h, head); } static void ghash_reflect(u64 h[], const be128 *k) @@ -151,41 +70,6 @@ static void ghash_reflect(u64 h[], const be128 *k) h[1] ^= 0xc200000000000000UL; } -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *inkey, unsigned int keylen) -{ - struct ghash_key *key = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - /* needed for the fallback */ - memcpy(&key->k, inkey, GHASH_BLOCK_SIZE); - - ghash_reflect(key->h[0], &key->k); - return 0; -} - -static struct shash_alg ghash_alg = { - .base.cra_name = "ghash", - .base.cra_driver_name = "ghash-neon", - .base.cra_priority = 150, - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .base.cra_blocksize = GHASH_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]), - .base.cra_module = THIS_MODULE, - - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .export = ghash_export, - .import = ghash_import, - .descsize = sizeof(struct arm_ghash_desc_ctx), - .statesize = sizeof(struct ghash_desc_ctx), -}; - static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey, unsigned int keylen) { @@ -240,9 +124,7 @@ static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[], int blocks = count / GHASH_BLOCK_SIZE; ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key, - *buf_count ? buf : NULL, - pmull_ghash_update_p64); - + *buf_count ? buf : NULL); src += blocks * GHASH_BLOCK_SIZE; count %= GHASH_BLOCK_SIZE; *buf_count = 0; @@ -275,8 +157,7 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) if (buf_count) { memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count); - ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL, - pmull_ghash_update_p64); + ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL); } } @@ -505,22 +386,15 @@ static struct aead_alg gcm_aes_algs[] = {{ static int __init ghash_ce_mod_init(void) { - if (!cpu_have_named_feature(ASIMD)) + if (!cpu_have_named_feature(ASIMD) || !cpu_have_named_feature(PMULL)) return -ENODEV; - if (cpu_have_named_feature(PMULL)) - return crypto_register_aeads(gcm_aes_algs, - ARRAY_SIZE(gcm_aes_algs)); - - return crypto_register_shash(&ghash_alg); + return crypto_register_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); } static void __exit ghash_ce_mod_exit(void) { - if (cpu_have_named_feature(PMULL)) - crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); - else - crypto_unregister_shash(&ghash_alg); + crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs)); } static const struct cpu_feature __maybe_unused ghash_cpu_feature[] = { diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 8a06dd6a43ea..4ce0bac8fd93 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -159,7 +159,8 @@ libgf128hash-y := gf128hash.o ifeq ($(CONFIG_CRYPTO_LIB_GF128HASH_ARCH),y) CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH) libgf128hash-$(CONFIG_ARM) += arm/ghash-neon-core.o -libgf128hash-$(CONFIG_ARM64) += arm64/polyval-ce-core.o +libgf128hash-$(CONFIG_ARM64) += arm64/ghash-neon-core.o \ + arm64/polyval-ce-core.o libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o endif diff --git a/lib/crypto/arm64/gf128hash.h b/lib/crypto/arm64/gf128hash.h index 796c36804dda..b2c85585b758 100644 --- a/lib/crypto/arm64/gf128hash.h +++ b/lib/crypto/arm64/gf128hash.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * POLYVAL library functions, arm64 optimized + * GHASH and POLYVAL, arm64 optimized * * Copyright 2025 Google LLC */ @@ -9,8 +9,12 @@ #define NUM_H_POWERS 8 +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); +asmlinkage void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + const u8 *src, + const struct polyval_elem *h); asmlinkage void polyval_mul_pmull(struct polyval_elem *a, const struct polyval_elem *b); asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, @@ -41,15 +45,62 @@ static void polyval_preparekey_arch(struct polyval_key *key, } } +static void polyval_mul_arm64(struct polyval_elem *a, + const struct polyval_elem *b) +{ + if (static_branch_likely(&have_asimd) && may_use_simd()) { + static const u8 zeroes[GHASH_BLOCK_SIZE]; + + scoped_ksimd() { + if (static_branch_likely(&have_pmull)) { + polyval_mul_pmull(a, b); + } else { + /* + * Note that this is indeed equivalent to a + * POLYVAL multiplication, since it takes the + * accumulator and key in POLYVAL format, and + * byte-swapping a block of zeroes is a no-op. + */ + pmull_ghash_update_p8(1, a, zeroes, b); + } + } + } else { + polyval_mul_generic(a, b); + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + polyval_mul_arm64(acc, &key->h); +} + #define polyval_mul_arch polyval_mul_arch static void polyval_mul_arch(struct polyval_elem *acc, const struct polyval_key *key) { - if (static_branch_likely(&have_pmull) && may_use_simd()) { - scoped_ksimd() - polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]); + polyval_mul_arm64(acc, &key->h_powers[NUM_H_POWERS - 1]); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_asimd) && may_use_simd()) { + do { + /* Allow rescheduling every 4 KiB. */ + size_t n = min_t(size_t, nblocks, + 4096 / GHASH_BLOCK_SIZE); + + scoped_ksimd() + pmull_ghash_update_p8(n, acc, data, &key->h); + data += n * GHASH_BLOCK_SIZE; + nblocks -= n; + } while (nblocks); } else { - polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); + ghash_blocks_generic(acc, &key->h, data, nblocks); } } @@ -78,6 +129,9 @@ static void polyval_blocks_arch(struct polyval_elem *acc, #define gf128hash_mod_init_arch gf128hash_mod_init_arch static void gf128hash_mod_init_arch(void) { - if (cpu_have_named_feature(PMULL)) - static_branch_enable(&have_pmull); + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_asimd); + if (cpu_have_named_feature(PMULL)) + static_branch_enable(&have_pmull); + } } diff --git a/arch/arm64/crypto/ghash-neon-core.S b/lib/crypto/arm64/ghash-neon-core.S similarity index 93% rename from arch/arm64/crypto/ghash-neon-core.S rename to lib/crypto/arm64/ghash-neon-core.S index 6157135ad566..85b20fcd98fe 100644 --- a/arch/arm64/crypto/ghash-neon-core.S +++ b/lib/crypto/arm64/ghash-neon-core.S @@ -6,7 +6,6 @@ */ #include -#include #include SHASH .req v0 @@ -179,25 +178,20 @@ .endm /* - * void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, - * u64 const h[][2], const char *head) + * void pmull_ghash_update_p8(size_t blocks, struct polyval_elem *dg, + * const u8 *src, + * const struct polyval_elem *h) */ -SYM_TYPED_FUNC_START(pmull_ghash_update_p8) +SYM_FUNC_START(pmull_ghash_update_p8) ld1 {SHASH.2d}, [x3] ld1 {XL.2d}, [x1] __pmull_pre_p8 - /* do the head block first, if supplied */ - cbz x4, 0f - ld1 {T1.2d}, [x4] - mov x4, xzr - b 3f - 0: ld1 {T1.2d}, [x2], #16 - sub w0, w0, #1 + sub x0, x0, #1 -3: /* multiply XL by SHASH in GF(2^128) */ + /* multiply XL by SHASH in GF(2^128) */ CPU_LE( rev64 T1.16b, T1.16b ) ext T2.16b, XL.16b, XL.16b, #8 @@ -219,7 +213,7 @@ CPU_LE( rev64 T1.16b, T1.16b ) eor T2.16b, T2.16b, XH.16b eor XL.16b, XL.16b, T2.16b - cbnz w0, 0b + cbnz x0, 0b st1 {XL.2d}, [x1] ret From 631a84e49e5b16ab83098350c2695d806ad54d23 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:11 -0700 Subject: [PATCH 28/66] crypto: arm64/aes-gcm - Rename struct ghash_key and make fixed-sized Rename the 'struct ghash_key' in arch/arm64/crypto/ghash-ce-glue.c to prevent a naming conflict with the library 'struct ghash_key'. In addition, declare the 'h' field with an explicit size, now that there's no longer any reason for it to be a flexible array. Update the comments in the assembly file to match the C code. Note that some of these were out-of-date. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-11-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/crypto/ghash-ce-core.S | 15 ++++++++------- arch/arm64/crypto/ghash-ce-glue.c | 20 +++++++++----------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index a01f136f4fb2..33772d8fe6b5 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -64,7 +64,7 @@ /* * void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, - * u64 const h[][2], const char *head) + * u64 const h[4][2], const char *head) */ SYM_FUNC_START(pmull_ghash_update_p64) ld1 {SHASH.2d}, [x3] @@ -413,18 +413,19 @@ CPU_LE( rev w8, w8 ) .endm /* - * void pmull_gcm_encrypt(int blocks, u8 dst[], const u8 src[], - * struct ghash_key const *k, u64 dg[], u8 ctr[], - * int rounds, u8 tag) + * void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[], + * u64 const h[4][2], u64 dg[], u8 ctr[], + * u32 const rk[], int rounds, u8 tag[]) */ SYM_FUNC_START(pmull_gcm_encrypt) pmull_gcm_do_crypt 1 SYM_FUNC_END(pmull_gcm_encrypt) /* - * void pmull_gcm_decrypt(int blocks, u8 dst[], const u8 src[], - * struct ghash_key const *k, u64 dg[], u8 ctr[], - * int rounds, u8 tag) + * int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], + * u64 const h[4][2], u64 dg[], u8 ctr[], + * u32 const rk[], int rounds, const u8 l[], + * const u8 tag[], u64 authsize) */ SYM_FUNC_START(pmull_gcm_decrypt) pmull_gcm_do_crypt 0 diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 590054c3260d..eaf2932ceaf5 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -30,30 +30,30 @@ MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))"); #define RFC4106_NONCE_SIZE 4 -struct ghash_key { +struct arm_ghash_key { be128 k; - u64 h[][2]; + u64 h[4][2]; }; struct gcm_aes_ctx { struct aes_enckey aes_key; u8 nonce[RFC4106_NONCE_SIZE]; - struct ghash_key ghash_key; + struct arm_ghash_key ghash_key; }; asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, - u64 const h[][2], const char *head); + u64 const h[4][2], const char *head); asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[], - u64 const h[][2], u64 dg[], u8 ctr[], + u64 const h[4][2], u64 dg[], u8 ctr[], u32 const rk[], int rounds, u8 tag[]); asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], - u64 const h[][2], u64 dg[], u8 ctr[], + u64 const h[4][2], u64 dg[], u8 ctr[], u32 const rk[], int rounds, const u8 l[], const u8 tag[], u64 authsize); static void ghash_do_simd_update(int blocks, u64 dg[], const char *src, - struct ghash_key *key, const char *head) + struct arm_ghash_key *key, const char *head) { scoped_ksimd() pmull_ghash_update_p64(blocks, dg, src, key->h, head); @@ -363,8 +363,7 @@ static struct aead_alg gcm_aes_algs[] = {{ .base.cra_driver_name = "gcm-aes-ce", .base.cra_priority = 300, .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct gcm_aes_ctx) + - 4 * sizeof(u64[2]), + .base.cra_ctxsize = sizeof(struct gcm_aes_ctx), .base.cra_module = THIS_MODULE, }, { .ivsize = GCM_RFC4106_IV_SIZE, @@ -379,8 +378,7 @@ static struct aead_alg gcm_aes_algs[] = {{ .base.cra_driver_name = "rfc4106-gcm-aes-ce", .base.cra_priority = 300, .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct gcm_aes_ctx) + - 4 * sizeof(u64[2]), + .base.cra_ctxsize = sizeof(struct gcm_aes_ctx), .base.cra_module = THIS_MODULE, }}; From 73f315c15d6ec1ef33202e7253af90dd44ff4a3b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:12 -0700 Subject: [PATCH 29/66] lib/crypto: powerpc/ghash: Migrate optimized code into library Remove the "p8_ghash" crypto_shash algorithm. Move the corresponding assembly code into lib/crypto/, and wire it up to the GHASH library. This makes the GHASH library be optimized for POWER8. It also greatly reduces the amount of powerpc-specific glue code that is needed, and it fixes the issue where this optimized GHASH code was disabled by default. Note that previously the C code defined the POWER8 GHASH key format as "u128 htable[16]", despite the assembly code only using four entries. Fix the C code to use the correct key format. To fulfill the library API contract, also make the key preparation work in all contexts. Note that the POWER8 assembly code takes the accumulator in GHASH format, but it actually byte-reflects it to get it into POLYVAL format. The library already works with POLYVAL natively. For now, just wire up this existing code by converting it to/from GHASH format in C code. This should be cleaned up to eliminate the unnecessary conversion later. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-12-ebiggers@kernel.org Signed-off-by: Eric Biggers --- MAINTAINERS | 4 +- arch/powerpc/crypto/Kconfig | 5 +- arch/powerpc/crypto/Makefile | 8 +- arch/powerpc/crypto/aesp8-ppc.h | 1 - arch/powerpc/crypto/ghash.c | 160 ------------------ arch/powerpc/crypto/vmx.c | 10 +- include/crypto/gf128hash.h | 4 + lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 25 ++- lib/crypto/powerpc/.gitignore | 1 + lib/crypto/powerpc/gf128hash.h | 109 ++++++++++++ .../crypto/powerpc}/ghashp8-ppc.pl | 1 + 12 files changed, 143 insertions(+), 186 deletions(-) delete mode 100644 arch/powerpc/crypto/ghash.c create mode 100644 lib/crypto/powerpc/gf128hash.h rename {arch/powerpc/crypto => lib/crypto/powerpc}/ghashp8-ppc.pl (98%) diff --git a/MAINTAINERS b/MAINTAINERS index 77fdfcb55f06..f088f4085653 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12267,10 +12267,10 @@ F: arch/powerpc/crypto/aes_cbc.c F: arch/powerpc/crypto/aes_ctr.c F: arch/powerpc/crypto/aes_xts.c F: arch/powerpc/crypto/aesp8-ppc.* -F: arch/powerpc/crypto/ghash.c -F: arch/powerpc/crypto/ghashp8-ppc.pl F: arch/powerpc/crypto/ppc-xlate.pl F: arch/powerpc/crypto/vmx.c +F: lib/crypto/powerpc/gf128hash.h +F: lib/crypto/powerpc/ghashp8-ppc.pl IBM ServeRAID RAID DRIVER S: Orphan diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 2d056f1fc90f..b247f7ed973e 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -54,12 +54,11 @@ config CRYPTO_DEV_VMX_ENCRYPT select CRYPTO_AES select CRYPTO_CBC select CRYPTO_CTR - select CRYPTO_GHASH select CRYPTO_XTS default m help Support for VMX cryptographic acceleration instructions on Power8 CPU. - This module supports acceleration for AES and GHASH in hardware. If you - choose 'M' here, this module will be called vmx-crypto. + This module supports acceleration for AES in hardware. If you choose + 'M' here, this module will be called vmx-crypto. endmenu diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile index 3ac0886282a2..a1fe102a90ae 100644 --- a/arch/powerpc/crypto/Makefile +++ b/arch/powerpc/crypto/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o aes-ppc-spe-y := aes-spe-glue.o aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o -vmx-crypto-objs := vmx.o ghashp8-ppc.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o +vmx-crypto-objs := vmx.o aes_cbc.o aes_ctr.o aes_xts.o ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y) override flavour := linux-ppc64le @@ -26,14 +26,10 @@ endif quiet_cmd_perl = PERL $@ cmd_perl = $(PERL) $< $(flavour) > $@ -targets += aesp10-ppc.S ghashp10-ppc.S ghashp8-ppc.S +targets += aesp10-ppc.S ghashp10-ppc.S $(obj)/aesp10-ppc.S $(obj)/ghashp10-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE $(call if_changed,perl) -$(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE - $(call if_changed,perl) - OBJECT_FILES_NON_STANDARD_aesp10-ppc.o := y OBJECT_FILES_NON_STANDARD_ghashp10-ppc.o := y -OBJECT_FILES_NON_STANDARD_ghashp8-ppc.o := y diff --git a/arch/powerpc/crypto/aesp8-ppc.h b/arch/powerpc/crypto/aesp8-ppc.h index 6862c605cc33..c68f5b6965fa 100644 --- a/arch/powerpc/crypto/aesp8-ppc.h +++ b/arch/powerpc/crypto/aesp8-ppc.h @@ -2,7 +2,6 @@ #include #include -extern struct shash_alg p8_ghash_alg; extern struct skcipher_alg p8_aes_cbc_alg; extern struct skcipher_alg p8_aes_ctr_alg; extern struct skcipher_alg p8_aes_xts_alg; diff --git a/arch/powerpc/crypto/ghash.c b/arch/powerpc/crypto/ghash.c deleted file mode 100644 index 7308735bdb33..000000000000 --- a/arch/powerpc/crypto/ghash.c +++ /dev/null @@ -1,160 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * GHASH routines supporting VMX instructions on the Power 8 - * - * Copyright (C) 2015, 2019 International Business Machines Inc. - * - * Author: Marcelo Henrique Cerri - * - * Extended by Daniel Axtens to replace the fallback - * mechanism. The new approach is based on arm64 code, which is: - * Copyright (C) 2014 - 2018 Linaro Ltd. - */ - -#include "aesp8-ppc.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -void gcm_init_p8(u128 htable[16], const u64 Xi[2]); -void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); -void gcm_ghash_p8(u64 Xi[2], const u128 htable[16], - const u8 *in, size_t len); - -struct p8_ghash_ctx { - /* key used by vector asm */ - u128 htable[16]; - /* key used by software fallback */ - be128 key; -}; - -struct p8_ghash_desc_ctx { - u64 shash[2]; -}; - -static int p8_ghash_init(struct shash_desc *desc) -{ - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memset(dctx->shash, 0, GHASH_DIGEST_SIZE); - return 0; -} - -static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) -{ - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm)); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - gcm_init_p8(ctx->htable, (const u64 *) key); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - - memcpy(&ctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static inline void __ghash_block(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx, - const u8 *src) -{ - if (crypto_simd_usable()) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - gcm_ghash_p8(dctx->shash, ctx->htable, src, GHASH_BLOCK_SIZE); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - } else { - crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); - gf128mul_lle((be128 *)dctx->shash, &ctx->key); - } -} - -static inline int __ghash_blocks(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - int remain = srclen - round_down(srclen, GHASH_BLOCK_SIZE); - - srclen -= remain; - if (crypto_simd_usable()) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - gcm_ghash_p8(dctx->shash, ctx->htable, - src, srclen); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - } else { - do { - crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); - gf128mul_lle((be128 *)dctx->shash, &ctx->key); - srclen -= GHASH_BLOCK_SIZE; - src += GHASH_BLOCK_SIZE; - } while (srclen); - } - - return remain; -} - -static int p8_ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - return __ghash_blocks(ctx, dctx, src, srclen); -} - -static int p8_ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - __ghash_block(ctx, dctx, buf); - memzero_explicit(buf, sizeof(buf)); - } - memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); - return 0; -} - -struct shash_alg p8_ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = p8_ghash_init, - .update = p8_ghash_update, - .finup = p8_ghash_finup, - .setkey = p8_ghash_setkey, - .descsize = sizeof(struct p8_ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "p8_ghash", - .cra_priority = 1000, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct p8_ghash_ctx), - .cra_module = THIS_MODULE, - }, -}; diff --git a/arch/powerpc/crypto/vmx.c b/arch/powerpc/crypto/vmx.c index 7d2beb774f99..08da5311dfdf 100644 --- a/arch/powerpc/crypto/vmx.c +++ b/arch/powerpc/crypto/vmx.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include "aesp8-ppc.h" @@ -23,13 +22,9 @@ static int __init p8_init(void) { int ret; - ret = crypto_register_shash(&p8_ghash_alg); - if (ret) - goto err; - ret = crypto_register_skcipher(&p8_aes_cbc_alg); if (ret) - goto err_unregister_ghash; + goto err; ret = crypto_register_skcipher(&p8_aes_ctr_alg); if (ret) @@ -45,8 +40,6 @@ err_unregister_aes_ctr: crypto_unregister_skcipher(&p8_aes_ctr_alg); err_unregister_aes_cbc: crypto_unregister_skcipher(&p8_aes_cbc_alg); -err_unregister_ghash: - crypto_unregister_shash(&p8_ghash_alg); err: return ret; } @@ -56,7 +49,6 @@ static void __exit p8_exit(void) crypto_unregister_skcipher(&p8_aes_xts_alg); crypto_unregister_skcipher(&p8_aes_ctr_alg); crypto_unregister_skcipher(&p8_aes_cbc_alg); - crypto_unregister_shash(&p8_ghash_alg); } module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, p8_init); diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h index 5090fbaa87f8..650652dd6003 100644 --- a/include/crypto/gf128hash.h +++ b/include/crypto/gf128hash.h @@ -41,6 +41,10 @@ struct polyval_elem { * Use ghash_preparekey() to initialize this. */ struct ghash_key { +#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64) + /** @htable: GHASH key format used by the POWER8 assembly code */ + u64 htable[4][2]; +#endif /** @h: The hash key H, in POLYVAL format */ struct polyval_elem h; }; diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 4f1a79883a56..f54add7d9070 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -121,6 +121,7 @@ config CRYPTO_LIB_GF128HASH_ARCH depends on CRYPTO_LIB_GF128HASH && !UML default y if ARM && KERNEL_MODE_NEON default y if ARM64 + default y if PPC64 && VSX default y if X86_64 config CRYPTO_LIB_MD5 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 4ce0bac8fd93..8a9084188778 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -8,6 +8,10 @@ quiet_cmd_perlasm = PERLASM $@ quiet_cmd_perlasm_with_args = PERLASM $@ cmd_perlasm_with_args = $(PERL) $(<) void $(@) +ppc64-perlasm-flavour-y := linux-ppc64 +ppc64-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2 +ppc64-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le + obj-$(CONFIG_KUNIT) += tests/ obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o @@ -36,11 +40,8 @@ libaes-y += powerpc/aes-spe-core.o \ powerpc/aes-tab-4k.o else libaes-y += powerpc/aesp8-ppc.o -aes-perlasm-flavour-y := linux-ppc64 -aes-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2 -aes-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le quiet_cmd_perlasm_aes = PERLASM $@ - cmd_perlasm_aes = $(PERL) $< $(aes-perlasm-flavour-y) $@ + cmd_perlasm_aes = $(PERL) $< $(ppc64-perlasm-flavour-y) $@ # Use if_changed instead of cmd, in case the flavour changed. $(obj)/powerpc/aesp8-ppc.S: $(src)/powerpc/aesp8-ppc.pl FORCE $(call if_changed,perlasm_aes) @@ -161,9 +162,23 @@ CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH) libgf128hash-$(CONFIG_ARM) += arm/ghash-neon-core.o libgf128hash-$(CONFIG_ARM64) += arm64/ghash-neon-core.o \ arm64/polyval-ce-core.o -libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o + +ifeq ($(CONFIG_PPC),y) +libgf128hash-y += powerpc/ghashp8-ppc.o +quiet_cmd_perlasm_ghash = PERLASM $@ + cmd_perlasm_ghash = $(PERL) $< $(ppc64-perlasm-flavour-y) $@ +$(obj)/powerpc/ghashp8-ppc.S: $(src)/powerpc/ghashp8-ppc.pl FORCE + $(call if_changed,perlasm_ghash) +targets += powerpc/ghashp8-ppc.S +OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y endif +libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o +endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH + +# clean-files must be defined unconditionally +clean-files += powerpc/ghashp8-ppc.S + ################################################################################ obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o diff --git a/lib/crypto/powerpc/.gitignore b/lib/crypto/powerpc/.gitignore index 598ca7aff6b1..7aa71d83f739 100644 --- a/lib/crypto/powerpc/.gitignore +++ b/lib/crypto/powerpc/.gitignore @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only aesp8-ppc.S +ghashp8-ppc.S diff --git a/lib/crypto/powerpc/gf128hash.h b/lib/crypto/powerpc/gf128hash.h new file mode 100644 index 000000000000..629cd325d0c7 --- /dev/null +++ b/lib/crypto/powerpc/gf128hash.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * GHASH routines supporting VMX instructions on the Power 8 + * + * Copyright (C) 2015, 2019 International Business Machines Inc. + * Copyright (C) 2014 - 2018 Linaro Ltd. + * Copyright 2026 Google LLC + */ + +#include +#include +#include +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto); + +void gcm_init_p8(u64 htable[4][2], const u8 h[16]); +void gcm_gmult_p8(u8 Xi[16], const u64 htable[4][2]); +void gcm_ghash_p8(u8 Xi[16], const u64 htable[4][2], const u8 *in, size_t len); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + ghash_key_to_polyval(raw_key, &key->h); + + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_init_p8(key->htable, raw_key); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + } else { + /* This reproduces gcm_init_p8() on both LE and BE systems. */ + key->htable[0][0] = 0; + key->htable[0][1] = 0xc200000000000000; + + key->htable[1][0] = 0; + key->htable[1][1] = le64_to_cpu(key->h.lo); + + key->htable[2][0] = le64_to_cpu(key->h.lo); + key->htable[2][1] = le64_to_cpu(key->h.hi); + + key->htable[3][0] = le64_to_cpu(key->h.hi); + key->htable[3][1] = 0; + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_gmult_p8(ghash_acc, key->htable); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + polyval_mul_generic(acc, &key->h); + } +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + gcm_ghash_p8(ghash_acc, key->htable, data, + nblocks * GHASH_BLOCK_SIZE); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_has_feature(CPU_FTR_ARCH_207S) && + (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) + static_branch_enable(&have_vec_crypto); +} diff --git a/arch/powerpc/crypto/ghashp8-ppc.pl b/lib/crypto/powerpc/ghashp8-ppc.pl similarity index 98% rename from arch/powerpc/crypto/ghashp8-ppc.pl rename to lib/crypto/powerpc/ghashp8-ppc.pl index 041e633c214f..7c38eedc02cc 100644 --- a/arch/powerpc/crypto/ghashp8-ppc.pl +++ b/lib/crypto/powerpc/ghashp8-ppc.pl @@ -47,6 +47,7 @@ if ($flavour =~ /64/) { $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or +( $xlate="${dir}../../../arch/powerpc/crypto/ppc-xlate.pl" and -f $xlate) or die "can't locate ppc-xlate.pl"; open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!"; From af413d71f09d4dde28277319926c1c3a6ec8b8d4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:13 -0700 Subject: [PATCH 30/66] lib/crypto: riscv/ghash: Migrate optimized code into library Remove the "ghash-riscv64-zvkg" crypto_shash algorithm. Move the corresponding assembly code into lib/crypto/, modify it to take the length in blocks instead of bytes, and wire it up to the GHASH library. This makes the GHASH library be optimized with the RISC-V Vector Cryptography Extension. It also greatly reduces the amount of riscv-specific glue code that is needed, and it fixes the issue where this optimized GHASH code was disabled by default. Note that this RISC-V code has multiple opportunities for improvement, such as adding more parallelism, providing an optimized multiplication function, and directly supporting POLYVAL. But for now, this commit simply tweaks ghash_zvkg() slightly to make it compatible with the library, then wires it up to ghash_blocks_arch(). ghash_preparekey_arch() is also implemented to store the copy of the raw key needed by the vghsh.vv instruction. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-13-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/riscv/crypto/Kconfig | 11 -- arch/riscv/crypto/Makefile | 3 - arch/riscv/crypto/ghash-riscv64-glue.c | 146 ------------------ include/crypto/gf128hash.h | 3 + lib/crypto/Kconfig | 2 + lib/crypto/Makefile | 1 + lib/crypto/riscv/gf128hash.h | 57 +++++++ .../crypto/riscv}/ghash-riscv64-zvkg.S | 13 +- 8 files changed, 70 insertions(+), 166 deletions(-) delete mode 100644 arch/riscv/crypto/ghash-riscv64-glue.c create mode 100644 lib/crypto/riscv/gf128hash.h rename {arch/riscv/crypto => lib/crypto/riscv}/ghash-riscv64-zvkg.S (91%) diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index 22d4eaab15f3..c208f54afbcd 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -17,17 +17,6 @@ config CRYPTO_AES_RISCV64 - Zvkb vector crypto extension (CTR) - Zvkg vector crypto extension (XTS) -config CRYPTO_GHASH_RISCV64 - tristate "Hash functions: GHASH" - depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ - RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS - select CRYPTO_GCM - help - GCM GHASH function (NIST SP 800-38D) - - Architecture: riscv64 using: - - Zvkg vector crypto extension - config CRYPTO_SM3_RISCV64 tristate "Hash functions: SM3 (ShangMi 3)" depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index 183495a95cc0..5c9ee1b876fa 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -4,9 +4,6 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o -obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o -ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o - obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o diff --git a/arch/riscv/crypto/ghash-riscv64-glue.c b/arch/riscv/crypto/ghash-riscv64-glue.c deleted file mode 100644 index d86073d25387..000000000000 --- a/arch/riscv/crypto/ghash-riscv64-glue.c +++ /dev/null @@ -1,146 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * GHASH using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, - size_t len); - -struct riscv64_ghash_tfm_ctx { - be128 key; -}; - -struct riscv64_ghash_desc_ctx { - be128 accumulator; -}; - -static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) -{ - struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - memcpy(&tctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int riscv64_ghash_init(struct shash_desc *desc) -{ - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - *dctx = (struct riscv64_ghash_desc_ctx){}; - - return 0; -} - -static inline void -riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx, - struct riscv64_ghash_desc_ctx *dctx, - const u8 *src, size_t srclen) -{ - /* The srclen is nonzero and a multiple of 16. */ - if (crypto_simd_usable()) { - kernel_vector_begin(); - ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen); - kernel_vector_end(); - } else { - do { - crypto_xor((u8 *)&dctx->accumulator, src, - GHASH_BLOCK_SIZE); - gf128mul_lle(&dctx->accumulator, &tctx->key); - src += GHASH_BLOCK_SIZE; - srclen -= GHASH_BLOCK_SIZE; - } while (srclen); - } -} - -static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src, - unsigned int srclen) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - riscv64_ghash_blocks(tctx, dctx, src, - round_down(srclen, GHASH_BLOCK_SIZE)); - return srclen - round_down(srclen, GHASH_BLOCK_SIZE); -} - -static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE); - memzero_explicit(buf, sizeof(buf)); - } - - memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE); - return 0; -} - -static struct shash_alg riscv64_ghash_alg = { - .init = riscv64_ghash_init, - .update = riscv64_ghash_update, - .finup = riscv64_ghash_finup, - .setkey = riscv64_ghash_setkey, - .descsize = sizeof(struct riscv64_ghash_desc_ctx), - .digestsize = GHASH_DIGEST_SIZE, - .base = { - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx), - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_name = "ghash", - .cra_driver_name = "ghash-riscv64-zvkg", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_ghash_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKG) && - riscv_vector_vlen() >= 128) - return crypto_register_shash(&riscv64_ghash_alg); - - return -ENODEV; -} - -static void __exit riscv64_ghash_mod_exit(void) -{ - crypto_unregister_shash(&riscv64_ghash_alg); -} - -module_init(riscv64_ghash_mod_init); -module_exit(riscv64_ghash_mod_exit); - -MODULE_DESCRIPTION("GHASH (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner "); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("ghash"); diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h index 650652dd6003..b798438cce23 100644 --- a/include/crypto/gf128hash.h +++ b/include/crypto/gf128hash.h @@ -44,6 +44,9 @@ struct ghash_key { #if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64) /** @htable: GHASH key format used by the POWER8 assembly code */ u64 htable[4][2]; +#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_RISCV) + /** @h_raw: The hash key H, in GHASH format */ + u8 h_raw[GHASH_BLOCK_SIZE]; #endif /** @h: The hash key H, in POLYVAL format */ struct polyval_elem h; diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index f54add7d9070..027802e0de33 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -122,6 +122,8 @@ config CRYPTO_LIB_GF128HASH_ARCH default y if ARM && KERNEL_MODE_NEON default y if ARM64 default y if PPC64 && VSX + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS default y if X86_64 config CRYPTO_LIB_MD5 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 8a9084188778..8950509833af 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -173,6 +173,7 @@ targets += powerpc/ghashp8-ppc.S OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y endif +libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH diff --git a/lib/crypto/riscv/gf128hash.h b/lib/crypto/riscv/gf128hash.h new file mode 100644 index 000000000000..4301a0384f60 --- /dev/null +++ b/lib/crypto/riscv/gf128hash.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * GHASH, RISC-V optimized + * + * Copyright (C) 2023 VRULL GmbH + * Copyright (C) 2023 SiFive, Inc. + * Copyright 2026 Google LLC + */ + +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkg); + +asmlinkage void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], + const u8 key[GHASH_BLOCK_SIZE], + const u8 *data, size_t nblocks); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + /* Save key in POLYVAL format for fallback */ + ghash_key_to_polyval(raw_key, &key->h); + + /* Save key in GHASH format for zvkg */ + memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_zvkg) && likely(may_use_simd())) { + u8 ghash_acc[GHASH_BLOCK_SIZE]; + + polyval_acc_to_ghash(acc, ghash_acc); + + kernel_vector_begin(); + ghash_zvkg(ghash_acc, key->h_raw, data, nblocks); + kernel_vector_end(); + + ghash_acc_to_polyval(ghash_acc, acc); + memzero_explicit(ghash_acc, sizeof(ghash_acc)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKG) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_zvkg); +} diff --git a/arch/riscv/crypto/ghash-riscv64-zvkg.S b/lib/crypto/riscv/ghash-riscv64-zvkg.S similarity index 91% rename from arch/riscv/crypto/ghash-riscv64-zvkg.S rename to lib/crypto/riscv/ghash-riscv64-zvkg.S index f2b43fb4d434..6a2a2f2bc7c8 100644 --- a/arch/riscv/crypto/ghash-riscv64-zvkg.S +++ b/lib/crypto/riscv/ghash-riscv64-zvkg.S @@ -50,12 +50,13 @@ #define ACCUMULATOR a0 #define KEY a1 #define DATA a2 -#define LEN a3 +#define NBLOCKS a3 -// void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, -// size_t len); +// void ghash_zvkg(u8 accumulator[GHASH_BLOCK_SIZE], +// const u8 key[GHASH_BLOCK_SIZE], +// const u8 *data, size_t nblocks); // -// |len| must be nonzero and a multiple of 16 (GHASH_BLOCK_SIZE). +// |nblocks| must be nonzero. SYM_FUNC_START(ghash_zvkg) vsetivli zero, 4, e32, m1, ta, ma vle32.v v1, (ACCUMULATOR) @@ -64,8 +65,8 @@ SYM_FUNC_START(ghash_zvkg) vle32.v v3, (DATA) vghsh.vv v1, v2, v3 addi DATA, DATA, 16 - addi LEN, LEN, -16 - bnez LEN, .Lnext_block + addi NBLOCKS, NBLOCKS, -1 + bnez NBLOCKS, .Lnext_block vse32.v v1, (ACCUMULATOR) ret From efd1d2c8f3c073c43d5616d0c2d698cbe8a3ecde Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:14 -0700 Subject: [PATCH 31/66] lib/crypto: s390/ghash: Migrate optimized code into library Remove the "ghash-s390" crypto_shash algorithm, and replace it with an implementation of ghash_blocks_arch() for the GHASH library. This makes the GHASH library be optimized with CPACF. It also greatly reduces the amount of s390-specific glue code that is needed, and it fixes the issue where this GHASH optimization was disabled by default. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-14-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/s390/configs/debug_defconfig | 1 - arch/s390/configs/defconfig | 1 - arch/s390/crypto/Kconfig | 10 --- arch/s390/crypto/Makefile | 1 - arch/s390/crypto/ghash_s390.c | 144 ------------------------------ include/crypto/gf128hash.h | 3 +- lib/crypto/Kconfig | 1 + lib/crypto/s390/gf128hash.h | 54 +++++++++++ 8 files changed, 57 insertions(+), 158 deletions(-) delete mode 100644 arch/s390/crypto/ghash_s390.c create mode 100644 lib/crypto/s390/gf128hash.h diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 98fd0a2f51c6..aa862d4fcc68 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -809,7 +809,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_HMAC_S390=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 0f4cedcab3ce..74f943307c46 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -794,7 +794,6 @@ CONFIG_CRYPTO_USER_API_HASH=m CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_HMAC_S390=m diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index 79a2d0034258..ee83052dbc15 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -2,16 +2,6 @@ menu "Accelerated Cryptographic Algorithms for CPU (s390)" -config CRYPTO_GHASH_S390 - tristate "Hash functions: GHASH" - select CRYPTO_HASH - help - GCM GHASH hash function (NIST SP800-38D) - - Architecture: s390 - - It is available as of z196. - config CRYPTO_AES_S390 tristate "Ciphers: AES, modes: ECB, CBC, CTR, XTS, GCM" select CRYPTO_SKCIPHER diff --git a/arch/s390/crypto/Makefile b/arch/s390/crypto/Makefile index 387a229e1038..4449c1b19ef5 100644 --- a/arch/s390/crypto/Makefile +++ b/arch/s390/crypto/Makefile @@ -7,7 +7,6 @@ obj-$(CONFIG_CRYPTO_DES_S390) += des_s390.o obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o obj-$(CONFIG_CRYPTO_PAES_S390) += paes_s390.o obj-$(CONFIG_S390_PRNG) += prng.o -obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o obj-$(CONFIG_CRYPTO_HMAC_S390) += hmac_s390.o obj-$(CONFIG_CRYPTO_PHMAC_S390) += phmac_s390.o obj-y += arch_random.o diff --git a/arch/s390/crypto/ghash_s390.c b/arch/s390/crypto/ghash_s390.c deleted file mode 100644 index dcbcee37cb63..000000000000 --- a/arch/s390/crypto/ghash_s390.c +++ /dev/null @@ -1,144 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Cryptographic API. - * - * s390 implementation of the GHASH algorithm for GCM (Galois/Counter Mode). - * - * Copyright IBM Corp. 2011 - * Author(s): Gerald Schaefer - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -struct s390_ghash_ctx { - u8 key[GHASH_BLOCK_SIZE]; -}; - -struct s390_ghash_desc_ctx { - u8 icv[GHASH_BLOCK_SIZE]; - u8 key[GHASH_BLOCK_SIZE]; -}; - -static int ghash_init(struct shash_desc *desc) -{ - struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memset(dctx, 0, sizeof(*dctx)); - memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen) -{ - struct s390_ghash_ctx *ctx = crypto_shash_ctx(tfm); - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - memcpy(ctx->key, key, GHASH_BLOCK_SIZE); - - return 0; -} - -static int ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int n; - - n = srclen & ~(GHASH_BLOCK_SIZE - 1); - cpacf_kimd(CPACF_KIMD_GHASH, dctx, src, n); - return srclen - n; -} - -static void ghash_flush(struct s390_ghash_desc_ctx *dctx, const u8 *src, - unsigned int len) -{ - if (len) { - u8 buf[GHASH_BLOCK_SIZE] = {}; - - memcpy(buf, src, len); - cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE); - memzero_explicit(buf, sizeof(buf)); - } -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - ghash_flush(dctx, src, len); - memcpy(dst, dctx->icv, GHASH_BLOCK_SIZE); - return 0; -} - -static int ghash_export(struct shash_desc *desc, void *out) -{ - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memcpy(out, dctx->icv, GHASH_DIGEST_SIZE); - return 0; -} - -static int ghash_import(struct shash_desc *desc, const void *in) -{ - struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memcpy(dctx->icv, in, GHASH_DIGEST_SIZE); - memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE); - return 0; -} - -static struct shash_alg ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .export = ghash_export, - .import = ghash_import, - .statesize = sizeof(struct ghash_desc_ctx), - .descsize = sizeof(struct s390_ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "ghash-s390", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct s390_ghash_ctx), - .cra_module = THIS_MODULE, - }, -}; - -static int __init ghash_mod_init(void) -{ - if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH)) - return -ENODEV; - - return crypto_register_shash(&ghash_alg); -} - -static void __exit ghash_mod_exit(void) -{ - crypto_unregister_shash(&ghash_alg); -} - -module_cpu_feature_match(S390_CPU_FEATURE_MSA, ghash_mod_init); -module_exit(ghash_mod_exit); - -MODULE_ALIAS_CRYPTO("ghash"); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("GHASH hash function, s390 implementation"); diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h index b798438cce23..0bc649d01e12 100644 --- a/include/crypto/gf128hash.h +++ b/include/crypto/gf128hash.h @@ -44,7 +44,8 @@ struct ghash_key { #if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64) /** @htable: GHASH key format used by the POWER8 assembly code */ u64 htable[4][2]; -#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_RISCV) +#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \ + (defined(CONFIG_RISCV) || defined(CONFIG_S390)) /** @h_raw: The hash key H, in GHASH format */ u8 h_raw[GHASH_BLOCK_SIZE]; #endif diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 027802e0de33..a39e7707e9ee 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -124,6 +124,7 @@ config CRYPTO_LIB_GF128HASH_ARCH default y if PPC64 && VSX default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS + default y if S390 default y if X86_64 config CRYPTO_LIB_MD5 diff --git a/lib/crypto/s390/gf128hash.h b/lib/crypto/s390/gf128hash.h new file mode 100644 index 000000000000..1e46ce4bca40 --- /dev/null +++ b/lib/crypto/s390/gf128hash.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GHASH optimized using the CP Assist for Cryptographic Functions (CPACF) + * + * Copyright 2026 Google LLC + */ +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_ghash); + +#define ghash_preparekey_arch ghash_preparekey_arch +static void ghash_preparekey_arch(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]) +{ + /* Save key in POLYVAL format for fallback */ + ghash_key_to_polyval(raw_key, &key->h); + + /* Save key in GHASH format for CPACF_KIMD_GHASH */ + memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_cpacf_ghash)) { + /* + * CPACF_KIMD_GHASH requires the accumulator and key in a single + * buffer, each using the GHASH convention. + */ + u8 ctx[2][GHASH_BLOCK_SIZE] __aligned(8); + + polyval_acc_to_ghash(acc, ctx[0]); + memcpy(ctx[1], key->h_raw, GHASH_BLOCK_SIZE); + + cpacf_kimd(CPACF_KIMD_GHASH, ctx, data, + nblocks * GHASH_BLOCK_SIZE); + + ghash_acc_to_polyval(ctx[0], acc); + memzero_explicit(ctx, sizeof(ctx)); + } else { + ghash_blocks_generic(acc, &key->h, data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_have_feature(S390_CPU_FEATURE_MSA) && + cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH)) + static_branch_enable(&have_cpacf_ghash); +} From 3e79c8ec49596288c4460029c4971b9c838103b9 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:15 -0700 Subject: [PATCH 32/66] lib/crypto: x86/ghash: Migrate optimized code into library Remove the "ghash-pclmulqdqni" crypto_shash algorithm. Move the corresponding assembly code into lib/crypto/, and wire it up to the GHASH library. This makes the GHASH library be optimized with x86's carryless multiplication instructions. It also greatly reduces the amount of x86-specific glue code that is needed, and it fixes the issue where this GHASH optimization was disabled by default. Rename and adjust the prototypes of the assembly functions to make them fit better with the library. Remove the byte-swaps (pshufb instructions) that are no longer necessary because the library keeps the accumulator in POLYVAL format rather than GHASH format. Rename clmul_ghash_mul() to polyval_mul_pclmul() to reflect that it really does a POLYVAL style multiplication. Wire it up to both ghash_mul_arch() and polyval_mul_arch(). Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-15-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/x86/crypto/Kconfig | 10 -- arch/x86/crypto/Makefile | 3 - arch/x86/crypto/ghash-clmulni-intel_glue.c | 163 ------------------ lib/crypto/Makefile | 3 +- lib/crypto/x86/gf128hash.h | 65 ++++++- .../crypto/x86/ghash-pclmul.S | 98 +++++------ 6 files changed, 104 insertions(+), 238 deletions(-) delete mode 100644 arch/x86/crypto/ghash-clmulni-intel_glue.c rename arch/x86/crypto/ghash-clmulni-intel_asm.S => lib/crypto/x86/ghash-pclmul.S (54%) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 7fb2319a0916..905e8a23cec3 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -344,14 +344,4 @@ config CRYPTO_SM3_AVX_X86_64 If unsure, say N. -config CRYPTO_GHASH_CLMUL_NI_INTEL - tristate "Hash functions: GHASH (CLMUL-NI)" - depends on 64BIT - select CRYPTO_CRYPTD - help - GCM GHASH hash function (NIST SP800-38D) - - Architecture: x86_64 using: - - CLMUL-NI (carry-less multiplication new instructions) - endmenu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index b21ad0978c52..d562f4341da6 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -50,9 +50,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \ aes-gcm-vaes-avx512.o \ aes-xts-avx-x86_64.o -obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o -ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o - obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c deleted file mode 100644 index aea5d4d06be7..000000000000 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ /dev/null @@ -1,163 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Accelerated GHASH implementation with Intel PCLMULQDQ-NI - * instructions. This file contains glue code. - * - * Copyright (c) 2009 Intel Corp. - * Author: Huang Ying - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void clmul_ghash_mul(char *dst, const le128 *shash); - -asmlinkage int clmul_ghash_update(char *dst, const char *src, - unsigned int srclen, const le128 *shash); - -struct x86_ghash_ctx { - le128 shash; -}; - -static int ghash_init(struct shash_desc *desc) -{ - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memset(dctx, 0, sizeof(*dctx)); - - return 0; -} - -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen) -{ - struct x86_ghash_ctx *ctx = crypto_shash_ctx(tfm); - u64 a, b; - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - /* - * GHASH maps bits to polynomial coefficients backwards, which makes it - * hard to implement. But it can be shown that the GHASH multiplication - * - * D * K (mod x^128 + x^7 + x^2 + x + 1) - * - * (where D is a data block and K is the key) is equivalent to: - * - * bitreflect(D) * bitreflect(K) * x^(-127) - * (mod x^128 + x^127 + x^126 + x^121 + 1) - * - * So, the code below precomputes: - * - * bitreflect(K) * x^(-127) (mod x^128 + x^127 + x^126 + x^121 + 1) - * - * ... but in Montgomery form (so that Montgomery multiplication can be - * used), i.e. with an extra x^128 factor, which means actually: - * - * bitreflect(K) * x (mod x^128 + x^127 + x^126 + x^121 + 1) - * - * The within-a-byte part of bitreflect() cancels out GHASH's built-in - * reflection, and thus bitreflect() is actually a byteswap. - */ - a = get_unaligned_be64(key); - b = get_unaligned_be64(key + 8); - ctx->shash.a = cpu_to_le64((a << 1) | (b >> 63)); - ctx->shash.b = cpu_to_le64((b << 1) | (a >> 63)); - if (a >> 63) - ctx->shash.a ^= cpu_to_le64((u64)0xc2 << 56); - return 0; -} - -static int ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct x86_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - u8 *dst = dctx->buffer; - int remain; - - kernel_fpu_begin(); - remain = clmul_ghash_update(dst, src, srclen, &ctx->shash); - kernel_fpu_end(); - return remain; -} - -static void ghash_flush(struct x86_ghash_ctx *ctx, struct ghash_desc_ctx *dctx, - const u8 *src, unsigned int len) -{ - u8 *dst = dctx->buffer; - - kernel_fpu_begin(); - if (len) { - crypto_xor(dst, src, len); - clmul_ghash_mul(dst, &ctx->shash); - } - kernel_fpu_end(); -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct x86_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - u8 *buf = dctx->buffer; - - ghash_flush(ctx, dctx, src, len); - memcpy(dst, buf, GHASH_BLOCK_SIZE); - - return 0; -} - -static struct shash_alg ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .descsize = sizeof(struct ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "ghash-pclmulqdqni", - .cra_priority = 400, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct x86_ghash_ctx), - .cra_module = THIS_MODULE, - }, -}; - -static const struct x86_cpu_id pcmul_cpu_id[] = { - X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), /* Pickle-Mickle-Duck */ - {} -}; -MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id); - -static int __init ghash_pclmulqdqni_mod_init(void) -{ - if (!x86_match_cpu(pcmul_cpu_id)) - return -ENODEV; - - return crypto_register_shash(&ghash_alg); -} - -static void __exit ghash_pclmulqdqni_mod_exit(void) -{ - crypto_unregister_shash(&ghash_alg); -} - -module_init(ghash_pclmulqdqni_mod_init); -module_exit(ghash_pclmulqdqni_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("GHASH hash function, accelerated by PCLMULQDQ-NI"); -MODULE_ALIAS_CRYPTO("ghash"); diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 8950509833af..19c67f70fb38 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -174,7 +174,8 @@ OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y endif libgf128hash-$(CONFIG_RISCV) += riscv/ghash-riscv64-zvkg.o -libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o +libgf128hash-$(CONFIG_X86) += x86/ghash-pclmul.o \ + x86/polyval-pclmul-avx.o endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH # clean-files must be defined unconditionally diff --git a/lib/crypto/x86/gf128hash.h b/lib/crypto/x86/gf128hash.h index adf6147ea677..6b79b06caab0 100644 --- a/lib/crypto/x86/gf128hash.h +++ b/lib/crypto/x86/gf128hash.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * POLYVAL library functions, x86_64 optimized + * GHASH and POLYVAL, x86_64 optimized * * Copyright 2025 Google LLC */ @@ -9,10 +9,17 @@ #define NUM_H_POWERS 8 +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx); +asmlinkage void polyval_mul_pclmul(struct polyval_elem *a, + const struct polyval_elem *b); asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a, const struct polyval_elem *b); + +asmlinkage void ghash_blocks_pclmul(struct polyval_elem *acc, + const struct polyval_elem *key, + const u8 *data, size_t nblocks); asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc, const struct polyval_key *key, const u8 *data, size_t nblocks); @@ -41,16 +48,54 @@ static void polyval_preparekey_arch(struct polyval_key *key, } } +static void polyval_mul_x86(struct polyval_elem *a, + const struct polyval_elem *b) +{ + if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) { + kernel_fpu_begin(); + if (static_branch_likely(&have_pclmul_avx)) + polyval_mul_pclmul_avx(a, b); + else + polyval_mul_pclmul(a, b); + kernel_fpu_end(); + } else { + polyval_mul_generic(a, b); + } +} + +#define ghash_mul_arch ghash_mul_arch +static void ghash_mul_arch(struct polyval_elem *acc, + const struct ghash_key *key) +{ + polyval_mul_x86(acc, &key->h); +} + #define polyval_mul_arch polyval_mul_arch static void polyval_mul_arch(struct polyval_elem *acc, const struct polyval_key *key) { - if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { - kernel_fpu_begin(); - polyval_mul_pclmul_avx(acc, &key->h_powers[NUM_H_POWERS - 1]); - kernel_fpu_end(); + polyval_mul_x86(acc, &key->h_powers[NUM_H_POWERS - 1]); +} + +#define ghash_blocks_arch ghash_blocks_arch +static void ghash_blocks_arch(struct polyval_elem *acc, + const struct ghash_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) { + do { + /* Allow rescheduling every 4 KiB. */ + size_t n = min_t(size_t, nblocks, + 4096 / GHASH_BLOCK_SIZE); + + kernel_fpu_begin(); + ghash_blocks_pclmul(acc, &key->h, data, n); + kernel_fpu_end(); + data += n * GHASH_BLOCK_SIZE; + nblocks -= n; + } while (nblocks); } else { - polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); + ghash_blocks_generic(acc, &key->h, data, nblocks); } } @@ -80,7 +125,9 @@ static void polyval_blocks_arch(struct polyval_elem *acc, #define gf128hash_mod_init_arch gf128hash_mod_init_arch static void gf128hash_mod_init_arch(void) { - if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) && - boot_cpu_has(X86_FEATURE_AVX)) - static_branch_enable(&have_pclmul_avx); + if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { + static_branch_enable(&have_pclmul); + if (boot_cpu_has(X86_FEATURE_AVX)) + static_branch_enable(&have_pclmul_avx); + } } diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/lib/crypto/x86/ghash-pclmul.S similarity index 54% rename from arch/x86/crypto/ghash-clmulni-intel_asm.S rename to lib/crypto/x86/ghash-pclmul.S index c4fbaa82ed7a..6ffb5aea6063 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_asm.S +++ b/lib/crypto/x86/ghash-pclmul.S @@ -21,8 +21,8 @@ .Lbswap_mask: .octa 0x000102030405060708090a0b0c0d0e0f -#define DATA %xmm0 -#define SHASH %xmm1 +#define ACC %xmm0 +#define KEY %xmm1 #define T1 %xmm2 #define T2 %xmm3 #define T3 %xmm4 @@ -34,100 +34,94 @@ /* * __clmul_gf128mul_ble: internal ABI * input: - * DATA: operand1 - * SHASH: operand2, hash_key << 1 mod poly + * ACC: operand1 + * KEY: operand2, hash_key << 1 mod poly * output: - * DATA: operand1 * operand2 mod poly + * ACC: operand1 * operand2 mod poly * changed: * T1 * T2 * T3 */ SYM_FUNC_START_LOCAL(__clmul_gf128mul_ble) - movaps DATA, T1 - pshufd $0b01001110, DATA, T2 - pshufd $0b01001110, SHASH, T3 - pxor DATA, T2 - pxor SHASH, T3 + movaps ACC, T1 + pshufd $0b01001110, ACC, T2 + pshufd $0b01001110, KEY, T3 + pxor ACC, T2 + pxor KEY, T3 - pclmulqdq $0x00, SHASH, DATA # DATA = a0 * b0 - pclmulqdq $0x11, SHASH, T1 # T1 = a1 * b1 + pclmulqdq $0x00, KEY, ACC # ACC = a0 * b0 + pclmulqdq $0x11, KEY, T1 # T1 = a1 * b1 pclmulqdq $0x00, T3, T2 # T2 = (a1 + a0) * (b1 + b0) - pxor DATA, T2 + pxor ACC, T2 pxor T1, T2 # T2 = a0 * b1 + a1 * b0 movaps T2, T3 pslldq $8, T3 psrldq $8, T2 - pxor T3, DATA - pxor T2, T1 # is result of + pxor T3, ACC + pxor T2, T1 # is result of # carry-less multiplication # first phase of the reduction - movaps DATA, T3 + movaps ACC, T3 psllq $1, T3 - pxor DATA, T3 + pxor ACC, T3 psllq $5, T3 - pxor DATA, T3 + pxor ACC, T3 psllq $57, T3 movaps T3, T2 pslldq $8, T2 psrldq $8, T3 - pxor T2, DATA + pxor T2, ACC pxor T3, T1 # second phase of the reduction - movaps DATA, T2 + movaps ACC, T2 psrlq $5, T2 - pxor DATA, T2 + pxor ACC, T2 psrlq $1, T2 - pxor DATA, T2 + pxor ACC, T2 psrlq $1, T2 pxor T2, T1 - pxor T1, DATA + pxor T1, ACC RET SYM_FUNC_END(__clmul_gf128mul_ble) -/* void clmul_ghash_mul(char *dst, const le128 *shash) */ -SYM_FUNC_START(clmul_ghash_mul) +/* + * void polyval_mul_pclmul(struct polyval_elem *a, + * const struct polyval_elem *b) + */ +SYM_FUNC_START(polyval_mul_pclmul) FRAME_BEGIN - movups (%rdi), DATA - movups (%rsi), SHASH - movaps .Lbswap_mask(%rip), BSWAP - pshufb BSWAP, DATA + movups (%rdi), ACC + movups (%rsi), KEY call __clmul_gf128mul_ble - pshufb BSWAP, DATA - movups DATA, (%rdi) + movups ACC, (%rdi) FRAME_END RET -SYM_FUNC_END(clmul_ghash_mul) +SYM_FUNC_END(polyval_mul_pclmul) /* - * int clmul_ghash_update(char *dst, const char *src, unsigned int srclen, - * const le128 *shash); + * void ghash_blocks_pclmul(struct polyval_elem *acc, + * const struct polyval_elem *key, + * const u8 *data, size_t nblocks) */ -SYM_FUNC_START(clmul_ghash_update) +SYM_FUNC_START(ghash_blocks_pclmul) FRAME_BEGIN - cmp $16, %rdx - jb .Lupdate_just_ret # check length movaps .Lbswap_mask(%rip), BSWAP - movups (%rdi), DATA - movups (%rcx), SHASH - pshufb BSWAP, DATA + movups (%rdi), ACC + movups (%rsi), KEY .align 4 -.Lupdate_loop: - movups (%rsi), IN1 +.Lnext_block: + movups (%rdx), IN1 pshufb BSWAP, IN1 - pxor IN1, DATA + pxor IN1, ACC call __clmul_gf128mul_ble - sub $16, %rdx - add $16, %rsi - cmp $16, %rdx - jge .Lupdate_loop - pshufb BSWAP, DATA - movups DATA, (%rdi) -.Lupdate_just_ret: - mov %rdx, %rax + add $16, %rdx + dec %rcx + jnz .Lnext_block + movups ACC, (%rdi) FRAME_END RET -SYM_FUNC_END(clmul_ghash_update) +SYM_FUNC_END(ghash_blocks_pclmul) From 9f4e9553a1f40841ebce9ab749896e9312b1701b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:16 -0700 Subject: [PATCH 33/66] crypto: gcm - Use GHASH library instead of crypto_ahash Make the "gcm" template access GHASH using the library API instead of crypto_ahash. This is much simpler and more efficient, especially given that all GHASH implementations are synchronous and CPU-based anyway. Note that this allows "ghash" to be removed from the crypto_ahash (and crypto_shash) API, which a later commit will do. This mirrors the similar cleanup that was done with POLYVAL. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-16-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 2 +- crypto/gcm.c | 427 ++++++--------------------- crypto/testmgr.c | 10 +- drivers/crypto/starfive/jh7110-aes.c | 2 +- 4 files changed, 92 insertions(+), 349 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 5627b3691561..13ccf5ac2f1a 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -794,7 +794,7 @@ config CRYPTO_GCM tristate "GCM (Galois/Counter Mode) and GMAC (GCM MAC)" select CRYPTO_CTR select CRYPTO_AEAD - select CRYPTO_GHASH + select CRYPTO_LIB_GF128HASH select CRYPTO_MANAGER help GCM (Galois/Counter Mode) authenticated encryption mode and GMAC diff --git a/crypto/gcm.c b/crypto/gcm.c index e1e878d37410..5f16b237b3c5 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -5,13 +5,11 @@ * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen */ -#include #include #include -#include #include #include -#include +#include #include #include #include @@ -20,12 +18,11 @@ struct gcm_instance_ctx { struct crypto_skcipher_spawn ctr; - struct crypto_ahash_spawn ghash; }; struct crypto_gcm_ctx { struct crypto_skcipher *ctr; - struct crypto_ahash *ghash; + struct ghash_key ghash; }; struct crypto_rfc4106_ctx { @@ -52,31 +49,15 @@ struct crypto_rfc4543_req_ctx { struct aead_request subreq; }; -struct crypto_gcm_ghash_ctx { - unsigned int cryptlen; - struct scatterlist *src; - int (*complete)(struct aead_request *req, u32 flags); -}; - struct crypto_gcm_req_priv_ctx { u8 iv[16]; u8 auth_tag[16]; u8 iauth_tag[16]; struct scatterlist src[3]; struct scatterlist dst[3]; - struct scatterlist sg; - struct crypto_gcm_ghash_ctx ghash_ctx; - union { - struct ahash_request ahreq; - struct skcipher_request skreq; - } u; + struct skcipher_request skreq; /* Must be last */ }; -static struct { - u8 buf[16]; - struct scatterlist sg; -} *gcm_zeroes; - static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( struct aead_request *req) { @@ -89,10 +70,9 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); - struct crypto_ahash *ghash = ctx->ghash; struct crypto_skcipher *ctr = ctx->ctr; struct { - be128 hash; + u8 h[GHASH_BLOCK_SIZE]; u8 iv[16]; struct crypto_wait wait; @@ -115,14 +95,14 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, return -ENOMEM; crypto_init_wait(&data->wait); - sg_init_one(data->sg, &data->hash, sizeof(data->hash)); + sg_init_one(data->sg, data->h, sizeof(data->h)); skcipher_request_set_tfm(&data->req, ctr); skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &data->wait); skcipher_request_set_crypt(&data->req, data->sg, data->sg, - sizeof(data->hash), data->iv); + sizeof(data->h), data->iv); err = crypto_wait_req(crypto_skcipher_encrypt(&data->req), &data->wait); @@ -130,10 +110,7 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (err) goto out; - crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); - crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & - CRYPTO_TFM_REQ_MASK); - err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); + ghash_preparekey(&ctx->ghash, data->h); out: kfree_sensitive(data); return err; @@ -176,7 +153,7 @@ static void crypto_gcm_init_crypt(struct aead_request *req, struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct skcipher_request *skreq = &pctx->u.skreq; + struct skcipher_request *skreq = &pctx->skreq; struct scatterlist *dst; dst = req->src == req->dst ? pctx->src : pctx->dst; @@ -187,244 +164,65 @@ static void crypto_gcm_init_crypt(struct aead_request *req, pctx->iv); } -static inline unsigned int gcm_remain(unsigned int len) +static void ghash_update_sg_and_pad(struct ghash_ctx *ghash, + struct scatterlist *sg, unsigned int len) { - len &= 0xfU; - return len ? 16 - len : 0; + static const u8 zeroes[GHASH_BLOCK_SIZE]; + + if (len) { + unsigned int pad_len = -len % GHASH_BLOCK_SIZE; + struct scatter_walk walk; + + scatterwalk_start(&walk, sg); + do { + unsigned int n = scatterwalk_next(&walk, len); + + ghash_update(ghash, walk.addr, n); + scatterwalk_done_src(&walk, n); + len -= n; + } while (len); + + if (pad_len) + ghash_update(ghash, zeroes, pad_len); + } } -static void gcm_hash_len_done(void *data, int err); - -static int gcm_hash_update(struct aead_request *req, - crypto_completion_t compl, - struct scatterlist *src, - unsigned int len, u32 flags) +static void gcm_hash(struct aead_request *req, struct scatterlist *ctext, + unsigned int datalen, u8 out[GHASH_BLOCK_SIZE]) { - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ahash_request *ahreq = &pctx->u.ahreq; + const struct crypto_gcm_ctx *ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + __be64 lengths[2] = { + cpu_to_be64(8 * (u64)req->assoclen), + cpu_to_be64(8 * (u64)datalen), + }; + struct ghash_ctx ghash; - ahash_request_set_callback(ahreq, flags, compl, req); - ahash_request_set_crypt(ahreq, src, NULL, len); + ghash_init(&ghash, &ctx->ghash); - return crypto_ahash_update(ahreq); + /* Associated data, then zero-padding to the next 16-byte boundary */ + ghash_update_sg_and_pad(&ghash, req->src, req->assoclen); + + /* Ciphertext, then zero-padding to the next 16-byte boundary */ + ghash_update_sg_and_pad(&ghash, ctext, datalen); + + /* Lengths block */ + ghash_update(&ghash, (const u8 *)lengths, sizeof(lengths)); + + ghash_final(&ghash, out); } -static int gcm_hash_remain(struct aead_request *req, - unsigned int remain, - crypto_completion_t compl, u32 flags) +static int gcm_add_auth_tag(struct aead_request *req) { - return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags); -} - -static int gcm_hash_len(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ahash_request *ahreq = &pctx->u.ahreq; - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - be128 lengths; - - lengths.a = cpu_to_be64(req->assoclen * 8); - lengths.b = cpu_to_be64(gctx->cryptlen * 8); - memcpy(pctx->iauth_tag, &lengths, 16); - sg_init_one(&pctx->sg, pctx->iauth_tag, 16); - ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req); - ahash_request_set_crypt(ahreq, &pctx->sg, - pctx->iauth_tag, sizeof(lengths)); - - return crypto_ahash_finup(ahreq); -} - -static int gcm_hash_len_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - - return gctx->complete(req, flags); -} - -static void gcm_hash_len_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_len_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags) -{ - return gcm_hash_len(req, flags) ?: - gcm_hash_len_continue(req, flags); -} - -static void gcm_hash_crypt_remain_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_crypt_remain_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - unsigned int remain; - - remain = gcm_remain(gctx->cryptlen); - if (remain) - return gcm_hash_remain(req, remain, - gcm_hash_crypt_remain_done, flags) ?: - gcm_hash_crypt_remain_continue(req, flags); - - return gcm_hash_crypt_remain_continue(req, flags); -} - -static void gcm_hash_crypt_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_crypt_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - - if (gctx->cryptlen) - return gcm_hash_update(req, gcm_hash_crypt_done, - gctx->src, gctx->cryptlen, flags) ?: - gcm_hash_crypt_continue(req, flags); - - return gcm_hash_crypt_remain_continue(req, flags); -} - -static void gcm_hash_assoc_remain_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_assoc_remain_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags) -{ - unsigned int remain; - - remain = gcm_remain(req->assoclen); - if (remain) - return gcm_hash_remain(req, remain, - gcm_hash_assoc_remain_done, flags) ?: - gcm_hash_assoc_remain_continue(req, flags); - - return gcm_hash_assoc_remain_continue(req, flags); -} - -static void gcm_hash_assoc_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_assoc_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_init_continue(struct aead_request *req, u32 flags) -{ - if (req->assoclen) - return gcm_hash_update(req, gcm_hash_assoc_done, - req->src, req->assoclen, flags) ?: - gcm_hash_assoc_continue(req, flags); - - return gcm_hash_assoc_remain_continue(req, flags); -} - -static void gcm_hash_init_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_init_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ahash_request *ahreq = &pctx->u.ahreq; - struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - - ahash_request_set_tfm(ahreq, ctx->ghash); - - ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req); - return crypto_ahash_init(ahreq) ?: - gcm_hash_init_continue(req, flags); -} - -static int gcm_enc_copy_hash(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_aead *aead = crypto_aead_reqtfm(req); - u8 *auth_tag = pctx->auth_tag; - - crypto_xor(auth_tag, pctx->iauth_tag, 16); - scatterwalk_map_and_copy(auth_tag, req->dst, - req->assoclen + req->cryptlen, - crypto_aead_authsize(aead), 1); - return 0; -} - -static int gcm_encrypt_continue(struct aead_request *req, u32 flags) -{ struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst); - gctx->cryptlen = req->cryptlen; - gctx->complete = gcm_enc_copy_hash; - - return gcm_hash(req, flags); + gcm_hash(req, sg_next(req->src == req->dst ? pctx->src : pctx->dst), + req->cryptlen, pctx->iauth_tag); + crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); + memcpy_to_sglist(req->dst, req->assoclen + req->cryptlen, + pctx->auth_tag, crypto_aead_authsize(aead)); + return 0; } static void gcm_encrypt_done(void *data, int err) @@ -434,9 +232,7 @@ static void gcm_encrypt_done(void *data, int err) if (err) goto out; - err = gcm_encrypt_continue(req, 0); - if (err == -EINPROGRESS) - return; + err = gcm_add_auth_tag(req); out: aead_request_complete(req, err); @@ -445,15 +241,14 @@ out: static int crypto_gcm_encrypt(struct aead_request *req) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct skcipher_request *skreq = &pctx->u.skreq; + struct skcipher_request *skreq = &pctx->skreq; u32 flags = aead_request_flags(req); crypto_gcm_init_common(req); crypto_gcm_init_crypt(req, req->cryptlen); skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req); - return crypto_skcipher_encrypt(skreq) ?: - gcm_encrypt_continue(req, flags); + return crypto_skcipher_encrypt(skreq) ?: gcm_add_auth_tag(req); } static int crypto_gcm_verify(struct aead_request *req) @@ -481,35 +276,21 @@ static void gcm_decrypt_done(void *data, int err) aead_request_complete(req, err); } -static int gcm_dec_hash_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct skcipher_request *skreq = &pctx->u.skreq; - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - - crypto_gcm_init_crypt(req, gctx->cryptlen); - skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req); - return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); -} - static int crypto_gcm_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - unsigned int authsize = crypto_aead_authsize(aead); - unsigned int cryptlen = req->cryptlen; - u32 flags = aead_request_flags(req); - - cryptlen -= authsize; + struct skcipher_request *skreq = &pctx->skreq; + unsigned int datalen = req->cryptlen - crypto_aead_authsize(aead); crypto_gcm_init_common(req); - gctx->src = sg_next(pctx->src); - gctx->cryptlen = cryptlen; - gctx->complete = gcm_dec_hash_continue; + gcm_hash(req, sg_next(pctx->src), datalen, pctx->iauth_tag); - return gcm_hash(req, flags); + crypto_gcm_init_crypt(req, datalen); + skcipher_request_set_callback(skreq, aead_request_flags(req), + gcm_decrypt_done, req); + return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); } static int crypto_gcm_init_tfm(struct crypto_aead *tfm) @@ -518,43 +299,26 @@ static int crypto_gcm_init_tfm(struct crypto_aead *tfm) struct gcm_instance_ctx *ictx = aead_instance_ctx(inst); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_skcipher *ctr; - struct crypto_ahash *ghash; unsigned long align; - int err; - - ghash = crypto_spawn_ahash(&ictx->ghash); - if (IS_ERR(ghash)) - return PTR_ERR(ghash); ctr = crypto_spawn_skcipher(&ictx->ctr); - err = PTR_ERR(ctr); if (IS_ERR(ctr)) - goto err_free_hash; + return PTR_ERR(ctr); ctx->ctr = ctr; - ctx->ghash = ghash; align = crypto_aead_alignmask(tfm); align &= ~(crypto_tfm_ctx_alignment() - 1); crypto_aead_set_reqsize(tfm, - align + offsetof(struct crypto_gcm_req_priv_ctx, u) + - max(sizeof(struct skcipher_request) + - crypto_skcipher_reqsize(ctr), - sizeof(struct ahash_request) + - crypto_ahash_reqsize(ghash))); - + align + sizeof(struct crypto_gcm_req_priv_ctx) + + crypto_skcipher_reqsize(ctr)); return 0; - -err_free_hash: - crypto_free_ahash(ghash); - return err; } static void crypto_gcm_exit_tfm(struct crypto_aead *tfm) { struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); - crypto_free_ahash(ctx->ghash); crypto_free_skcipher(ctx->ctr); } @@ -563,20 +327,16 @@ static void crypto_gcm_free(struct aead_instance *inst) struct gcm_instance_ctx *ctx = aead_instance_ctx(inst); crypto_drop_skcipher(&ctx->ctr); - crypto_drop_ahash(&ctx->ghash); kfree(inst); } static int crypto_gcm_create_common(struct crypto_template *tmpl, - struct rtattr **tb, - const char *ctr_name, - const char *ghash_name) + struct rtattr **tb, const char *ctr_name) { struct skcipher_alg_common *ctr; u32 mask; struct aead_instance *inst; struct gcm_instance_ctx *ctx; - struct hash_alg_common *ghash; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); @@ -588,17 +348,6 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, return -ENOMEM; ctx = aead_instance_ctx(inst); - err = crypto_grab_ahash(&ctx->ghash, aead_crypto_instance(inst), - ghash_name, 0, mask); - if (err) - goto err_free_inst; - ghash = crypto_spawn_ahash_alg(&ctx->ghash); - - err = -EINVAL; - if (strcmp(ghash->base.cra_name, "ghash") != 0 || - ghash->digestsize != 16) - goto err_free_inst; - err = crypto_grab_skcipher(&ctx->ctr, aead_crypto_instance(inst), ctr_name, 0, mask); if (err) @@ -617,13 +366,11 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, goto err_free_inst; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "gcm_base(%s,%s)", ctr->base.cra_driver_name, - ghash->base.cra_driver_name) >= - CRYPTO_MAX_ALG_NAME) + "gcm_base(%s,ghash-lib)", + ctr->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_free_inst; - inst->alg.base.cra_priority = (ghash->base.cra_priority + - ctr->base.cra_priority) / 2; + inst->alg.base.cra_priority = ctr->base.cra_priority; inst->alg.base.cra_blocksize = 1; inst->alg.base.cra_alignmask = ctr->base.cra_alignmask; inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx); @@ -660,7 +407,7 @@ static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) CRYPTO_MAX_ALG_NAME) return -ENAMETOOLONG; - return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash"); + return crypto_gcm_create_common(tmpl, tb, ctr_name); } static int crypto_gcm_base_create(struct crypto_template *tmpl, @@ -677,7 +424,16 @@ static int crypto_gcm_base_create(struct crypto_template *tmpl, if (IS_ERR(ghash_name)) return PTR_ERR(ghash_name); - return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name); + /* + * Originally this parameter allowed requesting a specific + * implementation of GHASH. This is no longer supported. Now the best + * implementation of GHASH is just always used. + */ + if (strcmp(ghash_name, "ghash") != 0 && + strcmp(ghash_name, "ghash-lib") != 0) + return -EINVAL; + + return crypto_gcm_create_common(tmpl, tb, ctr_name); } static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, @@ -1096,25 +852,12 @@ static struct crypto_template crypto_gcm_tmpls[] = { static int __init crypto_gcm_module_init(void) { - int err; - - gcm_zeroes = kzalloc_obj(*gcm_zeroes); - if (!gcm_zeroes) - return -ENOMEM; - - sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf)); - - err = crypto_register_templates(crypto_gcm_tmpls, - ARRAY_SIZE(crypto_gcm_tmpls)); - if (err) - kfree(gcm_zeroes); - - return err; + return crypto_register_templates(crypto_gcm_tmpls, + ARRAY_SIZE(crypto_gcm_tmpls)); } static void __exit crypto_gcm_module_exit(void) { - kfree(gcm_zeroes); crypto_unregister_templates(crypto_gcm_tmpls, ARRAY_SIZE(crypto_gcm_tmpls)); } diff --git a/crypto/testmgr.c b/crypto/testmgr.c index fec950f1628b..0b0ad358e091 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4965,7 +4965,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, { #endif /* CONFIG_CRYPTO_DH_RFC7919_GROUPS */ .alg = "gcm(aes)", - .generic_driver = "gcm_base(ctr(aes-lib),ghash-generic)", + .generic_driver = "gcm_base(ctr(aes-lib),ghash-lib)", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -4973,14 +4973,14 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "gcm(aria)", - .generic_driver = "gcm_base(ctr(aria-generic),ghash-generic)", + .generic_driver = "gcm_base(ctr(aria-generic),ghash-lib)", .test = alg_test_aead, .suite = { .aead = __VECS(aria_gcm_tv_template) } }, { .alg = "gcm(sm4)", - .generic_driver = "gcm_base(ctr(sm4-generic),ghash-generic)", + .generic_driver = "gcm_base(ctr(sm4-generic),ghash-lib)", .test = alg_test_aead, .suite = { .aead = __VECS(sm4_gcm_tv_template) @@ -5314,7 +5314,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4106(gcm(aes))", - .generic_driver = "rfc4106(gcm_base(ctr(aes-lib),ghash-generic))", + .generic_driver = "rfc4106(gcm_base(ctr(aes-lib),ghash-lib))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -5338,7 +5338,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4543(gcm(aes))", - .generic_driver = "rfc4543(gcm_base(ctr(aes-lib),ghash-generic))", + .generic_driver = "rfc4543(gcm_base(ctr(aes-lib),ghash-lib))", .test = alg_test_aead, .suite = { .aead = { diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index 2e2d97d17e6c..a0713aa21250 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -1008,7 +1008,7 @@ static int starfive_aes_ccm_init_tfm(struct crypto_aead *tfm) static int starfive_aes_gcm_init_tfm(struct crypto_aead *tfm) { - return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-lib),ghash-generic)"); + return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-lib),ghash-lib)"); } static struct skcipher_engine_alg skcipher_algs[] = { From 662a05a245078e7d03e75895403c851967dc8384 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:17 -0700 Subject: [PATCH 34/66] crypto: ghash - Remove ghash from crypto_shash API Now that there are no users of the "ghash" crypto_shash algorithm, remove it. GHASH remains supported via the library API. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-17-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 7 -- crypto/Makefile | 1 - crypto/ghash-generic.c | 162 ----------------------------------------- crypto/tcrypt.c | 9 --- crypto/testmgr.c | 6 -- crypto/testmgr.h | 109 --------------------------- 6 files changed, 294 deletions(-) delete mode 100644 crypto/ghash-generic.c diff --git a/crypto/Kconfig b/crypto/Kconfig index 13ccf5ac2f1a..efb482ea192d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -888,13 +888,6 @@ config CRYPTO_CMAC CMAC (Cipher-based Message Authentication Code) authentication mode (NIST SP800-38B and IETF RFC4493) -config CRYPTO_GHASH - tristate "GHASH" - select CRYPTO_HASH - select CRYPTO_LIB_GF128MUL - help - GCM GHASH function (NIST SP800-38D) - config CRYPTO_HMAC tristate "HMAC (Keyed-Hash MAC)" select CRYPTO_HASH diff --git a/crypto/Makefile b/crypto/Makefile index 04e269117589..17f4fca9b9e5 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -171,7 +171,6 @@ UBSAN_SANITIZE_jitterentropy.o = n jitterentropy_rng-y := jitterentropy.o jitterentropy-kcapi.o obj-$(CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE) += jitterentropy-testing.o obj-$(CONFIG_CRYPTO_BENCHMARK) += tcrypt.o -obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c deleted file mode 100644 index e5803c249c12..000000000000 --- a/crypto/ghash-generic.c +++ /dev/null @@ -1,162 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * GHASH: hash function for GCM (Galois/Counter Mode). - * - * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen - * Copyright (c) 2009 Intel Corp. - * Author: Huang Ying - */ - -/* - * GHASH is a keyed hash function used in GCM authentication tag generation. - * - * The original GCM paper [1] presents GHASH as a function GHASH(H, A, C) which - * takes a 16-byte hash key H, additional authenticated data A, and a ciphertext - * C. It formats A and C into a single byte string X, interprets X as a - * polynomial over GF(2^128), and evaluates this polynomial at the point H. - * - * However, the NIST standard for GCM [2] presents GHASH as GHASH(H, X) where X - * is the already-formatted byte string containing both A and C. - * - * "ghash" in the Linux crypto API uses the 'X' (pre-formatted) convention, - * since the API supports only a single data stream per hash. Thus, the - * formatting of 'A' and 'C' is done in the "gcm" template, not in "ghash". - * - * The reason "ghash" is separate from "gcm" is to allow "gcm" to use an - * accelerated "ghash" when a standalone accelerated "gcm(aes)" is unavailable. - * It is generally inappropriate to use "ghash" for other purposes, since it is - * an "ε-almost-XOR-universal hash function", not a cryptographic hash function. - * It can only be used securely in crypto modes specially designed to use it. - * - * [1] The Galois/Counter Mode of Operation (GCM) - * (http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.694.695&rep=rep1&type=pdf) - * [2] Recommendation for Block Cipher Modes of Operation: Galois/Counter Mode (GCM) and GMAC - * (https://csrc.nist.gov/publications/detail/sp/800-38d/final) - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -static int ghash_init(struct shash_desc *desc) -{ - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memset(dctx, 0, sizeof(*dctx)); - - return 0; -} - -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen) -{ - struct ghash_ctx *ctx = crypto_shash_ctx(tfm); - be128 k; - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - if (ctx->gf128) - gf128mul_free_4k(ctx->gf128); - - BUILD_BUG_ON(sizeof(k) != GHASH_BLOCK_SIZE); - memcpy(&k, key, GHASH_BLOCK_SIZE); /* avoid violating alignment rules */ - ctx->gf128 = gf128mul_init_4k_lle(&k); - memzero_explicit(&k, GHASH_BLOCK_SIZE); - - if (!ctx->gf128) - return -ENOMEM; - - return 0; -} - -static int ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - u8 *dst = dctx->buffer; - - do { - crypto_xor(dst, src, GHASH_BLOCK_SIZE); - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - src += GHASH_BLOCK_SIZE; - srclen -= GHASH_BLOCK_SIZE; - } while (srclen >= GHASH_BLOCK_SIZE); - - return srclen; -} - -static void ghash_flush(struct shash_desc *desc, const u8 *src, - unsigned int len) -{ - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - u8 *dst = dctx->buffer; - - if (len) { - crypto_xor(dst, src, len); - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - } -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - u8 *buf = dctx->buffer; - - ghash_flush(desc, src, len); - memcpy(dst, buf, GHASH_BLOCK_SIZE); - - return 0; -} - -static void ghash_exit_tfm(struct crypto_tfm *tfm) -{ - struct ghash_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->gf128) - gf128mul_free_4k(ctx->gf128); -} - -static struct shash_alg ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .descsize = sizeof(struct ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "ghash-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ghash_ctx), - .cra_module = THIS_MODULE, - .cra_exit = ghash_exit_tfm, - }, -}; - -static int __init ghash_mod_init(void) -{ - return crypto_register_shash(&ghash_alg); -} - -static void __exit ghash_mod_exit(void) -{ - crypto_unregister_shash(&ghash_alg); -} - -module_init(ghash_mod_init); -module_exit(ghash_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("GHASH hash function"); -MODULE_ALIAS_CRYPTO("ghash"); -MODULE_ALIAS_CRYPTO("ghash-generic"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index aded37546137..1773f5f71351 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1650,10 +1650,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret = min(ret, tcrypt_test("rfc4309(ccm(aes))")); break; - case 46: - ret = min(ret, tcrypt_test("ghash")); - break; - case 48: ret = min(ret, tcrypt_test("sha3-224")); break; @@ -2251,11 +2247,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_hash_speed("blake2b-512", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; - case 318: - klen = 16; - test_hash_speed("ghash", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; case 319: test_hash_speed("crc32c", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 0b0ad358e091..dd01f86dd6fe 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4985,12 +4985,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .aead = __VECS(sm4_gcm_tv_template) } - }, { - .alg = "ghash", - .test = alg_test_hash, - .suite = { - .hash = __VECS(ghash_tv_template) - } }, { .alg = "hctr2(aes)", .generic_driver = "hctr2_base(xctr(aes-lib),polyval-lib)", diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 1c69c11c0cdb..a3274abacfde 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -6183,115 +6183,6 @@ static const struct hash_testvec wp256_tv_template[] = { }, }; -static const struct hash_testvec ghash_tv_template[] = -{ - { - .key = "\xdf\xa6\xbf\x4d\xed\x81\xdb\x03" - "\xff\xca\xff\x95\xf8\x30\xf0\x61", - .ksize = 16, - .plaintext = "\x95\x2b\x2a\x56\xa5\x60\x04a\xc0" - "\xb3\x2b\x66\x56\xa0\x5b\x40\xb6", - .psize = 16, - .digest = "\xda\x53\xeb\x0a\xd2\xc5\x5b\xb6" - "\x4f\xc4\x80\x2c\xc3\xfe\xda\x60", - }, { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" - "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", - .ksize = 16, - .plaintext = "what do ya want for nothing?", - .psize = 28, - .digest = "\x3e\x1f\x5c\x4d\x65\xf0\xef\xce" - "\x0d\x61\x06\x27\x66\x51\xd5\xe2", - }, { - .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", - .ksize = 16, - .plaintext = "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", - .psize = 50, - .digest = "\xfb\x49\x8a\x36\xe1\x96\xe1\x96" - "\xe1\x96\xe1\x96\xe1\x96\xe1\x96", - }, { - .key = "\xda\x53\xeb\x0a\xd2\xc5\x5b\xb6" - "\x4f\xc4\x80\x2c\xc3\xfe\xda\x60", - .ksize = 16, - .plaintext = "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", - .psize = 50, - .digest = "\x2b\x5c\x0c\x7f\x52\xd1\x60\xc2" - "\x49\xed\x6e\x32\x7a\xa9\xbe\x08", - }, { - .key = "\x95\x2b\x2a\x56\xa5\x60\x04a\xc0" - "\xb3\x2b\x66\x56\xa0\x5b\x40\xb6", - .ksize = 16, - .plaintext = "Test With Truncation", - .psize = 20, - .digest = "\xf8\x94\x87\x2a\x4b\x63\x99\x28" - "\x23\xf7\x93\xf7\x19\xf5\x96\xd9", - }, { - .key = "\x0a\x1b\x2c\x3d\x4e\x5f\x64\x71" - "\x82\x93\xa4\xb5\xc6\xd7\xe8\xf9", - .ksize = 16, - .plaintext = "\x56\x6f\x72\x20\x6c\x61\x75\x74" - "\x65\x72\x20\x4c\x61\x75\x73\x63" - "\x68\x65\x6e\x20\x75\x6e\x64\x20" - "\x53\x74\x61\x75\x6e\x65\x6e\x20" - "\x73\x65\x69\x20\x73\x74\x69\x6c" - "\x6c\x2c\x0a\x64\x75\x20\x6d\x65" - "\x69\x6e\x20\x74\x69\x65\x66\x74" - "\x69\x65\x66\x65\x73\x20\x4c\x65" - "\x62\x65\x6e\x3b\x0a\x64\x61\x73" - "\x73\x20\x64\x75\x20\x77\x65\x69" - "\xc3\x9f\x74\x20\x77\x61\x73\x20" - "\x64\x65\x72\x20\x57\x69\x6e\x64" - "\x20\x64\x69\x72\x20\x77\x69\x6c" - "\x6c\x2c\x0a\x65\x68\x20\x6e\x6f" - "\x63\x68\x20\x64\x69\x65\x20\x42" - "\x69\x72\x6b\x65\x6e\x20\x62\x65" - "\x62\x65\x6e\x2e\x0a\x0a\x55\x6e" - "\x64\x20\x77\x65\x6e\x6e\x20\x64" - "\x69\x72\x20\x65\x69\x6e\x6d\x61" - "\x6c\x20\x64\x61\x73\x20\x53\x63" - "\x68\x77\x65\x69\x67\x65\x6e\x20" - "\x73\x70\x72\x61\x63\x68\x2c\x0a" - "\x6c\x61\x73\x73\x20\x64\x65\x69" - "\x6e\x65\x20\x53\x69\x6e\x6e\x65" - "\x20\x62\x65\x73\x69\x65\x67\x65" - "\x6e\x2e\x0a\x4a\x65\x64\x65\x6d" - "\x20\x48\x61\x75\x63\x68\x65\x20" - "\x67\x69\x62\x74\x20\x64\x69\x63" - "\x68\x2c\x20\x67\x69\x62\x20\x6e" - "\x61\x63\x68\x2c\x0a\x65\x72\x20" - "\x77\x69\x72\x64\x20\x64\x69\x63" - "\x68\x20\x6c\x69\x65\x62\x65\x6e" - "\x20\x75\x6e\x64\x20\x77\x69\x65" - "\x67\x65\x6e\x2e\x0a\x0a\x55\x6e" - "\x64\x20\x64\x61\x6e\x6e\x20\x6d" - "\x65\x69\x6e\x65\x20\x53\x65\x65" - "\x6c\x65\x20\x73\x65\x69\x74\x20" - "\x77\x65\x69\x74\x2c\x20\x73\x65" - "\x69\x20\x77\x65\x69\x74\x2c\x0a" - "\x64\x61\x73\x73\x20\x64\x69\x72" - "\x20\x64\x61\x73\x20\x4c\x65\x62" - "\x65\x6e\x20\x67\x65\x6c\x69\x6e" - "\x67\x65\x2c\x0a\x62\x72\x65\x69" - "\x74\x65\x20\x64\x69\x63\x68\x20" - "\x77\x69\x65\x20\x65\x69\x6e\x20" - "\x46\x65\x69\x65\x72\x6b\x6c\x65" - "\x69\x64\x0a\xc3\xbc\x62\x65\x72" - "\x20\x64\x69\x65\x20\x73\x69\x6e" - "\x6e\x65\x6e\x64\x65\x6e\x20\x44" - "\x69\x6e\x67\x65\x2e\x2e\x2e\x0a", - .psize = 400, - .digest = "\xad\xb1\xc1\xe9\x56\x70\x31\x1d" - "\xbb\x5b\xdf\x5e\x70\x72\x1a\x57", - }, -}; - /* * HMAC-MD5 test vectors from RFC2202 * (These need to be fixed to not use strlen). From 07241d6c922b6e2b53e072691647e34ef395573f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:18 -0700 Subject: [PATCH 35/66] lib/crypto: gf128mul: Remove unused 4k_lle functions Remove the 4k_lle multiplication functions and the associated gf128mul_table_le data table. Their only user was the generic implementation of GHASH, which has now been changed to use a different implementation based on standard integer multiplication. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-18-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/gf128mul.h | 17 ++------- lib/crypto/gf128mul.c | 73 +-------------------------------------- 2 files changed, 4 insertions(+), 86 deletions(-) diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h index b0853f7cada0..6ed2a8351902 100644 --- a/include/crypto/gf128mul.h +++ b/include/crypto/gf128mul.h @@ -215,25 +215,14 @@ static inline void gf128mul_x_ble(le128 *r, const le128 *x) r->b = cpu_to_le64((b << 1) ^ _tt); } -/* 4k table optimization */ - -struct gf128mul_4k { - be128 t[256]; -}; - -struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g); -void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t); void gf128mul_x8_ble(le128 *r, const le128 *x); -static inline void gf128mul_free_4k(struct gf128mul_4k *t) -{ - kfree_sensitive(t); -} - /* 64k table optimization, implemented for bbe */ struct gf128mul_64k { - struct gf128mul_4k *t[16]; + struct { + be128 t[256]; + } *t[16]; }; /* First initialize with the constant factor with which you diff --git a/lib/crypto/gf128mul.c b/lib/crypto/gf128mul.c index e5a727b15f07..7ebf07ce1168 100644 --- a/lib/crypto/gf128mul.c +++ b/lib/crypto/gf128mul.c @@ -127,27 +127,9 @@ (i & 0x02 ? 0x0384 : 0) ^ (i & 0x01 ? 0x01c2 : 0) \ ) -static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le); static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be); -/* - * The following functions multiply a field element by x^8 in - * the polynomial field representation. They use 64-bit word operations - * to gain speed but compensate for machine endianness and hence work - * correctly on both styles of machine. - */ - -static void gf128mul_x8_lle(be128 *x) -{ - u64 a = be64_to_cpu(x->a); - u64 b = be64_to_cpu(x->b); - u64 _tt = gf128mul_table_le[b & 0xff]; - - x->b = cpu_to_be64((b >> 8) | (a << 56)); - x->a = cpu_to_be64((a >> 8) ^ (_tt << 48)); -} - -/* time invariant version of gf128mul_x8_lle */ +/* A table-less implementation of multiplying by x^8 */ static void gf128mul_x8_lle_ti(be128 *x) { u64 a = be64_to_cpu(x->a); @@ -305,58 +287,5 @@ void gf128mul_64k_bbe(be128 *a, const struct gf128mul_64k *t) } EXPORT_SYMBOL(gf128mul_64k_bbe); -/* This version uses 4k bytes of table space. - A 16 byte buffer has to be multiplied by a 16 byte key - value in GF(2^128). If we consider a GF(2^128) value in a - single byte, we can construct a table of the 256 16 byte - values that result from the 256 values of this byte. - This requires 4096 bytes. If we take the highest byte in - the buffer and use this table to get the result, we then - have to multiply by x^120 to get the final value. For the - next highest byte the result has to be multiplied by x^112 - and so on. But we can do this by accumulating the result - in an accumulator starting with the result for the top - byte. We repeatedly multiply the accumulator value by - x^8 and then add in (i.e. xor) the 16 bytes of the next - lower byte in the buffer, stopping when we reach the - lowest byte. This requires a 4096 byte table. -*/ -struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g) -{ - struct gf128mul_4k *t; - int j, k; - - t = kzalloc_obj(*t); - if (!t) - goto out; - - t->t[128] = *g; - for (j = 64; j > 0; j >>= 1) - gf128mul_x_lle(&t->t[j], &t->t[j+j]); - - for (j = 2; j < 256; j += j) - for (k = 1; k < j; ++k) - be128_xor(&t->t[j + k], &t->t[j], &t->t[k]); - -out: - return t; -} -EXPORT_SYMBOL(gf128mul_init_4k_lle); - -void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t) -{ - u8 *ap = (u8 *)a; - be128 r[1]; - int i = 15; - - *r = t->t[ap[15]]; - while (i--) { - gf128mul_x8_lle(r); - be128_xor(r, r, &t->t[ap[i]]); - } - *a = *r; -} -EXPORT_SYMBOL(gf128mul_4k_lle); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Functions for multiplying elements of GF(2^128)"); From a78ae6e364aea8aec3996de274c4f5bc98e1d771 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:19 -0700 Subject: [PATCH 36/66] lib/crypto: gf128hash: Remove unused content from ghash.h Now that the structures in are no longer used, remove them. Since this leaves as just containing constants, include it from to deduplicate these definitions. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-19-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/gf128hash.h | 3 +-- include/crypto/ghash.h | 12 ------------ 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h index 0bc649d01e12..41c557d55965 100644 --- a/include/crypto/gf128hash.h +++ b/include/crypto/gf128hash.h @@ -8,11 +8,10 @@ #ifndef _CRYPTO_GF128HASH_H #define _CRYPTO_GF128HASH_H +#include #include #include -#define GHASH_BLOCK_SIZE 16 -#define GHASH_DIGEST_SIZE 16 #define POLYVAL_BLOCK_SIZE 16 #define POLYVAL_DIGEST_SIZE 16 diff --git a/include/crypto/ghash.h b/include/crypto/ghash.h index 043d938e9a2c..d187e5af9925 100644 --- a/include/crypto/ghash.h +++ b/include/crypto/ghash.h @@ -6,19 +6,7 @@ #ifndef __CRYPTO_GHASH_H__ #define __CRYPTO_GHASH_H__ -#include - #define GHASH_BLOCK_SIZE 16 #define GHASH_DIGEST_SIZE 16 -struct gf128mul_4k; - -struct ghash_ctx { - struct gf128mul_4k *gf128; -}; - -struct ghash_desc_ctx { - u8 buffer[GHASH_BLOCK_SIZE]; -}; - #endif From ea0c746ffa1e6e701d39a564f6286a3f5740826b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:20 -0700 Subject: [PATCH 37/66] lib/crypto: aesgcm: Use GHASH library API Make the AES-GCM library use the GHASH library instead of directly calling gf128mul_lle(). This allows the architecture-optimized GHASH implementations to be used, or the improved generic implementation if no architecture-optimized implementation is usable. Note: this means that no longer needs to include . Remove that inclusion, and include explicitly from arch/x86/crypto/aesni-intel_glue.c which previously was relying on the transitive inclusion. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-20-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/x86/crypto/aesni-intel_glue.c | 1 + include/crypto/gcm.h | 4 +-- lib/crypto/Kconfig | 2 +- lib/crypto/aesgcm.c | 55 +++++++++++++++--------------- 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index e6c38d1d8a92..f522fff9231e 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include diff --git a/include/crypto/gcm.h b/include/crypto/gcm.h index b524e47bd4d0..1d5f39ff1dc4 100644 --- a/include/crypto/gcm.h +++ b/include/crypto/gcm.h @@ -4,7 +4,7 @@ #include #include -#include +#include #define GCM_AES_IV_SIZE 12 #define GCM_RFC4106_IV_SIZE 8 @@ -65,7 +65,7 @@ static inline int crypto_ipsec_check_assoclen(unsigned int assoclen) } struct aesgcm_ctx { - be128 ghash_key; + struct ghash_key ghash_key; struct aes_enckey aes_key; unsigned int authsize; }; diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index a39e7707e9ee..32fafe245f47 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -41,7 +41,7 @@ config CRYPTO_LIB_AES_CBC_MACS config CRYPTO_LIB_AESGCM tristate select CRYPTO_LIB_AES - select CRYPTO_LIB_GF128MUL + select CRYPTO_LIB_GF128HASH select CRYPTO_LIB_UTILS config CRYPTO_LIB_ARC4 diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c index 02f5b5f32c76..8c7e74d2d147 100644 --- a/lib/crypto/aesgcm.c +++ b/lib/crypto/aesgcm.c @@ -5,9 +5,8 @@ * Copyright 2022 Google LLC */ -#include #include -#include +#include #include #include #include @@ -45,7 +44,7 @@ static void aesgcm_encrypt_block(const struct aes_enckey *key, void *dst, int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, unsigned int keysize, unsigned int authsize) { - u8 kin[AES_BLOCK_SIZE] = {}; + u8 h[AES_BLOCK_SIZE] = {}; int ret; ret = crypto_gcm_check_authsize(authsize) ?: @@ -54,24 +53,13 @@ int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, return ret; ctx->authsize = authsize; - aesgcm_encrypt_block(&ctx->aes_key, &ctx->ghash_key, kin); - + aesgcm_encrypt_block(&ctx->aes_key, h, h); + ghash_preparekey(&ctx->ghash_key, h); + memzero_explicit(h, sizeof(h)); return 0; } EXPORT_SYMBOL(aesgcm_expandkey); -static void aesgcm_ghash(be128 *ghash, const be128 *key, const void *src, - int len) -{ - while (len > 0) { - crypto_xor((u8 *)ghash, src, min(len, GHASH_BLOCK_SIZE)); - gf128mul_lle(ghash, key); - - src += GHASH_BLOCK_SIZE; - len -= GHASH_BLOCK_SIZE; - } -} - /** * aesgcm_mac - Generates the authentication tag using AES-GCM algorithm. * @ctx: The data structure that will hold the AES-GCM key schedule @@ -88,20 +76,33 @@ static void aesgcm_ghash(be128 *ghash, const be128 *key, const void *src, static void aesgcm_mac(const struct aesgcm_ctx *ctx, const u8 *src, int src_len, const u8 *assoc, int assoc_len, __be32 *ctr, u8 *authtag) { - be128 tail = { cpu_to_be64(assoc_len * 8), cpu_to_be64(src_len * 8) }; - u8 buf[AES_BLOCK_SIZE]; - be128 ghash = {}; + static const u8 zeroes[GHASH_BLOCK_SIZE]; + __be64 tail[2] = { + cpu_to_be64((u64)assoc_len * 8), + cpu_to_be64((u64)src_len * 8), + }; + struct ghash_ctx ghash; + u8 ghash_out[AES_BLOCK_SIZE]; + u8 enc_ctr[AES_BLOCK_SIZE]; - aesgcm_ghash(&ghash, &ctx->ghash_key, assoc, assoc_len); - aesgcm_ghash(&ghash, &ctx->ghash_key, src, src_len); - aesgcm_ghash(&ghash, &ctx->ghash_key, &tail, sizeof(tail)); + ghash_init(&ghash, &ctx->ghash_key); + + ghash_update(&ghash, assoc, assoc_len); + ghash_update(&ghash, zeroes, -assoc_len & (GHASH_BLOCK_SIZE - 1)); + + ghash_update(&ghash, src, src_len); + ghash_update(&ghash, zeroes, -src_len & (GHASH_BLOCK_SIZE - 1)); + + ghash_update(&ghash, (const u8 *)&tail, sizeof(tail)); + + ghash_final(&ghash, ghash_out); ctr[3] = cpu_to_be32(1); - aesgcm_encrypt_block(&ctx->aes_key, buf, ctr); - crypto_xor_cpy(authtag, buf, (u8 *)&ghash, ctx->authsize); + aesgcm_encrypt_block(&ctx->aes_key, enc_ctr, ctr); + crypto_xor_cpy(authtag, ghash_out, enc_ctr, ctx->authsize); - memzero_explicit(&ghash, sizeof(ghash)); - memzero_explicit(buf, sizeof(buf)); + memzero_explicit(ghash_out, sizeof(ghash_out)); + memzero_explicit(enc_ctr, sizeof(enc_ctr)); } static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, From 2a6b2dda5c324041e9e7db29a4eb8358c7ac8f9c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:24 -0700 Subject: [PATCH 38/66] crypto: sm3 - Fold sm3_init() into its caller Fold sm3_init() into its caller to free up the name for use in a library API mirroring the other hash function APIs. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sm3.h | 13 ------------- include/crypto/sm3_base.h | 12 +++++++++++- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index c8d02c86c298..c09f6bf4c0bf 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -46,19 +46,6 @@ struct sm3_state { * For details see lib/crypto/sm3.c */ -static inline void sm3_init(struct sm3_state *sctx) -{ - sctx->state[0] = SM3_IVA; - sctx->state[1] = SM3_IVB; - sctx->state[2] = SM3_IVC; - sctx->state[3] = SM3_IVD; - sctx->state[4] = SM3_IVE; - sctx->state[5] = SM3_IVF; - sctx->state[6] = SM3_IVG; - sctx->state[7] = SM3_IVH; - sctx->count = 0; -} - void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks); #endif diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h index 7c53570bc05e..9fa995617495 100644 --- a/include/crypto/sm3_base.h +++ b/include/crypto/sm3_base.h @@ -21,7 +21,17 @@ typedef void (sm3_block_fn)(struct sm3_state *sst, u8 const *src, int blocks); static inline int sm3_base_init(struct shash_desc *desc) { - sm3_init(shash_desc_ctx(desc)); + struct sm3_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SM3_IVA; + sctx->state[1] = SM3_IVB; + sctx->state[2] = SM3_IVC; + sctx->state[3] = SM3_IVD; + sctx->state[4] = SM3_IVE; + sctx->state[5] = SM3_IVF; + sctx->state[6] = SM3_IVG; + sctx->state[7] = SM3_IVH; + sctx->count = 0; return 0; } From 77e4ca814c2824d7aa0c4170678bfbc6e3caa556 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:25 -0700 Subject: [PATCH 39/66] crypto: sm3 - Remove sm3_zero_message_hash and SM3_T[1-2] Remove these, since they are unused. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/sm3_generic.c | 8 -------- include/crypto/sm3.h | 5 ----- 2 files changed, 13 deletions(-) diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c index 7529139fcc96..0c606f526347 100644 --- a/crypto/sm3_generic.c +++ b/crypto/sm3_generic.c @@ -14,14 +14,6 @@ #include #include -const u8 sm3_zero_message_hash[SM3_DIGEST_SIZE] = { - 0x1A, 0xB2, 0x1D, 0x83, 0x55, 0xCF, 0xA1, 0x7F, - 0x8e, 0x61, 0x19, 0x48, 0x31, 0xE8, 0x1A, 0x8F, - 0x22, 0xBE, 0xC8, 0xC7, 0x28, 0xFE, 0xFB, 0x74, - 0x7E, 0xD0, 0x35, 0xEB, 0x50, 0x82, 0xAA, 0x2B -}; -EXPORT_SYMBOL_GPL(sm3_zero_message_hash); - static int crypto_sm3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index c09f6bf4c0bf..918d318795ef 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -16,9 +16,6 @@ #define SM3_BLOCK_SIZE 64 #define SM3_STATE_SIZE 40 -#define SM3_T1 0x79CC4519 -#define SM3_T2 0x7A879D8A - #define SM3_IVA 0x7380166f #define SM3_IVB 0x4914b2b9 #define SM3_IVC 0x172442d7 @@ -28,8 +25,6 @@ #define SM3_IVG 0xe38dee4d #define SM3_IVH 0xb0fb0e4e -extern const u8 sm3_zero_message_hash[SM3_DIGEST_SIZE]; - struct sm3_state { u32 state[SM3_DIGEST_SIZE / 4]; u64 count; From 6dc7fce91041ec8d2f5e6fd589ee2962898d9f44 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:26 -0700 Subject: [PATCH 40/66] crypto: sm3 - Rename CRYPTO_SM3_GENERIC to CRYPTO_SM3 The kconfig options for generic crypto API modules have traditionally *not* had a "_GENERIC" suffix. Also, the "_GENERIC" suffix will make even less sense once the architecture-optimized SM3 code is moved into lib/crypto/ and the "sm3" crypto_shash is reimplemented on top of that. Thus, rename CRYPTO_SM3_GENERIC to CRYPTO_SM3. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/loongarch/configs/loongson32_defconfig | 2 +- arch/loongarch/configs/loongson64_defconfig | 2 +- arch/m68k/configs/amiga_defconfig | 2 +- arch/m68k/configs/apollo_defconfig | 2 +- arch/m68k/configs/atari_defconfig | 2 +- arch/m68k/configs/bvme6000_defconfig | 2 +- arch/m68k/configs/hp300_defconfig | 2 +- arch/m68k/configs/mac_defconfig | 2 +- arch/m68k/configs/multi_defconfig | 2 +- arch/m68k/configs/mvme147_defconfig | 2 +- arch/m68k/configs/mvme16x_defconfig | 2 +- arch/m68k/configs/q40_defconfig | 2 +- arch/m68k/configs/sun3_defconfig | 2 +- arch/m68k/configs/sun3x_defconfig | 2 +- arch/s390/configs/debug_defconfig | 2 +- arch/s390/configs/defconfig | 2 +- crypto/Kconfig | 2 +- crypto/Makefile | 2 +- drivers/crypto/Kconfig | 2 +- drivers/crypto/starfive/Kconfig | 2 +- security/integrity/ima/Kconfig | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/arch/loongarch/configs/loongson32_defconfig b/arch/loongarch/configs/loongson32_defconfig index 276b1577e0be..7abbb21f4f8f 100644 --- a/arch/loongarch/configs/loongson32_defconfig +++ b/arch/loongarch/configs/loongson32_defconfig @@ -1080,7 +1080,7 @@ CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m CONFIG_CRYPTO_CHACHA20POLY1305=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_DEFLATE=m CONFIG_CRYPTO_LZO=m diff --git a/arch/loongarch/configs/loongson64_defconfig b/arch/loongarch/configs/loongson64_defconfig index a14db1a95e7e..51ccd18ecdae 100644 --- a/arch/loongarch/configs/loongson64_defconfig +++ b/arch/loongarch/configs/loongson64_defconfig @@ -1113,7 +1113,7 @@ CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m CONFIG_CRYPTO_CHACHA20POLY1305=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_DEFLATE=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 31d16cba9879..03a8c192a7a3 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -581,7 +581,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index c0c419ec9a9e..0aee1939ac7a 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -538,7 +538,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 2b7547ecc4c4..756256770afc 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -558,7 +558,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index 0b63787cff0d..8cfb75bb0add 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -530,7 +530,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 308836b60bba..b2f5c9749e9b 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -540,7 +540,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 97e108c0d24f..c4fddaaa6a86 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -557,7 +557,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 7e9f83af9af4..926f12bc3d1d 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -644,7 +644,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 2fe33271d249..e507012dbbc1 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -530,7 +530,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 4308daaa7f74..6195cedd914b 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -531,7 +531,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index 36eb29ec54ee..9087bd9e3c35 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -547,7 +547,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 524a89fa6953..25115bda7c8a 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -528,7 +528,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index f4fbc65c52d9..15a086634ba5 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -528,7 +528,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index aa862d4fcc68..f8fcc29adbd3 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -797,7 +797,7 @@ CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_CRC32=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 74f943307c46..3c6ccc0de018 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -781,7 +781,7 @@ CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_CRC32=m diff --git a/crypto/Kconfig b/crypto/Kconfig index efb482ea192d..9919fe0db7f6 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -974,7 +974,7 @@ config CRYPTO_SHA3 help SHA-3 secure hash algorithms (FIPS 202, ISO/IEC 10118-3) -config CRYPTO_SM3_GENERIC +config CRYPTO_SM3 tristate "SM3 (ShangMi 3)" select CRYPTO_HASH select CRYPTO_LIB_SM3 diff --git a/crypto/Makefile b/crypto/Makefile index 17f4fca9b9e5..3fcbf0cd522d 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -83,7 +83,7 @@ obj-$(CONFIG_CRYPTO_SHA1) += sha1.o obj-$(CONFIG_CRYPTO_SHA256) += sha256.o obj-$(CONFIG_CRYPTO_SHA512) += sha512.o obj-$(CONFIG_CRYPTO_SHA3) += sha3.o -obj-$(CONFIG_CRYPTO_SM3_GENERIC) += sm3_generic.o +obj-$(CONFIG_CRYPTO_SM3) += sm3_generic.o obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o CFLAGS_wp512.o := $(call cc-option,-fno-schedule-insns) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149 diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 8d3b5d2890f8..9960100e6066 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -843,7 +843,7 @@ config CRYPTO_DEV_CCREE select CRYPTO_CTR select CRYPTO_XTS select CRYPTO_SM4_GENERIC - select CRYPTO_SM3_GENERIC + select CRYPTO_SM3 help Say 'Y' to enable a driver for the REE interface of the Arm TrustZone CryptoCell family of processors. Currently the diff --git a/drivers/crypto/starfive/Kconfig b/drivers/crypto/starfive/Kconfig index 0fe389e9f932..11518ca3eea1 100644 --- a/drivers/crypto/starfive/Kconfig +++ b/drivers/crypto/starfive/Kconfig @@ -10,7 +10,7 @@ config CRYPTO_DEV_JH7110 select CRYPTO_HMAC select CRYPTO_SHA256 select CRYPTO_SHA512 - select CRYPTO_SM3_GENERIC + select CRYPTO_SM3 select CRYPTO_RSA select CRYPTO_AES select CRYPTO_CCM diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig index 976e75f9b9ba..862fbee2b174 100644 --- a/security/integrity/ima/Kconfig +++ b/security/integrity/ima/Kconfig @@ -111,7 +111,7 @@ choice config IMA_DEFAULT_HASH_SM3 bool "SM3" - depends on CRYPTO_SM3_GENERIC=y + depends on CRYPTO_SM3=y endchoice config IMA_DEFAULT_HASH From 324bb3bb75ac21adbbc7e6ea5cdb0a735fb78a56 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:27 -0700 Subject: [PATCH 41/66] lib/crypto: sm3: Add SM3 library API Add a straightforward library API for SM3, mirroring the ones for the other hash algorithms. It uses the existing generic implementation of SM3's compression function in lib/crypto/sm3.c. Hooks are added for architecture-optimized implementations, which later commits will wire up to the existing optimized SM3 code for arm64, riscv, and x86. Note that the rationale for this is *not* that SM3 should be used, or that any kernel subsystem currently seems like a candidate for switching from the sm3 crypto_shash to SM3 library. (SM3, in fact, shouldn't be used. Likewise you shouldn't use MD5, SHA-1, RC4, etc...) Rather, it's just that this will simplify how the kernel's existing SM3 code is integrated and make it much easier to maintain and test. SM3 is one of the only hash algorithms with arch-optimized code that is still integrated in the old way. By converting it to the new lib/crypto/ code organization, we'll only have to keep track of one way of doing things. The library will also get a KUnit test suite (as usual for lib/crypto/), so it will become more easily and comprehensively tested as well. Skip adding functions for HMAC-SM3 for now, though. There's not as much point in adding those right now. Note: similar to the other hash algorithms, the library API uses 'struct sm3_ctx', not 'struct sm3_state'. The existing 'struct sm3_state' and the sm3_block_generic() function which uses it are temporarily kept around until their users are updated by later commits. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-5-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sm3.h | 74 +++++++++++++++++---- lib/crypto/Kconfig | 7 ++ lib/crypto/sm3.c | 155 +++++++++++++++++++++++++++++++++++++------ 3 files changed, 205 insertions(+), 31 deletions(-) diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index 918d318795ef..702c5326b4be 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Common values for SM3 algorithm + * SM3 hash algorithm * * Copyright (C) 2017 ARM Limited or its affiliates. * Copyright (C) 2017 Gilad Ben-Yossef @@ -31,16 +31,66 @@ struct sm3_state { u8 buffer[SM3_BLOCK_SIZE]; }; -/* - * Stand-alone implementation of the SM3 algorithm. It is designed to - * have as little dependencies as possible so it can be used in the - * kexec_file purgatory. In other cases you should generally use the - * hash APIs from include/crypto/hash.h. Especially when hashing large - * amounts of data as those APIs may be hw-accelerated. - * - * For details see lib/crypto/sm3.c - */ - void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks); -#endif +/* State for the SM3 compression function */ +struct sm3_block_state { + u32 h[SM3_DIGEST_SIZE / 4]; +}; + +/** + * struct sm3_ctx - Context for hashing a message with SM3 + * @state: the compression function state + * @bytecount: number of bytes processed so far + * @buf: partial block buffer; bytecount % SM3_BLOCK_SIZE bytes are valid + */ +struct sm3_ctx { + struct sm3_block_state state; + u64 bytecount; + u8 buf[SM3_BLOCK_SIZE] __aligned(__alignof__(__be64)); +}; + +/** + * sm3_init() - Initialize an SM3 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sm3() instead. + * + * Context: Any context. + */ +void sm3_init(struct sm3_ctx *ctx); + +/** + * sm3_update() - Update an SM3 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +void sm3_update(struct sm3_ctx *ctx, const u8 *data, size_t len); + +/** + * sm3_final() - Finish computing an SM3 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SM3 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sm3_final(struct sm3_ctx *ctx, u8 out[at_least SM3_DIGEST_SIZE]); + +/** + * sm3() - Compute SM3 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SM3 message digest + * + * Context: Any context. + */ +void sm3(const u8 *data, size_t len, u8 out[at_least SM3_DIGEST_SIZE]); + +#endif /* _CRYPTO_SM3_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 32fafe245f47..64c9a0bc4099 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -272,6 +272,13 @@ config CRYPTO_LIB_SHA3_ARCH config CRYPTO_LIB_SM3 tristate + help + The SM3 library functions. Select this if your module uses any of the + functions from . + +config CRYPTO_LIB_SM3_ARCH + bool + depends on CRYPTO_LIB_SM3 && !UML source "lib/crypto/tests/Kconfig" diff --git a/lib/crypto/sm3.c b/lib/crypto/sm3.c index c6b9ad8a3ac6..20500cf4b8c0 100644 --- a/lib/crypto/sm3.c +++ b/lib/crypto/sm3.c @@ -15,6 +15,13 @@ #include #include +static const struct sm3_block_state sm3_iv = { + .h = { + SM3_IVA, SM3_IVB, SM3_IVC, SM3_IVD, + SM3_IVE, SM3_IVF, SM3_IVG, SM3_IVH, + }, +}; + static const u32 ____cacheline_aligned K[64] = { 0x79cc4519, 0xf3988a32, 0xe7311465, 0xce6228cb, 0x9cc45197, 0x3988a32f, 0x7311465e, 0xe6228cbc, @@ -72,18 +79,19 @@ static const u32 ____cacheline_aligned K[64] = { ^ rol32(W[(i-13) & 0x0f], 7) \ ^ W[(i-6) & 0x0f]) -static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) +static void sm3_transform(struct sm3_block_state *state, + const u8 data[SM3_BLOCK_SIZE], u32 W[16]) { u32 a, b, c, d, e, f, g, h, ss1, ss2; - a = sctx->state[0]; - b = sctx->state[1]; - c = sctx->state[2]; - d = sctx->state[3]; - e = sctx->state[4]; - f = sctx->state[5]; - g = sctx->state[6]; - h = sctx->state[7]; + a = state->h[0]; + b = state->h[1]; + c = state->h[2]; + d = state->h[3]; + e = state->h[4]; + f = state->h[5]; + g = state->h[6]; + h = state->h[7]; R1(a, b, c, d, e, f, g, h, K[0], I(0), I(4)); R1(d, a, b, c, h, e, f, g, K[1], I(1), I(5)); @@ -153,14 +161,14 @@ static void sm3_transform(struct sm3_state *sctx, u8 const *data, u32 W[16]) R2(c, d, a, b, g, h, e, f, K[62], W1(62), W2(66)); R2(b, c, d, a, f, g, h, e, K[63], W1(63), W2(67)); - sctx->state[0] ^= a; - sctx->state[1] ^= b; - sctx->state[2] ^= c; - sctx->state[3] ^= d; - sctx->state[4] ^= e; - sctx->state[5] ^= f; - sctx->state[6] ^= g; - sctx->state[7] ^= h; + state->h[0] ^= a; + state->h[1] ^= b; + state->h[2] ^= c; + state->h[3] ^= d; + state->h[4] ^= e; + state->h[5] ^= f; + state->h[6] ^= g; + state->h[7] ^= h; } #undef R #undef R1 @@ -174,7 +182,7 @@ void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks) u32 W[16]; do { - sm3_transform(sctx, data, W); + sm3_transform((struct sm3_block_state *)sctx->state, data, W); data += SM3_BLOCK_SIZE; } while (--blocks); @@ -182,5 +190,114 @@ void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks) } EXPORT_SYMBOL_GPL(sm3_block_generic); -MODULE_DESCRIPTION("Generic SM3 library"); +static void __maybe_unused sm3_blocks_generic(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + u32 W[16]; + + do { + sm3_transform(state, data, W); + data += SM3_BLOCK_SIZE; + } while (--nblocks); + + memzero_explicit(W, sizeof(W)); +} + +#ifdef CONFIG_CRYPTO_LIB_SM3_ARCH +#include "sm3.h" /* $(SRCARCH)/sm3.h */ +#else +#define sm3_blocks sm3_blocks_generic +#endif + +void sm3_init(struct sm3_ctx *ctx) +{ + ctx->state = sm3_iv; + ctx->bytecount = 0; +} +EXPORT_SYMBOL_GPL(sm3_init); + +void sm3_update(struct sm3_ctx *ctx, const u8 *data, size_t len) +{ + size_t partial = ctx->bytecount % SM3_BLOCK_SIZE; + + ctx->bytecount += len; + + if (partial + len >= SM3_BLOCK_SIZE) { + size_t nblocks; + + if (partial) { + size_t l = SM3_BLOCK_SIZE - partial; + + memcpy(&ctx->buf[partial], data, l); + data += l; + len -= l; + + sm3_blocks(&ctx->state, ctx->buf, 1); + } + + nblocks = len / SM3_BLOCK_SIZE; + len %= SM3_BLOCK_SIZE; + + if (nblocks) { + sm3_blocks(&ctx->state, data, nblocks); + data += nblocks * SM3_BLOCK_SIZE; + } + partial = 0; + } + if (len) + memcpy(&ctx->buf[partial], data, len); +} +EXPORT_SYMBOL_GPL(sm3_update); + +static void __sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE]) +{ + u64 bitcount = ctx->bytecount << 3; + size_t partial = ctx->bytecount % SM3_BLOCK_SIZE; + + ctx->buf[partial++] = 0x80; + if (partial > SM3_BLOCK_SIZE - 8) { + memset(&ctx->buf[partial], 0, SM3_BLOCK_SIZE - partial); + sm3_blocks(&ctx->state, ctx->buf, 1); + partial = 0; + } + memset(&ctx->buf[partial], 0, SM3_BLOCK_SIZE - 8 - partial); + *(__be64 *)&ctx->buf[SM3_BLOCK_SIZE - 8] = cpu_to_be64(bitcount); + sm3_blocks(&ctx->state, ctx->buf, 1); + + for (size_t i = 0; i < SM3_DIGEST_SIZE; i += 4) + put_unaligned_be32(ctx->state.h[i / 4], out + i); +} + +void sm3_final(struct sm3_ctx *ctx, u8 out[SM3_DIGEST_SIZE]) +{ + __sm3_final(ctx, out); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(sm3_final); + +void sm3(const u8 *data, size_t len, u8 out[SM3_DIGEST_SIZE]) +{ + struct sm3_ctx ctx; + + sm3_init(&ctx); + sm3_update(&ctx, data, len); + sm3_final(&ctx, out); +} +EXPORT_SYMBOL_GPL(sm3); + +#ifdef sm3_mod_init_arch +static int __init sm3_mod_init(void) +{ + sm3_mod_init_arch(); + return 0; +} +subsys_initcall(sm3_mod_init); + +static void __exit sm3_mod_exit(void) +{ +} +module_exit(sm3_mod_exit); +#endif + +MODULE_DESCRIPTION("SM3 library functions"); MODULE_LICENSE("GPL v2"); From d6781b8ba33ae9f6ab2e88c1158e989a24847c4b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:28 -0700 Subject: [PATCH 42/66] lib/crypto: tests: Add KUnit tests for SM3 Add a KUnit test suite for the SM3 library. It closely mirrors the test suites for the other cryptographic hash functions. The actual test and benchmark logic is already in hash-test-template.h; this just wires it up for SM3 in the usual way. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-6-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/.kunitconfig | 1 + lib/crypto/tests/Kconfig | 9 ++ lib/crypto/tests/Makefile | 1 + lib/crypto/tests/sm3-testvecs.h | 231 ++++++++++++++++++++++++++++ lib/crypto/tests/sm3_kunit.c | 31 ++++ scripts/crypto/gen-hash-testvecs.py | 3 + 6 files changed, 276 insertions(+) create mode 100644 lib/crypto/tests/sm3-testvecs.h create mode 100644 lib/crypto/tests/sm3_kunit.c diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig index 391836511c8b..f8a3c6e6367c 100644 --- a/lib/crypto/.kunitconfig +++ b/lib/crypto/.kunitconfig @@ -16,3 +16,4 @@ CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST=y CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_SM3_KUNIT_TEST=y diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 5b60d5c3644b..7a5ad109aefc 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -124,6 +124,14 @@ config CRYPTO_LIB_SHA3_KUNIT_TEST including SHA3-224, SHA3-256, SHA3-384, SHA3-512, SHAKE128 and SHAKE256. +config CRYPTO_LIB_SM3_KUNIT_TEST + tristate "KUnit tests for SM3" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_SM3 + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the SM3 cryptographic hash function. + config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT tristate "Enable all crypto library code for KUnit tests" depends on KUNIT @@ -139,6 +147,7 @@ config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT select CRYPTO_LIB_SHA256 select CRYPTO_LIB_SHA512 select CRYPTO_LIB_SHA3 + select CRYPTO_LIB_SM3 help Enable all the crypto library code that has KUnit tests. diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile index 751ae507fdd0..ad1cbb88132f 100644 --- a/lib/crypto/tests/Makefile +++ b/lib/crypto/tests/Makefile @@ -14,3 +14,4 @@ obj-$(CONFIG_CRYPTO_LIB_SHA1_KUNIT_TEST) += sha1_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA256_KUNIT_TEST) += sha224_kunit.o sha256_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA512_KUNIT_TEST) += sha384_kunit.o sha512_kunit.o obj-$(CONFIG_CRYPTO_LIB_SHA3_KUNIT_TEST) += sha3_kunit.o +obj-$(CONFIG_CRYPTO_LIB_SM3_KUNIT_TEST) += sm3_kunit.o diff --git a/lib/crypto/tests/sm3-testvecs.h b/lib/crypto/tests/sm3-testvecs.h new file mode 100644 index 000000000000..5e788c29f487 --- /dev/null +++ b/lib/crypto/tests/sm3-testvecs.h @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* This file was generated by: ./scripts/crypto/gen-hash-testvecs.py sm3 */ + +static const struct { + size_t data_len; + u8 digest[SM3_DIGEST_SIZE]; +} hash_testvecs[] = { + { + .data_len = 0, + .digest = { + 0x1a, 0xb2, 0x1d, 0x83, 0x55, 0xcf, 0xa1, 0x7f, + 0x8e, 0x61, 0x19, 0x48, 0x31, 0xe8, 0x1a, 0x8f, + 0x22, 0xbe, 0xc8, 0xc7, 0x28, 0xfe, 0xfb, 0x74, + 0x7e, 0xd0, 0x35, 0xeb, 0x50, 0x82, 0xaa, 0x2b, + }, + }, + { + .data_len = 1, + .digest = { + 0xb6, 0x22, 0x2c, 0x39, 0xdc, 0x14, 0x9a, 0xee, + 0x01, 0x9a, 0xcb, 0x0d, 0xe6, 0xc6, 0x75, 0x6e, + 0x8f, 0x18, 0x7b, 0x0e, 0xe8, 0x98, 0x61, 0x71, + 0x2b, 0xd8, 0x38, 0xa9, 0xee, 0x2c, 0x1e, 0x93, + }, + }, + { + .data_len = 2, + .digest = { + 0x62, 0x0c, 0x66, 0x77, 0x67, 0x28, 0x74, 0x8a, + 0xe3, 0x64, 0xea, 0x44, 0x6a, 0x3f, 0x34, 0x61, + 0x55, 0xc5, 0xaa, 0xb2, 0x6c, 0x67, 0x97, 0x68, + 0x68, 0xae, 0x4d, 0x64, 0xa8, 0xb6, 0x72, 0x3e, + }, + }, + { + .data_len = 3, + .digest = { + 0x71, 0xd4, 0x63, 0xb1, 0xfa, 0x27, 0xc7, 0xae, + 0x65, 0xed, 0x5c, 0x93, 0x70, 0xe0, 0x09, 0x34, + 0x2f, 0x42, 0xe6, 0x71, 0x16, 0x8e, 0x90, 0x90, + 0x9a, 0xdd, 0xa6, 0x44, 0x66, 0x71, 0x18, 0xf9, + }, + }, + { + .data_len = 16, + .digest = { + 0x79, 0x0b, 0x68, 0xb5, 0x41, 0xc1, 0x97, 0xa0, + 0x50, 0xe6, 0x93, 0x70, 0xf6, 0x98, 0x49, 0xea, + 0x92, 0xc9, 0xd0, 0xb1, 0x46, 0xbd, 0x4a, 0x0c, + 0x8e, 0xe8, 0xf3, 0xe4, 0x8f, 0x90, 0x33, 0x3c, + }, + }, + { + .data_len = 32, + .digest = { + 0x32, 0x9f, 0xa3, 0x18, 0x18, 0x45, 0xe0, 0x28, + 0xd3, 0xa4, 0x41, 0x3a, 0x25, 0x62, 0x9c, 0x95, + 0xab, 0xfe, 0x02, 0xe0, 0x37, 0x7d, 0x3c, 0xc4, + 0xce, 0x69, 0x57, 0x5a, 0x07, 0x0e, 0xb9, 0xf5, + }, + }, + { + .data_len = 48, + .digest = { + 0x0c, 0xcf, 0x7c, 0x48, 0x44, 0xa0, 0xb0, 0x8d, + 0xdf, 0xbe, 0x22, 0x14, 0x7e, 0xd4, 0xc3, 0x8d, + 0x6a, 0x23, 0xfc, 0x44, 0x0e, 0x0f, 0xde, 0xa5, + 0x7c, 0x8b, 0xc4, 0x8b, 0xab, 0x8c, 0x87, 0x41, + }, + }, + { + .data_len = 49, + .digest = { + 0xb3, 0x76, 0x8b, 0x19, 0xf9, 0x10, 0xa9, 0x56, + 0x4f, 0xce, 0x27, 0xaa, 0x65, 0x96, 0xe5, 0xdb, + 0x90, 0x9b, 0x92, 0xcd, 0x32, 0x0d, 0x16, 0xac, + 0xf8, 0xd0, 0x66, 0x62, 0x10, 0xf0, 0x44, 0xdf, + }, + }, + { + .data_len = 63, + .digest = { + 0x07, 0xc9, 0x45, 0x65, 0x9f, 0x68, 0x75, 0xc3, + 0x74, 0xb2, 0x3b, 0x0c, 0x97, 0x05, 0xd3, 0x13, + 0xc0, 0xb6, 0x21, 0xed, 0xf6, 0x10, 0x7a, 0xed, + 0xec, 0xd8, 0x10, 0x29, 0xbf, 0x7a, 0x78, 0x37, + }, + }, + { + .data_len = 64, + .digest = { + 0x3e, 0x69, 0x18, 0x45, 0xd8, 0x25, 0x6f, 0x44, + 0xc0, 0x02, 0xe5, 0xcf, 0xcd, 0x94, 0x42, 0xa9, + 0xd5, 0x12, 0x62, 0x10, 0x15, 0xa0, 0xf9, 0x16, + 0x19, 0x2d, 0x8d, 0x63, 0x31, 0xf2, 0x2f, 0x36, + }, + }, + { + .data_len = 65, + .digest = { + 0x6b, 0x3e, 0xc0, 0x20, 0xb7, 0x74, 0x30, 0xa0, + 0xc6, 0x5c, 0xee, 0xbe, 0xdc, 0xe6, 0xe5, 0x4f, + 0x3c, 0x61, 0x8d, 0x91, 0xac, 0x31, 0x4b, 0x2a, + 0xdf, 0x1c, 0xef, 0x24, 0xdc, 0x0a, 0x10, 0xe8, + }, + }, + { + .data_len = 127, + .digest = { + 0xab, 0xd6, 0xa1, 0xbf, 0x39, 0x43, 0x75, 0xda, + 0xbf, 0xc7, 0x22, 0xcc, 0x4e, 0xfc, 0xe4, 0x42, + 0x6d, 0x1b, 0x87, 0x25, 0x64, 0x7f, 0x88, 0xf7, + 0xc3, 0x0a, 0x0a, 0x4c, 0xd6, 0xa7, 0x68, 0xae, + }, + }, + { + .data_len = 128, + .digest = { + 0x1b, 0x70, 0xd4, 0x5f, 0x6c, 0xe4, 0x2d, 0x58, + 0x2d, 0x0f, 0x9a, 0x12, 0x34, 0xbb, 0x5e, 0x38, + 0xd8, 0x1f, 0x6a, 0x46, 0x8a, 0xef, 0xdb, 0x68, + 0x18, 0x62, 0xbb, 0x85, 0xfc, 0xc4, 0x6e, 0x2e, + }, + }, + { + .data_len = 129, + .digest = { + 0x33, 0x62, 0xba, 0xa7, 0x4a, 0xbc, 0xd7, 0x7b, + 0xd4, 0x67, 0x6d, 0x3e, 0xea, 0xe8, 0xb0, 0x64, + 0x0d, 0xf3, 0xae, 0x1d, 0x52, 0x24, 0x11, 0x9f, + 0xda, 0xa9, 0x7f, 0xd5, 0x22, 0x1a, 0xde, 0x8a, + }, + }, + { + .data_len = 256, + .digest = { + 0x70, 0xa8, 0xa6, 0x2b, 0xfb, 0x1f, 0x3b, 0x5a, + 0xcc, 0x71, 0x76, 0x9e, 0x25, 0x4c, 0xfa, 0x8f, + 0x39, 0x4a, 0x21, 0x8a, 0x9d, 0x74, 0x8d, 0x2c, + 0x31, 0xa5, 0xb5, 0xff, 0x30, 0xc1, 0x14, 0xc4, + }, + }, + { + .data_len = 511, + .digest = { + 0x39, 0xd0, 0x8c, 0x5f, 0xfc, 0x36, 0xc2, 0x7c, + 0xdb, 0x8b, 0x2e, 0xdc, 0x9d, 0x1b, 0xd1, 0xba, + 0x9b, 0x52, 0x6b, 0x35, 0x46, 0x46, 0x75, 0x73, + 0xe5, 0x62, 0x96, 0x6e, 0xf3, 0xba, 0xd9, 0x19, + }, + }, + { + .data_len = 513, + .digest = { + 0x76, 0xa0, 0x3d, 0xa2, 0x5f, 0xd4, 0xa6, 0xbe, + 0x6b, 0xdb, 0xed, 0x14, 0x9e, 0xa8, 0x15, 0x77, + 0xa9, 0x38, 0x30, 0x6b, 0x68, 0xfa, 0xb6, 0xe2, + 0x93, 0x19, 0x24, 0x72, 0x67, 0x20, 0x72, 0xc3, + }, + }, + { + .data_len = 1000, + .digest = { + 0x16, 0xbc, 0x33, 0x77, 0x0b, 0xcf, 0x93, 0x5e, + 0xec, 0x7d, 0x8d, 0x3c, 0xae, 0xd9, 0x75, 0xdf, + 0x46, 0x24, 0x17, 0x7e, 0x03, 0x88, 0xf2, 0x75, + 0xa9, 0x18, 0xa6, 0x1c, 0x7a, 0x74, 0x0d, 0xf3, + }, + }, + { + .data_len = 3333, + .digest = { + 0xdb, 0x54, 0x89, 0xe7, 0x1c, 0x50, 0xf2, 0xbf, + 0xde, 0x3a, 0xbf, 0x5b, 0xee, 0x5a, 0x46, 0x62, + 0x20, 0xb1, 0x80, 0x48, 0xac, 0x56, 0x33, 0xb3, + 0xbb, 0x3f, 0xfa, 0x02, 0xc6, 0x43, 0xb5, 0x8c, + }, + }, + { + .data_len = 4096, + .digest = { + 0xdf, 0x0d, 0xed, 0x3b, 0x8f, 0xea, 0x81, 0xfd, + 0xd6, 0x34, 0xae, 0x74, 0x24, 0x3a, 0x15, 0x38, + 0xe7, 0xcf, 0x45, 0x7e, 0x8f, 0xf5, 0x50, 0x6c, + 0xaa, 0x27, 0x23, 0x92, 0x6d, 0xab, 0x3b, 0xde, + }, + }, + { + .data_len = 4128, + .digest = { + 0x6a, 0xbd, 0x56, 0x5a, 0xf1, 0xc6, 0x40, 0x4d, + 0xf3, 0x50, 0x77, 0x87, 0x86, 0x63, 0x1b, 0x4d, + 0x21, 0x99, 0x96, 0xad, 0x24, 0x62, 0xce, 0xc0, + 0x3e, 0xb7, 0x35, 0x52, 0x56, 0x0e, 0x55, 0x85, + }, + }, + { + .data_len = 4160, + .digest = { + 0x5b, 0xc1, 0x1f, 0x25, 0x43, 0xa3, 0x1c, 0xa0, + 0x8c, 0xfc, 0x41, 0xf1, 0xcc, 0xb3, 0x95, 0x95, + 0xe0, 0xb9, 0xd3, 0x29, 0xf4, 0x08, 0x31, 0x47, + 0x6d, 0x09, 0xa8, 0x2e, 0xa5, 0xf4, 0xf1, 0x8d, + }, + }, + { + .data_len = 4224, + .digest = { + 0xec, 0x56, 0x1e, 0xa6, 0x1f, 0xb2, 0x87, 0xb2, + 0x7e, 0x15, 0xd6, 0x30, 0x08, 0x74, 0xb0, 0x48, + 0x90, 0x2a, 0xbe, 0x2f, 0x80, 0x3a, 0x88, 0xcc, + 0xd7, 0xc5, 0x87, 0x8c, 0x04, 0xef, 0x78, 0x71, + }, + }, + { + .data_len = 16384, + .digest = { + 0xe7, 0xb8, 0x84, 0x20, 0xff, 0xd5, 0x53, 0xe6, + 0x14, 0x31, 0x12, 0x75, 0xb7, 0x9a, 0x4f, 0x63, + 0x63, 0x00, 0xfe, 0x2c, 0x54, 0xee, 0x06, 0xfc, + 0x12, 0x16, 0xe5, 0xdc, 0xa4, 0x40, 0x62, 0x12, + }, + }, +}; + +static const u8 hash_testvec_consolidated[SM3_DIGEST_SIZE] = { + 0xe6, 0x58, 0xd4, 0x8e, 0x74, 0x92, 0xdf, 0xfe, + 0x58, 0x05, 0xe5, 0x29, 0x83, 0xfb, 0xb7, 0x51, + 0x7e, 0x66, 0x0c, 0x49, 0x3e, 0x11, 0x7e, 0x9b, + 0xb1, 0x83, 0x3a, 0xa6, 0xb0, 0x3c, 0xf5, 0xe0, +}; diff --git a/lib/crypto/tests/sm3_kunit.c b/lib/crypto/tests/sm3_kunit.c new file mode 100644 index 000000000000..dc8136acdff6 --- /dev/null +++ b/lib/crypto/tests/sm3_kunit.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2026 Google LLC + */ +#include +#include "sm3-testvecs.h" + +#define HASH sm3 +#define HASH_CTX sm3_ctx +#define HASH_SIZE SM3_DIGEST_SIZE +#define HASH_INIT sm3_init +#define HASH_UPDATE sm3_update +#define HASH_FINAL sm3_final +#include "hash-test-template.h" + +static struct kunit_case sm3_test_cases[] = { + HASH_KUNIT_CASES, + KUNIT_CASE(benchmark_hash), + {}, +}; + +static struct kunit_suite sm3_test_suite = { + .name = "sm3", + .test_cases = sm3_test_cases, + .suite_init = hash_suite_init, + .suite_exit = hash_suite_exit, +}; +kunit_test_suite(sm3_test_suite); + +MODULE_DESCRIPTION("KUnit tests and benchmark for SM3"); +MODULE_LICENSE("GPL"); diff --git a/scripts/crypto/gen-hash-testvecs.py b/scripts/crypto/gen-hash-testvecs.py index e69ce213fb33..f356f87e1c77 100755 --- a/scripts/crypto/gen-hash-testvecs.py +++ b/scripts/crypto/gen-hash-testvecs.py @@ -356,6 +356,9 @@ elif alg == 'sha3': print() print('/* SHAKE test vectors */') gen_additional_sha3_testvecs() +elif alg == 'sm3': + gen_unkeyed_testvecs(alg) + # Kernel doesn't implement HMAC-SM3 library functions yet. else: gen_unkeyed_testvecs(alg) gen_hmac_testvecs(alg) From ed065bd06ebe8d92d1647d230a14b9c035ad5b30 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:29 -0700 Subject: [PATCH 43/66] crypto: sm3 - Replace with wrapper around library Reimplement the "sm3" crypto_shash on top of the SM3 library, closely mirroring the other hash algorithms (e.g. SHA-*). The result, after later commits migrate the architecture-optimized SM3 code into the library as well, is that crypto/sm3.c will be the single point of integration between crypto_shash and the actual SM3 implementations, simplifying the code. Note: to see the diff from crypto/sm3_generic.c to crypto/sm3.c, view this commit with 'git show -M10'. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-7-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Makefile | 2 +- crypto/sm3.c | 89 +++++++++++++++++++++++++++ crypto/sm3_generic.c | 64 ------------------- crypto/testmgr.c | 2 + drivers/crypto/starfive/jh7110-hash.c | 4 +- 5 files changed, 94 insertions(+), 67 deletions(-) create mode 100644 crypto/sm3.c delete mode 100644 crypto/sm3_generic.c diff --git a/crypto/Makefile b/crypto/Makefile index 3fcbf0cd522d..68646175f6ab 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -83,7 +83,7 @@ obj-$(CONFIG_CRYPTO_SHA1) += sha1.o obj-$(CONFIG_CRYPTO_SHA256) += sha256.o obj-$(CONFIG_CRYPTO_SHA512) += sha512.o obj-$(CONFIG_CRYPTO_SHA3) += sha3.o -obj-$(CONFIG_CRYPTO_SM3) += sm3_generic.o +obj-$(CONFIG_CRYPTO_SM3) += sm3.o obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o CFLAGS_wp512.o := $(call cc-option,-fno-schedule-insns) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149 diff --git a/crypto/sm3.c b/crypto/sm3.c new file mode 100644 index 000000000000..05111a99b851 --- /dev/null +++ b/crypto/sm3.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * SM3 secure hash, as specified by OSCCA GM/T 0004-2012 SM3 and + * described at https://tools.ietf.org/html/draft-shen-sm3-hash-01 + * + * Copyright (C) 2017 ARM Limited or its affiliates. + * Written by Gilad Ben-Yossef + * Copyright (C) 2021 Tianjia Zhang + * Copyright 2026 Google LLC + */ + +#include +#include +#include +#include + +#define SM3_CTX(desc) ((struct sm3_ctx *)shash_desc_ctx(desc)) + +static int crypto_sm3_init(struct shash_desc *desc) +{ + sm3_init(SM3_CTX(desc)); + return 0; +} + +static int crypto_sm3_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + sm3_update(SM3_CTX(desc), data, len); + return 0; +} + +static int crypto_sm3_final(struct shash_desc *desc, u8 *out) +{ + sm3_final(SM3_CTX(desc), out); + return 0; +} + +static int crypto_sm3_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) +{ + sm3(data, len, out); + return 0; +} + +static int crypto_sm3_export_core(struct shash_desc *desc, void *out) +{ + memcpy(out, SM3_CTX(desc), sizeof(struct sm3_ctx)); + return 0; +} + +static int crypto_sm3_import_core(struct shash_desc *desc, const void *in) +{ + memcpy(SM3_CTX(desc), in, sizeof(struct sm3_ctx)); + return 0; +} + +static struct shash_alg sm3_alg = { + .base.cra_name = "sm3", + .base.cra_driver_name = "sm3-lib", + .base.cra_priority = 300, + .base.cra_blocksize = SM3_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SM3_DIGEST_SIZE, + .init = crypto_sm3_init, + .update = crypto_sm3_update, + .final = crypto_sm3_final, + .digest = crypto_sm3_digest, + .export_core = crypto_sm3_export_core, + .import_core = crypto_sm3_import_core, + .descsize = sizeof(struct sm3_ctx), +}; + +static int __init crypto_sm3_mod_init(void) +{ + return crypto_register_shash(&sm3_alg); +} +module_init(crypto_sm3_mod_init); + +static void __exit crypto_sm3_mod_exit(void) +{ + crypto_unregister_shash(&sm3_alg); +} +module_exit(crypto_sm3_mod_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Crypto API support for SM3"); + +MODULE_ALIAS_CRYPTO("sm3"); +MODULE_ALIAS_CRYPTO("sm3-lib"); diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c deleted file mode 100644 index 0c606f526347..000000000000 --- a/crypto/sm3_generic.c +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * SM3 secure hash, as specified by OSCCA GM/T 0004-2012 SM3 and - * described at https://tools.ietf.org/html/draft-shen-sm3-hash-01 - * - * Copyright (C) 2017 ARM Limited or its affiliates. - * Written by Gilad Ben-Yossef - * Copyright (C) 2021 Tianjia Zhang - */ - -#include -#include -#include -#include -#include - -static int crypto_sm3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sm3_base_do_update_blocks(desc, data, len, sm3_block_generic); -} - -static int crypto_sm3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash) -{ - sm3_base_do_finup(desc, data, len, sm3_block_generic); - return sm3_base_finish(desc, hash); -} - -static struct shash_alg sm3_alg = { - .digestsize = SM3_DIGEST_SIZE, - .init = sm3_base_init, - .update = crypto_sm3_update, - .finup = crypto_sm3_finup, - .descsize = SM3_STATE_SIZE, - .base = { - .cra_name = "sm3", - .cra_driver_name = "sm3-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sm3_generic_mod_init(void) -{ - return crypto_register_shash(&sm3_alg); -} - -static void __exit sm3_generic_mod_fini(void) -{ - crypto_unregister_shash(&sm3_alg); -} - -module_init(sm3_generic_mod_init); -module_exit(sm3_generic_mod_fini); - -MODULE_LICENSE("GPL v2"); -MODULE_DESCRIPTION("SM3 Secure Hash Algorithm"); - -MODULE_ALIAS_CRYPTO("sm3"); -MODULE_ALIAS_CRYPTO("sm3-generic"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index dd01f86dd6fe..60b6e4379aa6 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -5079,6 +5079,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "hmac(sm3)", + .generic_driver = "hmac(sm3-lib)", .test = alg_test_hash, .suite = { .hash = __VECS(hmac_sm3_tv_template) @@ -5446,6 +5447,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "sm3", + .generic_driver = "sm3-lib", .test = alg_test_hash, .suite = { .hash = __VECS(sm3_tv_template) diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c index 54b7af4a7aee..742038a5201a 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -520,7 +520,7 @@ static int starfive_sha512_init_tfm(struct crypto_ahash *hash) static int starfive_sm3_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sm3-generic", + return starfive_hash_init_tfm(hash, "sm3-lib", STARFIVE_HASH_SM3, 0); } @@ -550,7 +550,7 @@ static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash) static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sm3-generic)", + return starfive_hash_init_tfm(hash, "hmac(sm3-lib)", STARFIVE_HASH_SM3, 1); } From 9f69f52b462cdaed83b782d0408ce9286f054f92 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:30 -0700 Subject: [PATCH 44/66] lib/crypto: arm64/sm3: Migrate optimized code into library Instead of exposing the arm64-optimized SM3 code via arm64-specific crypto_shash algorithms, instead just implement the sm3_blocks() library function. This is much simpler, it makes the SM3 library functions be arm64-optimized, and it fixes the longstanding issue where the arm64-optimized SM3 code was disabled by default. SM3 still remains available through crypto_shash, but individual architectures no longer need to handle it. Tweak the SM3 assembly function prototypes to match what the library expects, including changing the block count from 'int' to 'size_t'. sm3_ce_transform() had to be updated to access 'x2' instead of 'w2', while sm3_neon_transform() already used 'x2'. Remove the CFI stubs which are no longer needed because the SM3 assembly functions are no longer ever indirectly called. Remove the dependency on KERNEL_MODE_NEON. It was unnecessary, because KERNEL_MODE_NEON is always enabled on arm64. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-8-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/configs/defconfig | 2 +- arch/arm64/crypto/Kconfig | 22 ------ arch/arm64/crypto/Makefile | 6 -- arch/arm64/crypto/sm3-ce-glue.c | 70 ------------------- arch/arm64/crypto/sm3-neon-glue.c | 67 ------------------ lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 13 +++- .../crypto => lib/crypto/arm64}/sm3-ce-core.S | 11 ++- .../crypto/arm64}/sm3-neon-core.S | 9 ++- lib/crypto/arm64/sm3.h | 41 +++++++++++ 10 files changed, 62 insertions(+), 180 deletions(-) delete mode 100644 arch/arm64/crypto/sm3-ce-glue.c delete mode 100644 arch/arm64/crypto/sm3-neon-glue.c rename {arch/arm64/crypto => lib/crypto/arm64}/sm3-ce-core.S (93%) rename {arch/arm64/crypto => lib/crypto/arm64}/sm3-neon-core.S (98%) create mode 100644 lib/crypto/arm64/sm3.h diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index b67d5b1fc45b..b4458bee767a 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1916,9 +1916,9 @@ CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_ECHAINIV=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_SHA3=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_GHASH_ARM64_CE=y -CONFIG_CRYPTO_SM3_ARM64_CE=m CONFIG_CRYPTO_AES_ARM64_CE_BLK=y CONFIG_CRYPTO_AES_ARM64_BS=m CONFIG_CRYPTO_AES_ARM64_CE_CCM=y diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 1a0c553fbfd7..0ac0fbfea10c 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -14,28 +14,6 @@ config CRYPTO_GHASH_ARM64_CE Architecture: arm64 using: - ARMv8 Crypto Extensions -config CRYPTO_SM3_NEON - tristate "Hash functions: SM3 (NEON)" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_LIB_SM3 - help - SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) - - Architecture: arm64 using: - - NEON (Advanced SIMD) extensions - -config CRYPTO_SM3_ARM64_CE - tristate "Hash functions: SM3 (ARMv8.2 Crypto Extensions)" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_LIB_SM3 - help - SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) - - Architecture: arm64 using: - - ARMv8.2 Crypto Extensions - config CRYPTO_AES_ARM64_CE_BLK tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index 8a8e3e551ed3..a169f9033401 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -5,12 +5,6 @@ # Copyright (C) 2014 Linaro Ltd # -obj-$(CONFIG_CRYPTO_SM3_NEON) += sm3-neon.o -sm3-neon-y := sm3-neon-glue.o sm3-neon-core.o - -obj-$(CONFIG_CRYPTO_SM3_ARM64_CE) += sm3-ce.o -sm3-ce-y := sm3-ce-glue.o sm3-ce-core.o - obj-$(CONFIG_CRYPTO_SM4_ARM64_CE) += sm4-ce-cipher.o sm4-ce-cipher-y := sm4-ce-cipher-glue.o sm4-ce-cipher-core.o diff --git a/arch/arm64/crypto/sm3-ce-glue.c b/arch/arm64/crypto/sm3-ce-glue.c deleted file mode 100644 index 24c1fcfae072..000000000000 --- a/arch/arm64/crypto/sm3-ce-glue.c +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sm3-ce-glue.c - SM3 secure hash using ARMv8.2 Crypto Extensions - * - * Copyright (C) 2018 Linaro Ltd - */ - -#include -#include -#include -#include -#include -#include - -#include - -MODULE_DESCRIPTION("SM3 secure hash using ARMv8 Crypto Extensions"); -MODULE_AUTHOR("Ard Biesheuvel "); -MODULE_LICENSE("GPL v2"); - -asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src, - int blocks); - -static int sm3_ce_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - int remain; - - scoped_ksimd() { - remain = sm3_base_do_update_blocks(desc, data, len, sm3_ce_transform); - } - return remain; -} - -static int sm3_ce_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - scoped_ksimd() { - sm3_base_do_finup(desc, data, len, sm3_ce_transform); - } - return sm3_base_finish(desc, out); -} - -static struct shash_alg sm3_alg = { - .digestsize = SM3_DIGEST_SIZE, - .init = sm3_base_init, - .update = sm3_ce_update, - .finup = sm3_ce_finup, - .descsize = SM3_STATE_SIZE, - .base.cra_name = "sm3", - .base.cra_driver_name = "sm3-ce", - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SM3_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - .base.cra_priority = 400, -}; - -static int __init sm3_ce_mod_init(void) -{ - return crypto_register_shash(&sm3_alg); -} - -static void __exit sm3_ce_mod_fini(void) -{ - crypto_unregister_shash(&sm3_alg); -} - -module_cpu_feature_match(SM3, sm3_ce_mod_init); -module_exit(sm3_ce_mod_fini); diff --git a/arch/arm64/crypto/sm3-neon-glue.c b/arch/arm64/crypto/sm3-neon-glue.c deleted file mode 100644 index 15f30cc24f32..000000000000 --- a/arch/arm64/crypto/sm3-neon-glue.c +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * sm3-neon-glue.c - SM3 secure hash using NEON instructions - * - * Copyright (C) 2022 Tianjia Zhang - */ - -#include -#include -#include -#include -#include -#include -#include - - -asmlinkage void sm3_neon_transform(struct sm3_state *sst, u8 const *src, - int blocks); - -static int sm3_neon_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - scoped_ksimd() - return sm3_base_do_update_blocks(desc, data, len, - sm3_neon_transform); -} - -static int sm3_neon_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - scoped_ksimd() - sm3_base_do_finup(desc, data, len, sm3_neon_transform); - return sm3_base_finish(desc, out); -} - -static struct shash_alg sm3_alg = { - .digestsize = SM3_DIGEST_SIZE, - .init = sm3_base_init, - .update = sm3_neon_update, - .finup = sm3_neon_finup, - .descsize = SM3_STATE_SIZE, - .base.cra_name = "sm3", - .base.cra_driver_name = "sm3-neon", - .base.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .base.cra_blocksize = SM3_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - .base.cra_priority = 200, -}; - -static int __init sm3_neon_init(void) -{ - return crypto_register_shash(&sm3_alg); -} - -static void __exit sm3_neon_fini(void) -{ - crypto_unregister_shash(&sm3_alg); -} - -module_init(sm3_neon_init); -module_exit(sm3_neon_fini); - -MODULE_DESCRIPTION("SM3 secure hash using NEON instructions"); -MODULE_AUTHOR("Jussi Kivilinna "); -MODULE_AUTHOR("Tianjia Zhang "); -MODULE_LICENSE("GPL v2"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 64c9a0bc4099..c85956e443a2 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -279,6 +279,7 @@ config CRYPTO_LIB_SM3 config CRYPTO_LIB_SM3_ARCH bool depends on CRYPTO_LIB_SM3 && !UML + default y if ARM64 source "lib/crypto/tests/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 19c67f70fb38..9c2718012428 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -368,13 +368,20 @@ endif # CONFIG_CRYPTO_LIB_SHA3_ARCH ################################################################################ +obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o +libsm3-y := sm3.o +ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y) +CFLAGS_sm3.o += -I$(src)/$(SRCARCH) +libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \ + arm64/sm3-neon-core.o +endif # CONFIG_CRYPTO_LIB_SM3_ARCH + +################################################################################ + obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o -obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o -libsm3-y := sm3.o - # clean-files must be defined unconditionally clean-files += arm/sha256-core.S arm/sha512-core.S clean-files += arm64/sha256-core.S arm64/sha512-core.S diff --git a/arch/arm64/crypto/sm3-ce-core.S b/lib/crypto/arm64/sm3-ce-core.S similarity index 93% rename from arch/arm64/crypto/sm3-ce-core.S rename to lib/crypto/arm64/sm3-ce-core.S index ca70cfacd0d0..9cef7ea7f34f 100644 --- a/arch/arm64/crypto/sm3-ce-core.S +++ b/lib/crypto/arm64/sm3-ce-core.S @@ -6,7 +6,6 @@ */ #include -#include #include .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 @@ -70,11 +69,11 @@ .endm /* - * void sm3_ce_transform(struct sm3_state *sst, u8 const *src, - * int blocks) + * void sm3_ce_transform(struct sm3_block_state *state, + * const u8 *data, size_t nblocks) */ .text -SYM_TYPED_FUNC_START(sm3_ce_transform) +SYM_FUNC_START(sm3_ce_transform) /* load state */ ld1 {v8.4s-v9.4s}, [x0] rev64 v8.4s, v8.4s @@ -87,7 +86,7 @@ SYM_TYPED_FUNC_START(sm3_ce_transform) /* load input */ 0: ld1 {v0.16b-v3.16b}, [x1], #64 - sub w2, w2, #1 + sub x2, x2, #1 mov v15.16b, v8.16b mov v16.16b, v9.16b @@ -123,7 +122,7 @@ CPU_LE( rev32 v3.16b, v3.16b ) eor v9.16b, v9.16b, v16.16b /* handled all input blocks? */ - cbnz w2, 0b + cbnz x2, 0b /* save state */ rev64 v8.4s, v8.4s diff --git a/arch/arm64/crypto/sm3-neon-core.S b/lib/crypto/arm64/sm3-neon-core.S similarity index 98% rename from arch/arm64/crypto/sm3-neon-core.S rename to lib/crypto/arm64/sm3-neon-core.S index 4357e0e51be3..ad874af13802 100644 --- a/arch/arm64/crypto/sm3-neon-core.S +++ b/lib/crypto/arm64/sm3-neon-core.S @@ -9,7 +9,6 @@ */ #include -#include #include /* Context structure */ @@ -345,14 +344,14 @@ /* - * Transform blocks*64 bytes (blocks*16 32-bit words) at 'src'. + * Transform nblocks*64 bytes (nblocks*16 32-bit words) at 'data'. * - * void sm3_neon_transform(struct sm3_state *sst, u8 const *src, - * int blocks) + * void sm3_neon_transform(struct sm3_block_state *state, + * const u8 *data, size_t nblocks) */ .text .align 3 -SYM_TYPED_FUNC_START(sm3_neon_transform) +SYM_FUNC_START(sm3_neon_transform) ldp ra, rb, [RSTATE, #0] ldp rc, rd, [RSTATE, #8] ldp re, rf, [RSTATE, #16] diff --git a/lib/crypto/arm64/sm3.h b/lib/crypto/arm64/sm3.h new file mode 100644 index 000000000000..beb9cd82bb7d --- /dev/null +++ b/lib/crypto/arm64/sm3.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 optimized for ARM64 + * + * Copyright 2026 Google LLC + */ +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); + +asmlinkage void sm3_neon_transform(struct sm3_block_state *state, + const u8 *data, size_t nblocks); +asmlinkage void sm3_ce_transform(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { + scoped_ksimd() { + if (static_branch_likely(&have_ce)) + sm3_ce_transform(state, data, nblocks); + else + sm3_neon_transform(state, data, nblocks); + } + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_neon); + if (cpu_have_named_feature(SM3)) + static_branch_enable(&have_ce); + } +} From 5f6bbba5e9bb7f271557513d0ed77bb7b5a92698 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:31 -0700 Subject: [PATCH 45/66] lib/crypto: riscv/sm3: Migrate optimized code into library Instead of exposing the riscv-optimized SM3 code via a riscv-specific crypto_shash algorithm, instead just implement the sm3_blocks() library function. This is much simpler, it makes the SM3 library functions be riscv-optimized, and it fixes the longstanding issue where the riscv-optimized SM3 code was disabled by default. SM3 still remains available through crypto_shash, but individual architectures no longer need to handle it. Tweak the prototype of sm3_transform_zvksh_zvkb() to match what the library expects, including changing the block count to size_t. Note that the assembly code already treated it as size_t. Note: to see the diff from arch/riscv/crypto/sm3-riscv64-glue.c to lib/crypto/riscv/sm3.h, view this commit with 'git show -M10'. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-9-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/riscv/crypto/Kconfig | 13 --- arch/riscv/crypto/Makefile | 3 - arch/riscv/crypto/sm3-riscv64-glue.c | 97 ------------------- lib/crypto/Kconfig | 2 + lib/crypto/Makefile | 1 + .../crypto/riscv}/sm3-riscv64-zvksh-zvkb.S | 3 +- lib/crypto/riscv/sm3.h | 39 ++++++++ 7 files changed, 44 insertions(+), 114 deletions(-) delete mode 100644 arch/riscv/crypto/sm3-riscv64-glue.c rename {arch/riscv/crypto => lib/crypto/riscv}/sm3-riscv64-zvksh-zvkb.S (97%) create mode 100644 lib/crypto/riscv/sm3.h diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index c208f54afbcd..6905232ddb03 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -17,19 +17,6 @@ config CRYPTO_AES_RISCV64 - Zvkb vector crypto extension (CTR) - Zvkg vector crypto extension (XTS) -config CRYPTO_SM3_RISCV64 - tristate "Hash functions: SM3 (ShangMi 3)" - depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ - RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS - select CRYPTO_HASH - select CRYPTO_LIB_SM3 - help - SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) - - Architecture: riscv64 using: - - Zvksh vector crypto extension - - Zvkb vector crypto extension - config CRYPTO_SM4_RISCV64 tristate "Ciphers: SM4 (ShangMi 4)" depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile index 5c9ee1b876fa..8cf31db57fc4 100644 --- a/arch/riscv/crypto/Makefile +++ b/arch/riscv/crypto/Makefile @@ -4,8 +4,5 @@ obj-$(CONFIG_CRYPTO_AES_RISCV64) += aes-riscv64.o aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \ aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o -obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o -sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o - obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o diff --git a/arch/riscv/crypto/sm3-riscv64-glue.c b/arch/riscv/crypto/sm3-riscv64-glue.c deleted file mode 100644 index abdfe4a63a27..000000000000 --- a/arch/riscv/crypto/sm3-riscv64-glue.c +++ /dev/null @@ -1,97 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SM3 using the RISC-V vector crypto extensions - * - * Copyright (C) 2023 VRULL GmbH - * Author: Heiko Stuebner - * - * Copyright (C) 2023 SiFive, Inc. - * Author: Jerry Shih - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * Note: the asm function only uses the 'state' field of struct sm3_state. - * It is assumed to be the first field. - */ -asmlinkage void sm3_transform_zvksh_zvkb( - struct sm3_state *state, const u8 *data, int num_blocks); - -static void sm3_block(struct sm3_state *state, const u8 *data, - int num_blocks) -{ - /* - * Ensure struct sm3_state begins directly with the SM3 - * 256-bit internal state, as this is what the asm function expects. - */ - BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); - - if (crypto_simd_usable()) { - kernel_vector_begin(); - sm3_transform_zvksh_zvkb(state, data, num_blocks); - kernel_vector_end(); - } else { - sm3_block_generic(state, data, num_blocks); - } -} - -static int riscv64_sm3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sm3_base_do_update_blocks(desc, data, len, sm3_block); -} - -static int riscv64_sm3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - sm3_base_do_finup(desc, data, len, sm3_block); - return sm3_base_finish(desc, out); -} - -static struct shash_alg riscv64_sm3_alg = { - .init = sm3_base_init, - .update = riscv64_sm3_update, - .finup = riscv64_sm3_finup, - .descsize = SM3_STATE_SIZE, - .digestsize = SM3_DIGEST_SIZE, - .base = { - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_priority = 300, - .cra_name = "sm3", - .cra_driver_name = "sm3-riscv64-zvksh-zvkb", - .cra_module = THIS_MODULE, - }, -}; - -static int __init riscv64_sm3_mod_init(void) -{ - if (riscv_isa_extension_available(NULL, ZVKSH) && - riscv_isa_extension_available(NULL, ZVKB) && - riscv_vector_vlen() >= 128) - return crypto_register_shash(&riscv64_sm3_alg); - - return -ENODEV; -} - -static void __exit riscv64_sm3_mod_exit(void) -{ - crypto_unregister_shash(&riscv64_sm3_alg); -} - -module_init(riscv64_sm3_mod_init); -module_exit(riscv64_sm3_mod_exit); - -MODULE_DESCRIPTION("SM3 (RISC-V accelerated)"); -MODULE_AUTHOR("Heiko Stuebner "); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("sm3"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index c85956e443a2..b209597de5ff 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -280,6 +280,8 @@ config CRYPTO_LIB_SM3_ARCH bool depends on CRYPTO_LIB_SM3 && !UML default y if ARM64 + default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ + RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS source "lib/crypto/tests/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 9c2718012428..ad8da7f3af78 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -374,6 +374,7 @@ ifeq ($(CONFIG_CRYPTO_LIB_SM3_ARCH),y) CFLAGS_sm3.o += -I$(src)/$(SRCARCH) libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \ arm64/sm3-neon-core.o +libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o endif # CONFIG_CRYPTO_LIB_SM3_ARCH ################################################################################ diff --git a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S similarity index 97% rename from arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S rename to lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S index 4fe754846f65..a1d4468b0485 100644 --- a/arch/riscv/crypto/sm3-riscv64-zvksh-zvkb.S +++ b/lib/crypto/riscv/sm3-riscv64-zvksh-zvkb.S @@ -80,7 +80,8 @@ // For the next 8 rounds, w0 and w1 are swapped. .endm -// void sm3_transform_zvksh_zvkb(u32 state[8], const u8 *data, int num_blocks); +// void sm3_transform_zvksh_zvkb(struct sm3_block_state *state, +// const u8 *data, size_t nblocks); SYM_FUNC_START(sm3_transform_zvksh_zvkb) // Load the state and endian-swap each 32-bit word. diff --git a/lib/crypto/riscv/sm3.h b/lib/crypto/riscv/sm3.h new file mode 100644 index 000000000000..c1fbee7094e6 --- /dev/null +++ b/lib/crypto/riscv/sm3.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 using the RISC-V vector crypto extensions + * + * Copyright (C) 2023 VRULL GmbH + * Author: Heiko Stuebner + * + * Copyright (C) 2023 SiFive, Inc. + * Author: Jerry Shih + */ + +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions); + +asmlinkage void sm3_transform_zvksh_zvkb(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_extensions) && likely(may_use_simd())) { + kernel_vector_begin(); + sm3_transform_zvksh_zvkb(state, data, nblocks); + kernel_vector_end(); + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (riscv_isa_extension_available(NULL, ZVKSH) && + riscv_isa_extension_available(NULL, ZVKB) && + riscv_vector_vlen() >= 128) + static_branch_enable(&have_extensions); +} From 17ba6108d3e084652807826cc49c851c00976f1a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:32 -0700 Subject: [PATCH 46/66] lib/crypto: x86/sm3: Migrate optimized code into library Instead of exposing the x86-optimized SM3 code via an x86-specific crypto_shash algorithm, instead just implement the sm3_blocks() library function. This is much simpler, it makes the SM3 library functions be x86-optimized, and it fixes the longstanding issue where the x86-optimized SM3 code was disabled by default. SM3 still remains available through crypto_shash, but individual architectures no longer need to handle it. Tweak the prototype of sm3_transform_avx() to match what the library expects, including changing the block count to size_t. Note that the assembly code actually already treated this argument as size_t. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-10-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/x86/crypto/Kconfig | 13 --- arch/x86/crypto/Makefile | 3 - arch/x86/crypto/sm3_avx_glue.c | 100 ------------------ lib/crypto/Kconfig | 1 + lib/crypto/Makefile | 1 + .../crypto/x86}/sm3-avx-asm_64.S | 13 ++- lib/crypto/x86/sm3.h | 39 +++++++ 7 files changed, 47 insertions(+), 123 deletions(-) delete mode 100644 arch/x86/crypto/sm3_avx_glue.c rename {arch/x86/crypto => lib/crypto/x86}/sm3-avx-asm_64.S (98%) create mode 100644 lib/crypto/x86/sm3.h diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 905e8a23cec3..822cbf414269 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -331,17 +331,4 @@ config CRYPTO_AEGIS128_AESNI_SSE2 - AES-NI (AES New Instructions) - SSE4.1 (Streaming SIMD Extensions 4.1) -config CRYPTO_SM3_AVX_X86_64 - tristate "Hash functions: SM3 (AVX)" - depends on 64BIT - select CRYPTO_HASH - select CRYPTO_LIB_SM3 - help - SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3 - - Architecture: x86_64 using: - - AVX (Advanced Vector Extensions) - - If unsure, say N. - endmenu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index d562f4341da6..3d6d5087a65e 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -50,9 +50,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \ aes-gcm-vaes-avx512.o \ aes-xts-avx-x86_64.o -obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o -sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o - obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o diff --git a/arch/x86/crypto/sm3_avx_glue.c b/arch/x86/crypto/sm3_avx_glue.c deleted file mode 100644 index 6e8c42b9dc8e..000000000000 --- a/arch/x86/crypto/sm3_avx_glue.c +++ /dev/null @@ -1,100 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * SM3 Secure Hash Algorithm, AVX assembler accelerated. - * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02 - * - * Copyright (C) 2021 Tianjia Zhang - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void sm3_transform_avx(struct sm3_state *state, - const u8 *data, int nblocks); - -static int sm3_avx_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - int remain; - - /* - * Make sure struct sm3_state begins directly with the SM3 - * 256-bit internal state, as this is what the asm functions expect. - */ - BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0); - - kernel_fpu_begin(); - remain = sm3_base_do_update_blocks(desc, data, len, sm3_transform_avx); - kernel_fpu_end(); - return remain; -} - -static int sm3_avx_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - kernel_fpu_begin(); - sm3_base_do_finup(desc, data, len, sm3_transform_avx); - kernel_fpu_end(); - return sm3_base_finish(desc, out); -} - -static struct shash_alg sm3_avx_alg = { - .digestsize = SM3_DIGEST_SIZE, - .init = sm3_base_init, - .update = sm3_avx_update, - .finup = sm3_avx_finup, - .descsize = SM3_STATE_SIZE, - .base = { - .cra_name = "sm3", - .cra_driver_name = "sm3-avx", - .cra_priority = 300, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sm3_avx_mod_init(void) -{ - const char *feature_name; - - if (!boot_cpu_has(X86_FEATURE_AVX)) { - pr_info("AVX instruction are not detected.\n"); - return -ENODEV; - } - - if (!boot_cpu_has(X86_FEATURE_BMI2)) { - pr_info("BMI2 instruction are not detected.\n"); - return -ENODEV; - } - - if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, - &feature_name)) { - pr_info("CPU feature '%s' is not supported.\n", feature_name); - return -ENODEV; - } - - return crypto_register_shash(&sm3_avx_alg); -} - -static void __exit sm3_avx_mod_exit(void) -{ - crypto_unregister_shash(&sm3_avx_alg); -} - -module_init(sm3_avx_mod_init); -module_exit(sm3_avx_mod_exit); - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Tianjia Zhang "); -MODULE_DESCRIPTION("SM3 Secure Hash Algorithm, AVX assembler accelerated"); -MODULE_ALIAS_CRYPTO("sm3"); -MODULE_ALIAS_CRYPTO("sm3-avx"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index b209597de5ff..91b1d0eb13b0 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -282,6 +282,7 @@ config CRYPTO_LIB_SM3_ARCH default y if ARM64 default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS + default y if X86_64 source "lib/crypto/tests/Kconfig" diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index ad8da7f3af78..ec1747f51d07 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -375,6 +375,7 @@ CFLAGS_sm3.o += -I$(src)/$(SRCARCH) libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \ arm64/sm3-neon-core.o libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o +libsm3-$(CONFIG_X86) += x86/sm3-avx-asm_64.o endif # CONFIG_CRYPTO_LIB_SM3_ARCH ################################################################################ diff --git a/arch/x86/crypto/sm3-avx-asm_64.S b/lib/crypto/x86/sm3-avx-asm_64.S similarity index 98% rename from arch/x86/crypto/sm3-avx-asm_64.S rename to lib/crypto/x86/sm3-avx-asm_64.S index 503bab450a91..a1925b136010 100644 --- a/arch/x86/crypto/sm3-avx-asm_64.S +++ b/lib/crypto/x86/sm3-avx-asm_64.S @@ -12,10 +12,9 @@ */ #include -#include #include -/* Context structure */ +/* State structure */ #define state_h0 0 #define state_h1 4 @@ -325,13 +324,13 @@ /* * Transform nblocks*64 bytes (nblocks*16 32-bit words) at DATA. * - * void sm3_transform_avx(struct sm3_state *state, - * const u8 *data, int nblocks); + * void sm3_transform_avx(struct sm3_block_state *state, + * const u8 *data, size_t nblocks); */ -SYM_TYPED_FUNC_START(sm3_transform_avx) +SYM_FUNC_START(sm3_transform_avx) /* input: - * %rdi: ctx, CTX - * %rsi: data (64*nblks bytes) + * %rdi: state + * %rsi: data * %rdx: nblocks */ vzeroupper; diff --git a/lib/crypto/x86/sm3.h b/lib/crypto/x86/sm3.h new file mode 100644 index 000000000000..3834780f2f6a --- /dev/null +++ b/lib/crypto/x86/sm3.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM3 optimized for x86_64 + * + * Copyright 2026 Google LLC + */ +#include +#include + +asmlinkage void sm3_transform_avx(struct sm3_block_state *state, + const u8 *data, size_t nblocks); + +static void sm3_blocks_avx(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + if (likely(irq_fpu_usable())) { + kernel_fpu_begin(); + sm3_transform_avx(state, data, nblocks); + kernel_fpu_end(); + } else { + sm3_blocks_generic(state, data, nblocks); + } +} + +DEFINE_STATIC_CALL(sm3_blocks_x86, sm3_blocks_generic); + +static void sm3_blocks(struct sm3_block_state *state, + const u8 *data, size_t nblocks) +{ + static_call(sm3_blocks_x86)(state, data, nblocks); +} + +#define sm3_mod_init_arch sm3_mod_init_arch +static void sm3_mod_init_arch(void) +{ + if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_BMI2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) + static_call_update(sm3_blocks_x86, sm3_blocks_avx); +} From 9d7f2a6ed598e82b07582d4f5122f66111ef5c00 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:33 -0700 Subject: [PATCH 47/66] crypto: sm3 - Remove sm3_base.h Remove include/crypto/sm3_base.h, since it's no longer used. The corresponding logic was reimplemented in a central place in lib/crypto/. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-11-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sm3_base.h | 92 --------------------------------------- 1 file changed, 92 deletions(-) delete mode 100644 include/crypto/sm3_base.h diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h deleted file mode 100644 index 9fa995617495..000000000000 --- a/include/crypto/sm3_base.h +++ /dev/null @@ -1,92 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * sm3_base.h - core logic for SM3 implementations - * - * Copyright (C) 2017 ARM Limited or its affiliates. - * Written by Gilad Ben-Yossef - */ - -#ifndef _CRYPTO_SM3_BASE_H -#define _CRYPTO_SM3_BASE_H - -#include -#include -#include -#include -#include -#include -#include - -typedef void (sm3_block_fn)(struct sm3_state *sst, u8 const *src, int blocks); - -static inline int sm3_base_init(struct shash_desc *desc) -{ - struct sm3_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SM3_IVA; - sctx->state[1] = SM3_IVB; - sctx->state[2] = SM3_IVC; - sctx->state[3] = SM3_IVD; - sctx->state[4] = SM3_IVE; - sctx->state[5] = SM3_IVF; - sctx->state[6] = SM3_IVG; - sctx->state[7] = SM3_IVH; - sctx->count = 0; - return 0; -} - -static inline int sm3_base_do_update_blocks(struct shash_desc *desc, - const u8 *data, unsigned int len, - sm3_block_fn *block_fn) -{ - unsigned int remain = len - round_down(len, SM3_BLOCK_SIZE); - struct sm3_state *sctx = shash_desc_ctx(desc); - - sctx->count += len - remain; - block_fn(sctx, data, len / SM3_BLOCK_SIZE); - return remain; -} - -static inline int sm3_base_do_finup(struct shash_desc *desc, - const u8 *src, unsigned int len, - sm3_block_fn *block_fn) -{ - unsigned int bit_offset = SM3_BLOCK_SIZE / 8 - 1; - struct sm3_state *sctx = shash_desc_ctx(desc); - union { - __be64 b64[SM3_BLOCK_SIZE / 4]; - u8 u8[SM3_BLOCK_SIZE * 2]; - } block = {}; - - if (len >= SM3_BLOCK_SIZE) { - int remain; - - remain = sm3_base_do_update_blocks(desc, src, len, block_fn); - src += len - remain; - len = remain; - } - - if (len >= bit_offset * 8) - bit_offset += SM3_BLOCK_SIZE / 8; - memcpy(&block, src, len); - block.u8[len] = 0x80; - sctx->count += len; - block.b64[bit_offset] = cpu_to_be64(sctx->count << 3); - block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SM3_BLOCK_SIZE); - memzero_explicit(&block, sizeof(block)); - - return 0; -} - -static inline int sm3_base_finish(struct shash_desc *desc, u8 *out) -{ - struct sm3_state *sctx = shash_desc_ctx(desc); - __be32 *digest = (__be32 *)out; - int i; - - for (i = 0; i < SM3_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], digest++); - return 0; -} - -#endif /* _CRYPTO_SM3_BASE_H */ From ef01e1eafb20f74e6d951a42a870a40cd8b914ca Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:34 -0700 Subject: [PATCH 48/66] crypto: sm3 - Remove the original "sm3_block_generic()" Since the architecture-optimized SM3 code was migrated into lib/crypto/, sm3_block_generic() is no longer called. Remove it. Then, since this frees up the name, rename sm3_transform() to sm3_block_generic() (matching the naming convention used in other hash algorithms). Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-12-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/sm3.h | 2 -- lib/crypto/sm3.c | 19 +++---------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index 702c5326b4be..34d7eb32b7db 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -31,8 +31,6 @@ struct sm3_state { u8 buffer[SM3_BLOCK_SIZE]; }; -void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks); - /* State for the SM3 compression function */ struct sm3_block_state { u32 h[SM3_DIGEST_SIZE / 4]; diff --git a/lib/crypto/sm3.c b/lib/crypto/sm3.c index 20500cf4b8c0..b02b8a247adf 100644 --- a/lib/crypto/sm3.c +++ b/lib/crypto/sm3.c @@ -79,8 +79,8 @@ static const u32 ____cacheline_aligned K[64] = { ^ rol32(W[(i-13) & 0x0f], 7) \ ^ W[(i-6) & 0x0f]) -static void sm3_transform(struct sm3_block_state *state, - const u8 data[SM3_BLOCK_SIZE], u32 W[16]) +static void sm3_block_generic(struct sm3_block_state *state, + const u8 data[SM3_BLOCK_SIZE], u32 W[16]) { u32 a, b, c, d, e, f, g, h, ss1, ss2; @@ -177,26 +177,13 @@ static void sm3_transform(struct sm3_block_state *state, #undef W1 #undef W2 -void sm3_block_generic(struct sm3_state *sctx, u8 const *data, int blocks) -{ - u32 W[16]; - - do { - sm3_transform((struct sm3_block_state *)sctx->state, data, W); - data += SM3_BLOCK_SIZE; - } while (--blocks); - - memzero_explicit(W, sizeof(W)); -} -EXPORT_SYMBOL_GPL(sm3_block_generic); - static void __maybe_unused sm3_blocks_generic(struct sm3_block_state *state, const u8 *data, size_t nblocks) { u32 W[16]; do { - sm3_transform(state, data, W); + sm3_block_generic(state, data, W); data += SM3_BLOCK_SIZE; } while (--nblocks); From e37f28529b380265904af64996d34c647d917ef1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:35 -0700 Subject: [PATCH 49/66] crypto: sm3 - Remove 'struct sm3_state' Update one driver that used sizeof(struct sm3_state) to use sizeof(struct sm3_ctx) instead. Then, remove struct sm3_state and SM3_STATE_SIZE. This completes the replacement of struct sm3_state with struct sm3_ctx. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-13-ebiggers@kernel.org Signed-off-by: Eric Biggers --- drivers/crypto/starfive/jh7110-hash.c | 4 ++-- include/crypto/sm3.h | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c index 742038a5201a..008a47baa165 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -795,7 +795,7 @@ static struct ahash_engine_alg algs_sha2_sm3[] = { .base.exit_tfm = starfive_hash_exit_tfm, .base.halg = { .digestsize = SM3_DIGEST_SIZE, - .statesize = sizeof(struct sm3_state), + .statesize = sizeof(struct sm3_ctx), .base = { .cra_name = "sm3", .cra_driver_name = "sm3-starfive", @@ -824,7 +824,7 @@ static struct ahash_engine_alg algs_sha2_sm3[] = { .base.setkey = starfive_hash_setkey, .base.halg = { .digestsize = SM3_DIGEST_SIZE, - .statesize = sizeof(struct sm3_state), + .statesize = sizeof(struct sm3_ctx), .base = { .cra_name = "hmac(sm3)", .cra_driver_name = "sm3-hmac-starfive", diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index 34d7eb32b7db..371e8a661705 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -14,7 +14,6 @@ #define SM3_DIGEST_SIZE 32 #define SM3_BLOCK_SIZE 64 -#define SM3_STATE_SIZE 40 #define SM3_IVA 0x7380166f #define SM3_IVB 0x4914b2b9 @@ -25,12 +24,6 @@ #define SM3_IVG 0xe38dee4d #define SM3_IVH 0xb0fb0e4e -struct sm3_state { - u32 state[SM3_DIGEST_SIZE / 4]; - u64 count; - u8 buffer[SM3_BLOCK_SIZE]; -}; - /* State for the SM3 compression function */ struct sm3_block_state { u32 h[SM3_DIGEST_SIZE / 4]; From 7ac21b4032e5b9b8a6a312b6f1d54f4ba24d1c16 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 21 Mar 2026 20:24:38 -0700 Subject: [PATCH 50/66] lib: Move crypto library tests to Runtime Testing menu Currently the kconfig options for the crypto library KUnit tests appear in the menu: -> Library routines -> Crypto library routines However, this is the only content of "Crypto library routines". I.e., it is empty when CONFIG_KUNIT=n. This is because the crypto library routines themselves don't have (or need to have) prompts. Since this usually ends up as an unnecessary empty menu, let's remove this menu and instead source the lib/crypto/tests/Kconfig file from lib/Kconfig.debug inside the "Runtime Testing" menu: -> Kernel hacking -> Kernel Testing and Coverage -> Runtime Testing This puts the prompts alongside the ones for most of the other lib/ KUnit tests. This seems to be a much better match to how the kconfig menus are organized. Acked-by: Randy Dunlap Tested-by: Randy Dunlap Link: https://lore.kernel.org/r/20260322032438.286296-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/Kconfig.debug | 2 ++ lib/crypto/Kconfig | 6 ------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 93f356d2b3d9..146358530010 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -3058,6 +3058,8 @@ config HW_BREAKPOINT_KUNIT_TEST If unsure, say N. +source "lib/crypto/tests/Kconfig" + config SIPHASH_KUNIT_TEST tristate "Perform selftest on siphash functions" if !KUNIT_ALL_TESTS depends on KUNIT diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 91b1d0eb13b0..4b6f593dc72f 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -1,7 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -menu "Crypto library routines" - config CRYPTO_HASH_INFO bool @@ -283,7 +281,3 @@ config CRYPTO_LIB_SM3_ARCH default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \ RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS default y if X86_64 - -source "lib/crypto/tests/Kconfig" - -endmenu From 91cd9a03372be647fff09acab525c15e9c9b66f0 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 26 Mar 2026 13:48:24 -0700 Subject: [PATCH 51/66] lib/crypto: mips: Drop optimized MD5 code MD5 is obsolete. Continuing to maintain architecture-optimized implementations of MD5 is unnecessary and risky. It diverts resources from the modern algorithms that are actually important. While there was demand for continuing to maintain the PowerPC optimized MD5 code to accommodate userspace programs that are misusing AF_ALG (https://lore.kernel.org/linux-crypto/c4191597-341d-4fd7-bc3d-13daf7666c41@csgroup.eu/), no such demand has been seen for the MIPS Cavium Octeon optimized MD5 code. Note that this code runs on only one particular line of SoCs. Thus, let's drop it and focus effort on the more modern SHA algorithms, which already have optimized code for the same SoCs. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260326204824.62010-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/Kconfig | 1 - lib/crypto/mips/md5.h | 65 ------------------------------------------- 2 files changed, 66 deletions(-) delete mode 100644 lib/crypto/mips/md5.h diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 4b6f593dc72f..9f31f03062f0 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -134,7 +134,6 @@ config CRYPTO_LIB_MD5 config CRYPTO_LIB_MD5_ARCH bool depends on CRYPTO_LIB_MD5 && !UML - default y if MIPS && CPU_CAVIUM_OCTEON default y if PPC default y if SPARC64 diff --git a/lib/crypto/mips/md5.h b/lib/crypto/mips/md5.h deleted file mode 100644 index e08e28aeffa4..000000000000 --- a/lib/crypto/mips/md5.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Cryptographic API. - * - * MD5 Message Digest Algorithm (RFC1321). - * - * Adapted for OCTEON by Aaro Koskinen . - * - * Based on crypto/md5.c, which is: - * - * Derived from cryptoapi implementation, originally based on the - * public domain implementation written by Colin Plumb in 1993. - * - * Copyright (c) Cryptoapi developers. - * Copyright (c) 2002 James Morris - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - */ - -#include -#include - -/* - * We pass everything as 64-bit. OCTEON can handle misaligned data. - */ - -static void md5_blocks(struct md5_block_state *state, - const u8 *data, size_t nblocks) -{ - struct octeon_cop2_state cop2_state; - u64 *state64 = (u64 *)state; - unsigned long flags; - - if (!octeon_has_crypto()) - return md5_blocks_generic(state, data, nblocks); - - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - - flags = octeon_crypto_enable(&cop2_state); - write_octeon_64bit_hash_dword(state64[0], 0); - write_octeon_64bit_hash_dword(state64[1], 1); - - do { - const u64 *block = (const u64 *)data; - - write_octeon_64bit_block_dword(block[0], 0); - write_octeon_64bit_block_dword(block[1], 1); - write_octeon_64bit_block_dword(block[2], 2); - write_octeon_64bit_block_dword(block[3], 3); - write_octeon_64bit_block_dword(block[4], 4); - write_octeon_64bit_block_dword(block[5], 5); - write_octeon_64bit_block_dword(block[6], 6); - octeon_md5_start(block[7]); - - data += MD5_BLOCK_SIZE; - } while (--nblocks); - - state64[0] = read_octeon_64bit_hash_dword(0); - state64[1] = read_octeon_64bit_hash_dword(1); - octeon_crypto_disable(&cop2_state, flags); - - le32_to_cpu_array(state->h, ARRAY_SIZE(state->h)); -} From 23e5c306a207360bfda4f8e96a229dd5fde81cbd Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 26 Mar 2026 13:33:41 -0700 Subject: [PATCH 52/66] lib/crypto: sparc: Drop optimized MD5 code MD5 is obsolete. Continuing to maintain architecture-optimized implementations of MD5 is unnecessary and risky. It diverts resources from the modern algorithms that are actually important. While there was demand for continuing to maintain the PowerPC optimized MD5 code to accommodate userspace programs that are misusing AF_ALG (https://lore.kernel.org/linux-crypto/c4191597-341d-4fd7-bc3d-13daf7666c41@csgroup.eu/), no such demand has been seen for the SPARC optimized MD5 code. Thus, let's drop it and focus effort on the more modern SHA algorithms, which already have optimized code for SPARC. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260326203341.60393-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/Kconfig | 1 - lib/crypto/Makefile | 1 - lib/crypto/sparc/md5.h | 48 -------------------------- lib/crypto/sparc/md5_asm.S | 70 -------------------------------------- 4 files changed, 120 deletions(-) delete mode 100644 lib/crypto/sparc/md5.h delete mode 100644 lib/crypto/sparc/md5_asm.S diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 9f31f03062f0..d3904b72dae7 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -135,7 +135,6 @@ config CRYPTO_LIB_MD5_ARCH bool depends on CRYPTO_LIB_MD5 && !UML default y if PPC - default y if SPARC64 config CRYPTO_LIB_MLDSA tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index ec1747f51d07..4b47a2e5c67c 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -188,7 +188,6 @@ libmd5-y := md5.o ifeq ($(CONFIG_CRYPTO_LIB_MD5_ARCH),y) CFLAGS_md5.o += -I$(src)/$(SRCARCH) libmd5-$(CONFIG_PPC) += powerpc/md5-asm.o -libmd5-$(CONFIG_SPARC) += sparc/md5_asm.o endif # CONFIG_CRYPTO_LIB_MD5_ARCH ################################################################################ diff --git a/lib/crypto/sparc/md5.h b/lib/crypto/sparc/md5.h deleted file mode 100644 index 3995f3e075eb..000000000000 --- a/lib/crypto/sparc/md5.h +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * MD5 accelerated using the sparc64 crypto opcodes - * - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald - * Copyright (c) Jean-Francois Dive - * Copyright (c) Mathias Krause - * Copyright (c) Cryptoapi developers. - * Copyright (c) 2002 James Morris - */ - -#include -#include -#include - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_md5_opcodes); - -asmlinkage void md5_sparc64_transform(struct md5_block_state *state, - const u8 *data, size_t nblocks); - -static void md5_blocks(struct md5_block_state *state, - const u8 *data, size_t nblocks) -{ - if (static_branch_likely(&have_md5_opcodes)) { - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - md5_sparc64_transform(state, data, nblocks); - le32_to_cpu_array(state->h, ARRAY_SIZE(state->h)); - } else { - md5_blocks_generic(state, data, nblocks); - } -} - -#define md5_mod_init_arch md5_mod_init_arch -static void md5_mod_init_arch(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_MD5)) - return; - - static_branch_enable(&have_md5_opcodes); - pr_info("Using sparc64 md5 opcode optimized MD5 implementation\n"); -} diff --git a/lib/crypto/sparc/md5_asm.S b/lib/crypto/sparc/md5_asm.S deleted file mode 100644 index 60b544e4d205..000000000000 --- a/lib/crypto/sparc/md5_asm.S +++ /dev/null @@ -1,70 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include -#include -#include - -ENTRY(md5_sparc64_transform) - /* %o0 = digest, %o1 = data, %o2 = rounds */ - VISEntryHalf - ld [%o0 + 0x00], %f0 - ld [%o0 + 0x04], %f1 - andcc %o1, 0x7, %g0 - ld [%o0 + 0x08], %f2 - bne,pn %xcc, 10f - ld [%o0 + 0x0c], %f3 - -1: - ldd [%o1 + 0x00], %f8 - ldd [%o1 + 0x08], %f10 - ldd [%o1 + 0x10], %f12 - ldd [%o1 + 0x18], %f14 - ldd [%o1 + 0x20], %f16 - ldd [%o1 + 0x28], %f18 - ldd [%o1 + 0x30], %f20 - ldd [%o1 + 0x38], %f22 - - MD5 - - subcc %o2, 1, %o2 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - -5: - st %f0, [%o0 + 0x00] - st %f1, [%o0 + 0x04] - st %f2, [%o0 + 0x08] - st %f3, [%o0 + 0x0c] - retl - VISExitHalf -10: - alignaddr %o1, %g0, %o1 - - ldd [%o1 + 0x00], %f10 -1: - ldd [%o1 + 0x08], %f12 - ldd [%o1 + 0x10], %f14 - ldd [%o1 + 0x18], %f16 - ldd [%o1 + 0x20], %f18 - ldd [%o1 + 0x28], %f20 - ldd [%o1 + 0x30], %f22 - ldd [%o1 + 0x38], %f24 - ldd [%o1 + 0x40], %f26 - - faligndata %f10, %f12, %f8 - faligndata %f12, %f14, %f10 - faligndata %f14, %f16, %f12 - faligndata %f16, %f18, %f14 - faligndata %f18, %f20, %f16 - faligndata %f20, %f22, %f18 - faligndata %f22, %f24, %f20 - faligndata %f24, %f26, %f22 - - MD5 - - subcc %o2, 1, %o2 - fsrc2 %f26, %f10 - bne,pt %xcc, 1b - add %o1, 0x40, %o1 - - ba,a,pt %xcc, 5b -ENDPROC(md5_sparc64_transform) From d2a68aba8505ce88b39c34ecb3b707c776af79d4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 27 Mar 2026 15:42:29 -0700 Subject: [PATCH 53/66] lib/crypto: tests: Migrate ChaCha20Poly1305 self-test to KUnit Move the ChaCha20Poly1305 test from an ad-hoc self-test to a KUnit test. Keep the same test logic for now, just translated to KUnit. Moving to KUnit has multiple benefits, such as: - Consistency with the rest of the lib/crypto/ tests. - Kernel developers familiar with KUnit, which is used kernel-wide, can quickly understand the test and how to enable and run it. - The test will be automatically run by anyone using lib/crypto/.kunitconfig or KUnit's all_tests.config. - Results are reported using the standard KUnit mechanism. - It eliminates one of the few remaining back-references to crypto/ from lib/crypto/, specifically a reference to CONFIG_CRYPTO_SELFTESTS. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260327224229.137532-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- include/crypto/chacha20poly1305.h | 2 - lib/crypto/.kunitconfig | 1 + lib/crypto/Makefile | 1 - lib/crypto/chacha20poly1305.c | 14 - lib/crypto/tests/Kconfig | 10 + lib/crypto/tests/Makefile | 1 + .../chacha20poly1305_kunit.c} | 1493 +++++++++-------- 7 files changed, 760 insertions(+), 762 deletions(-) rename lib/crypto/{chacha20poly1305-selftest.c => tests/chacha20poly1305_kunit.c} (91%) diff --git a/include/crypto/chacha20poly1305.h b/include/crypto/chacha20poly1305.h index 0f71b037702d..0f6d99170aaf 100644 --- a/include/crypto/chacha20poly1305.h +++ b/include/crypto/chacha20poly1305.h @@ -46,6 +46,4 @@ bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len const u64 nonce, const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]); -bool chacha20poly1305_selftest(void); - #endif /* __CHACHA20POLY1305_H */ diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig index f8a3c6e6367c..3efc854a2c08 100644 --- a/lib/crypto/.kunitconfig +++ b/lib/crypto/.kunitconfig @@ -5,6 +5,7 @@ CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y +CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST=y CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST=y CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST=y CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST=y diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 4b47a2e5c67c..477aec03c651 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -122,7 +122,6 @@ endif # CONFIG_CRYPTO_LIB_CHACHA_ARCH obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o libchacha20poly1305-y += chacha20poly1305.o -libchacha20poly1305-$(CONFIG_CRYPTO_SELFTESTS) += chacha20poly1305-selftest.o ################################################################################ diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index 212ce33562af..ea42a28f4ff7 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -356,20 +356,6 @@ bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len } EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); -static int __init chacha20poly1305_init(void) -{ - if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) && - WARN_ON(!chacha20poly1305_selftest())) - return -ENODEV; - return 0; -} - -static void __exit chacha20poly1305_exit(void) -{ -} - -module_init(chacha20poly1305_init); -module_exit(chacha20poly1305_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); MODULE_AUTHOR("Jason A. Donenfeld "); diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 7a5ad109aefc..9409c1a935c3 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -27,6 +27,15 @@ config CRYPTO_LIB_BLAKE2S_KUNIT_TEST help KUnit tests for the BLAKE2s cryptographic hash function. +config CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST + tristate "KUnit tests for ChaCha20Poly1305" if !KUNIT_ALL_TESTS + depends on KUNIT && CRYPTO_LIB_CHACHA20POLY1305 + default KUNIT_ALL_TESTS + select CRYPTO_LIB_BENCHMARK_VISIBLE + help + KUnit tests for the ChaCha20Poly1305 authenticated encryption + algorithm. + config CRYPTO_LIB_CURVE25519_KUNIT_TEST tristate "KUnit tests for Curve25519" if !KUNIT_ALL_TESTS depends on KUNIT && CRYPTO_LIB_CURVE25519 @@ -137,6 +146,7 @@ config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT depends on KUNIT select CRYPTO_LIB_AES_CBC_MACS select CRYPTO_LIB_BLAKE2B + select CRYPTO_LIB_CHACHA20POLY1305 select CRYPTO_LIB_CURVE25519 select CRYPTO_LIB_GF128HASH select CRYPTO_LIB_MD5 diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile index ad1cbb88132f..a739413500b6 100644 --- a/lib/crypto/tests/Makefile +++ b/lib/crypto/tests/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST) += aes_cbc_macs_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o +obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST) += chacha20poly1305_kunit.o obj-$(CONFIG_CRYPTO_LIB_CURVE25519_KUNIT_TEST) += curve25519_kunit.o obj-$(CONFIG_CRYPTO_LIB_GHASH_KUNIT_TEST) += ghash_kunit.o obj-$(CONFIG_CRYPTO_LIB_MD5_KUNIT_TEST) += md5_kunit.o diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/tests/chacha20poly1305_kunit.c similarity index 91% rename from lib/crypto/chacha20poly1305-selftest.c rename to lib/crypto/tests/chacha20poly1305_kunit.c index e4c85bc5a6d7..97a68fab88a7 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/tests/chacha20poly1305_kunit.c @@ -6,9 +6,8 @@ #include #include #include - +#include #include -#include #include #include #include @@ -27,7 +26,7 @@ struct chacha20poly1305_testvec { * chapoly construction. */ -static const u8 enc_input001[] __initconst = { +static const u8 enc_input001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -63,7 +62,7 @@ static const u8 enc_input001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 enc_output001[] __initconst = { +static const u8 enc_output001[] = { 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, @@ -101,95 +100,95 @@ static const u8 enc_output001[] __initconst = { 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, 0x38 }; -static const u8 enc_assoc001[] __initconst = { +static const u8 enc_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 enc_nonce001[] __initconst = { +static const u8 enc_nonce001[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 }; -static const u8 enc_key001[] __initconst = { +static const u8 enc_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; -static const u8 enc_input002[] __initconst = { }; -static const u8 enc_output002[] __initconst = { +static const u8 enc_input002[] = { }; +static const u8 enc_output002[] = { 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 }; -static const u8 enc_assoc002[] __initconst = { }; -static const u8 enc_nonce002[] __initconst = { +static const u8 enc_assoc002[] = { }; +static const u8 enc_nonce002[] = { 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e }; -static const u8 enc_key002[] __initconst = { +static const u8 enc_key002[] = { 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 }; -static const u8 enc_input003[] __initconst = { }; -static const u8 enc_output003[] __initconst = { +static const u8 enc_input003[] = { }; +static const u8 enc_output003[] = { 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 }; -static const u8 enc_assoc003[] __initconst = { +static const u8 enc_assoc003[] = { 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b }; -static const u8 enc_nonce003[] __initconst = { +static const u8 enc_nonce003[] = { 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d }; -static const u8 enc_key003[] __initconst = { +static const u8 enc_key003[] = { 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d }; -static const u8 enc_input004[] __initconst = { +static const u8 enc_input004[] = { 0xa4 }; -static const u8 enc_output004[] __initconst = { +static const u8 enc_output004[] = { 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, 0x89 }; -static const u8 enc_assoc004[] __initconst = { +static const u8 enc_assoc004[] = { 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 }; -static const u8 enc_nonce004[] __initconst = { +static const u8 enc_nonce004[] = { 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 }; -static const u8 enc_key004[] __initconst = { +static const u8 enc_key004[] = { 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e }; -static const u8 enc_input005[] __initconst = { +static const u8 enc_input005[] = { 0x2d }; -static const u8 enc_output005[] __initconst = { +static const u8 enc_output005[] = { 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, 0xac }; -static const u8 enc_assoc005[] __initconst = { }; -static const u8 enc_nonce005[] __initconst = { +static const u8 enc_assoc005[] = { }; +static const u8 enc_nonce005[] = { 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 }; -static const u8 enc_key005[] __initconst = { +static const u8 enc_key005[] = { 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 }; -static const u8 enc_input006[] __initconst = { +static const u8 enc_input006[] = { 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, @@ -208,7 +207,7 @@ static const u8 enc_input006[] __initconst = { 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, 0x8f }; -static const u8 enc_output006[] __initconst = { +static const u8 enc_output006[] = { 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, @@ -229,20 +228,20 @@ static const u8 enc_output006[] __initconst = { 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, 0xeb }; -static const u8 enc_assoc006[] __initconst = { +static const u8 enc_assoc006[] = { 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b }; -static const u8 enc_nonce006[] __initconst = { +static const u8 enc_nonce006[] = { 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c }; -static const u8 enc_key006[] __initconst = { +static const u8 enc_key006[] = { 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 }; -static const u8 enc_input007[] __initconst = { +static const u8 enc_input007[] = { 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, @@ -276,7 +275,7 @@ static const u8 enc_input007[] __initconst = { 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 }; -static const u8 enc_output007[] __initconst = { +static const u8 enc_output007[] = { 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, @@ -312,18 +311,18 @@ static const u8 enc_output007[] __initconst = { 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 }; -static const u8 enc_assoc007[] __initconst = { }; -static const u8 enc_nonce007[] __initconst = { +static const u8 enc_assoc007[] = { }; +static const u8 enc_nonce007[] = { 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 }; -static const u8 enc_key007[] __initconst = { +static const u8 enc_key007[] = { 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 }; -static const u8 enc_input008[] __initconst = { +static const u8 enc_input008[] = { 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, @@ -389,7 +388,7 @@ static const u8 enc_input008[] __initconst = { 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 }; -static const u8 enc_output008[] __initconst = { +static const u8 enc_output008[] = { 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, @@ -457,18 +456,18 @@ static const u8 enc_output008[] __initconst = { 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 }; -static const u8 enc_assoc008[] __initconst = { }; -static const u8 enc_nonce008[] __initconst = { +static const u8 enc_assoc008[] = { }; +static const u8 enc_nonce008[] = { 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 }; -static const u8 enc_key008[] __initconst = { +static const u8 enc_key008[] = { 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba }; -static const u8 enc_input009[] __initconst = { +static const u8 enc_input009[] = { 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, @@ -535,7 +534,7 @@ static const u8 enc_input009[] __initconst = { 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, 0x65 }; -static const u8 enc_output009[] __initconst = { +static const u8 enc_output009[] = { 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, @@ -604,21 +603,21 @@ static const u8 enc_output009[] __initconst = { 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, 0xae }; -static const u8 enc_assoc009[] __initconst = { +static const u8 enc_assoc009[] = { 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, 0xef }; -static const u8 enc_nonce009[] __initconst = { +static const u8 enc_nonce009[] = { 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 }; -static const u8 enc_key009[] __initconst = { +static const u8 enc_key009[] = { 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b }; -static const u8 enc_input010[] __initconst = { +static const u8 enc_input010[] = { 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, @@ -748,7 +747,7 @@ static const u8 enc_input010[] __initconst = { 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f }; -static const u8 enc_output010[] __initconst = { +static const u8 enc_output010[] = { 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, @@ -880,21 +879,21 @@ static const u8 enc_output010[] __initconst = { 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 }; -static const u8 enc_assoc010[] __initconst = { +static const u8 enc_assoc010[] = { 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 }; -static const u8 enc_nonce010[] __initconst = { +static const u8 enc_nonce010[] = { 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 }; -static const u8 enc_key010[] __initconst = { +static const u8 enc_key010[] = { 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 }; -static const u8 enc_input011[] __initconst = { +static const u8 enc_input011[] = { 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, @@ -1138,7 +1137,7 @@ static const u8 enc_input011[] __initconst = { 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, 0x10, 0x1e, 0xbf, 0xec, 0xa8 }; -static const u8 enc_output011[] __initconst = { +static const u8 enc_output011[] = { 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, @@ -1384,20 +1383,20 @@ static const u8 enc_output011[] __initconst = { 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, 0x2b, 0xdf, 0xcd, 0xf9, 0x3c }; -static const u8 enc_assoc011[] __initconst = { +static const u8 enc_assoc011[] = { 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 }; -static const u8 enc_nonce011[] __initconst = { +static const u8 enc_nonce011[] = { 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa }; -static const u8 enc_key011[] __initconst = { +static const u8 enc_key011[] = { 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 }; -static const u8 enc_input012[] __initconst = { +static const u8 enc_input012[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -1651,7 +1650,7 @@ static const u8 enc_input012[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 enc_output012[] __initconst = { +static const u8 enc_output012[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -1907,7 +1906,7 @@ static const u8 enc_output012[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd6 }; -static const u8 enc_assoc012[] __initconst = { +static const u8 enc_assoc012[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -1917,10 +1916,10 @@ static const u8 enc_assoc012[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 enc_nonce012[] __initconst = { +static const u8 enc_nonce012[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 enc_key012[] __initconst = { +static const u8 enc_key012[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, @@ -1928,7 +1927,7 @@ static const u8 enc_key012[] __initconst = { }; /* wycheproof - rfc7539 */ -static const u8 enc_input013[] __initconst = { +static const u8 enc_input013[] = { 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, @@ -1945,7 +1944,7 @@ static const u8 enc_input013[] __initconst = { 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, 0x74, 0x2e }; -static const u8 enc_output013[] __initconst = { +static const u8 enc_output013[] = { 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, @@ -1964,15 +1963,15 @@ static const u8 enc_output013[] __initconst = { 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, 0x06, 0x91 }; -static const u8 enc_assoc013[] __initconst = { +static const u8 enc_assoc013[] = { 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7 }; -static const u8 enc_nonce013[] __initconst = { +static const u8 enc_nonce013[] = { 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47 }; -static const u8 enc_key013[] __initconst = { +static const u8 enc_key013[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -1980,17 +1979,17 @@ static const u8 enc_key013[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input014[] __initconst = { }; -static const u8 enc_output014[] __initconst = { +static const u8 enc_input014[] = { }; +static const u8 enc_output014[] = { 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d }; -static const u8 enc_assoc014[] __initconst = { }; -static const u8 enc_nonce014[] __initconst = { +static const u8 enc_assoc014[] = { }; +static const u8 enc_nonce014[] = { 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, 0xea, 0x12, 0x37, 0x9d }; -static const u8 enc_key014[] __initconst = { +static const u8 enc_key014[] = { 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, @@ -1998,19 +1997,19 @@ static const u8 enc_key014[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input015[] __initconst = { }; -static const u8 enc_output015[] __initconst = { +static const u8 enc_input015[] = { }; +static const u8 enc_output015[] = { 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 }; -static const u8 enc_assoc015[] __initconst = { +static const u8 enc_assoc015[] = { 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 }; -static const u8 enc_nonce015[] __initconst = { +static const u8 enc_nonce015[] = { 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, 0xa3, 0xc6, 0xf6, 0x89 }; -static const u8 enc_key015[] __initconst = { +static const u8 enc_key015[] = { 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, @@ -2018,20 +2017,20 @@ static const u8 enc_key015[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input016[] __initconst = { +static const u8 enc_input016[] = { 0x2a }; -static const u8 enc_output016[] __initconst = { +static const u8 enc_output016[] = { 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, 0x22 }; -static const u8 enc_assoc016[] __initconst = { }; -static const u8 enc_nonce016[] __initconst = { +static const u8 enc_assoc016[] = { }; +static const u8 enc_nonce016[] = { 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, 0xee, 0xab, 0x60, 0xf1 }; -static const u8 enc_key016[] __initconst = { +static const u8 enc_key016[] = { 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, @@ -2039,22 +2038,22 @@ static const u8 enc_key016[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input017[] __initconst = { +static const u8 enc_input017[] = { 0x51 }; -static const u8 enc_output017[] __initconst = { +static const u8 enc_output017[] = { 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, 0xb9 }; -static const u8 enc_assoc017[] __initconst = { +static const u8 enc_assoc017[] = { 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 }; -static const u8 enc_nonce017[] __initconst = { +static const u8 enc_nonce017[] = { 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, 0x78, 0x2f, 0x44, 0x03 }; -static const u8 enc_key017[] __initconst = { +static const u8 enc_key017[] = { 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, @@ -2062,20 +2061,20 @@ static const u8 enc_key017[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input018[] __initconst = { +static const u8 enc_input018[] = { 0x5c, 0x60 }; -static const u8 enc_output018[] __initconst = { +static const u8 enc_output018[] = { 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, 0xe5, 0xe2 }; -static const u8 enc_assoc018[] __initconst = { }; -static const u8 enc_nonce018[] __initconst = { +static const u8 enc_assoc018[] = { }; +static const u8 enc_nonce018[] = { 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, 0x7e, 0x03, 0xf2, 0xdb }; -static const u8 enc_key018[] __initconst = { +static const u8 enc_key018[] = { 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, @@ -2083,22 +2082,22 @@ static const u8 enc_key018[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input019[] __initconst = { +static const u8 enc_input019[] = { 0xdd, 0xf2 }; -static const u8 enc_output019[] __initconst = { +static const u8 enc_output019[] = { 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, 0x37, 0xa2 }; -static const u8 enc_assoc019[] __initconst = { +static const u8 enc_assoc019[] = { 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf }; -static const u8 enc_nonce019[] __initconst = { +static const u8 enc_nonce019[] = { 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, 0xb6, 0x04, 0x0a, 0xc6 }; -static const u8 enc_key019[] __initconst = { +static const u8 enc_key019[] = { 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, @@ -2106,20 +2105,20 @@ static const u8 enc_key019[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input020[] __initconst = { +static const u8 enc_input020[] = { 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 }; -static const u8 enc_output020[] __initconst = { +static const u8 enc_output020[] = { 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 }; -static const u8 enc_assoc020[] __initconst = { }; -static const u8 enc_nonce020[] __initconst = { +static const u8 enc_assoc020[] = { }; +static const u8 enc_nonce020[] = { 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, 0x6d, 0xc5, 0x5b, 0xb7 }; -static const u8 enc_key020[] __initconst = { +static const u8 enc_key020[] = { 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, @@ -2127,22 +2126,22 @@ static const u8 enc_key020[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input021[] __initconst = { +static const u8 enc_input021[] = { 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 }; -static const u8 enc_output021[] __initconst = { +static const u8 enc_output021[] = { 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba }; -static const u8 enc_assoc021[] __initconst = { +static const u8 enc_assoc021[] = { 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 }; -static const u8 enc_nonce021[] __initconst = { +static const u8 enc_nonce021[] = { 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, 0xde, 0x94, 0x83, 0x96 }; -static const u8 enc_key021[] __initconst = { +static const u8 enc_key021[] = { 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, @@ -2150,20 +2149,20 @@ static const u8 enc_key021[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input022[] __initconst = { +static const u8 enc_input022[] = { 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed }; -static const u8 enc_output022[] __initconst = { +static const u8 enc_output022[] = { 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 }; -static const u8 enc_assoc022[] __initconst = { }; -static const u8 enc_nonce022[] __initconst = { +static const u8 enc_assoc022[] = { }; +static const u8 enc_nonce022[] = { 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, 0x36, 0x18, 0x23, 0xd3 }; -static const u8 enc_key022[] __initconst = { +static const u8 enc_key022[] = { 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, @@ -2171,22 +2170,22 @@ static const u8 enc_key022[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input023[] __initconst = { +static const u8 enc_input023[] = { 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf }; -static const u8 enc_output023[] __initconst = { +static const u8 enc_output023[] = { 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 }; -static const u8 enc_assoc023[] __initconst = { +static const u8 enc_assoc023[] = { 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d }; -static const u8 enc_nonce023[] __initconst = { +static const u8 enc_nonce023[] = { 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, 0x66, 0x48, 0x4a, 0x78 }; -static const u8 enc_key023[] __initconst = { +static const u8 enc_key023[] = { 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, @@ -2194,22 +2193,22 @@ static const u8 enc_key023[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input024[] __initconst = { +static const u8 enc_input024[] = { 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, 0x36, 0x8d, 0x14, 0xe0 }; -static const u8 enc_output024[] __initconst = { +static const u8 enc_output024[] = { 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, 0x6d, 0xcb, 0x29, 0xb4 }; -static const u8 enc_assoc024[] __initconst = { }; -static const u8 enc_nonce024[] __initconst = { +static const u8 enc_assoc024[] = { }; +static const u8 enc_nonce024[] = { 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, 0x07, 0x53, 0x86, 0x56 }; -static const u8 enc_key024[] __initconst = { +static const u8 enc_key024[] = { 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, @@ -2217,24 +2216,24 @@ static const u8 enc_key024[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input025[] __initconst = { +static const u8 enc_input025[] = { 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, 0x59, 0xb1, 0xa0, 0xda }; -static const u8 enc_output025[] __initconst = { +static const u8 enc_output025[] = { 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, 0xcf, 0x05, 0x07, 0x2f }; -static const u8 enc_assoc025[] __initconst = { +static const u8 enc_assoc025[] = { 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 }; -static const u8 enc_nonce025[] __initconst = { +static const u8 enc_nonce025[] = { 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, 0xd9, 0x06, 0xe9, 0xce }; -static const u8 enc_key025[] __initconst = { +static const u8 enc_key025[] = { 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, @@ -2242,22 +2241,22 @@ static const u8 enc_key025[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input026[] __initconst = { +static const u8 enc_input026[] = { 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 }; -static const u8 enc_output026[] __initconst = { +static const u8 enc_output026[] = { 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 }; -static const u8 enc_assoc026[] __initconst = { }; -static const u8 enc_nonce026[] __initconst = { +static const u8 enc_assoc026[] = { }; +static const u8 enc_nonce026[] = { 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, 0xd4, 0x61, 0xd2, 0x3c }; -static const u8 enc_key026[] __initconst = { +static const u8 enc_key026[] = { 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, @@ -2265,24 +2264,24 @@ static const u8 enc_key026[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input027[] __initconst = { +static const u8 enc_input027[] = { 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 }; -static const u8 enc_output027[] __initconst = { +static const u8 enc_output027[] = { 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 }; -static const u8 enc_assoc027[] __initconst = { +static const u8 enc_assoc027[] = { 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 }; -static const u8 enc_nonce027[] __initconst = { +static const u8 enc_nonce027[] = { 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, 0xc5, 0x03, 0x5b, 0x6a }; -static const u8 enc_key027[] __initconst = { +static const u8 enc_key027[] = { 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, @@ -2290,22 +2289,22 @@ static const u8 enc_key027[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input028[] __initconst = { +static const u8 enc_input028[] = { 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 }; -static const u8 enc_output028[] __initconst = { +static const u8 enc_output028[] = { 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c }; -static const u8 enc_assoc028[] __initconst = { }; -static const u8 enc_nonce028[] __initconst = { +static const u8 enc_assoc028[] = { }; +static const u8 enc_nonce028[] = { 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, 0x47, 0x40, 0xad, 0x9b }; -static const u8 enc_key028[] __initconst = { +static const u8 enc_key028[] = { 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, @@ -2313,24 +2312,24 @@ static const u8 enc_key028[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input029[] __initconst = { +static const u8 enc_input029[] = { 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 }; -static const u8 enc_output029[] __initconst = { +static const u8 enc_output029[] = { 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d }; -static const u8 enc_assoc029[] __initconst = { +static const u8 enc_assoc029[] = { 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff }; -static const u8 enc_nonce029[] __initconst = { +static const u8 enc_nonce029[] = { 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, 0x07, 0x1f, 0x52, 0x66 }; -static const u8 enc_key029[] __initconst = { +static const u8 enc_key029[] = { 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, @@ -2338,24 +2337,24 @@ static const u8 enc_key029[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input030[] __initconst = { +static const u8 enc_input030[] = { 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, 0x1f }; -static const u8 enc_output030[] __initconst = { +static const u8 enc_output030[] = { 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, 0x13 }; -static const u8 enc_assoc030[] __initconst = { }; -static const u8 enc_nonce030[] __initconst = { +static const u8 enc_assoc030[] = { }; +static const u8 enc_nonce030[] = { 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, 0xdc, 0x03, 0x44, 0x5d }; -static const u8 enc_key030[] __initconst = { +static const u8 enc_key030[] = { 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, @@ -2363,26 +2362,26 @@ static const u8 enc_key030[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input031[] __initconst = { +static const u8 enc_input031[] = { 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, 0x9c }; -static const u8 enc_output031[] __initconst = { +static const u8 enc_output031[] = { 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, 0xeb }; -static const u8 enc_assoc031[] __initconst = { +static const u8 enc_assoc031[] = { 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 }; -static const u8 enc_nonce031[] __initconst = { +static const u8 enc_nonce031[] = { 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, 0x63, 0x3d, 0x99, 0x3d }; -static const u8 enc_key031[] __initconst = { +static const u8 enc_key031[] = { 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, @@ -2390,24 +2389,24 @@ static const u8 enc_key031[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input032[] __initconst = { +static const u8 enc_input032[] = { 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 }; -static const u8 enc_output032[] __initconst = { +static const u8 enc_output032[] = { 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a }; -static const u8 enc_assoc032[] __initconst = { }; -static const u8 enc_nonce032[] __initconst = { +static const u8 enc_assoc032[] = { }; +static const u8 enc_nonce032[] = { 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, 0x2c, 0x65, 0xf3, 0x1b }; -static const u8 enc_key032[] __initconst = { +static const u8 enc_key032[] = { 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, @@ -2415,26 +2414,26 @@ static const u8 enc_key032[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input033[] __initconst = { +static const u8 enc_input033[] = { 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 }; -static const u8 enc_output033[] __initconst = { +static const u8 enc_output033[] = { 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 }; -static const u8 enc_assoc033[] __initconst = { +static const u8 enc_assoc033[] = { 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 }; -static const u8 enc_nonce033[] __initconst = { +static const u8 enc_nonce033[] = { 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, 0x78, 0x34, 0xed, 0x55 }; -static const u8 enc_key033[] __initconst = { +static const u8 enc_key033[] = { 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, @@ -2442,13 +2441,13 @@ static const u8 enc_key033[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input034[] __initconst = { +static const u8 enc_input034[] = { 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 }; -static const u8 enc_output034[] __initconst = { +static const u8 enc_output034[] = { 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, @@ -2456,12 +2455,12 @@ static const u8 enc_output034[] __initconst = { 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 }; -static const u8 enc_assoc034[] __initconst = { }; -static const u8 enc_nonce034[] __initconst = { +static const u8 enc_assoc034[] = { }; +static const u8 enc_nonce034[] = { 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, 0x37, 0x1a, 0x6f, 0xd2 }; -static const u8 enc_key034[] __initconst = { +static const u8 enc_key034[] = { 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, @@ -2469,13 +2468,13 @@ static const u8 enc_key034[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input035[] __initconst = { +static const u8 enc_input035[] = { 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb }; -static const u8 enc_output035[] __initconst = { +static const u8 enc_output035[] = { 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, @@ -2483,14 +2482,14 @@ static const u8 enc_output035[] __initconst = { 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f }; -static const u8 enc_assoc035[] __initconst = { +static const u8 enc_assoc035[] = { 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 }; -static const u8 enc_nonce035[] __initconst = { +static const u8 enc_nonce035[] = { 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, 0xb8, 0x1a, 0x1f, 0x8b }; -static const u8 enc_key035[] __initconst = { +static const u8 enc_key035[] = { 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, @@ -2498,13 +2497,13 @@ static const u8 enc_key035[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input036[] __initconst = { +static const u8 enc_input036[] = { 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 }; -static const u8 enc_output036[] __initconst = { +static const u8 enc_output036[] = { 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, @@ -2512,12 +2511,12 @@ static const u8 enc_output036[] __initconst = { 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea }; -static const u8 enc_assoc036[] __initconst = { }; -static const u8 enc_nonce036[] __initconst = { +static const u8 enc_assoc036[] = { }; +static const u8 enc_nonce036[] = { 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, 0x2b, 0x7e, 0x6e, 0x6a }; -static const u8 enc_key036[] __initconst = { +static const u8 enc_key036[] = { 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, @@ -2525,13 +2524,13 @@ static const u8 enc_key036[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input037[] __initconst = { +static const u8 enc_input037[] = { 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e }; -static const u8 enc_output037[] __initconst = { +static const u8 enc_output037[] = { 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, @@ -2539,14 +2538,14 @@ static const u8 enc_output037[] __initconst = { 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 }; -static const u8 enc_assoc037[] __initconst = { +static const u8 enc_assoc037[] = { 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 }; -static const u8 enc_nonce037[] __initconst = { +static const u8 enc_nonce037[] = { 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, 0x34, 0x0d, 0xd1, 0xb8 }; -static const u8 enc_key037[] __initconst = { +static const u8 enc_key037[] = { 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, @@ -2554,7 +2553,7 @@ static const u8 enc_key037[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input038[] __initconst = { +static const u8 enc_input038[] = { 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, @@ -2562,7 +2561,7 @@ static const u8 enc_input038[] __initconst = { 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a }; -static const u8 enc_output038[] __initconst = { +static const u8 enc_output038[] = { 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, @@ -2572,12 +2571,12 @@ static const u8 enc_output038[] __initconst = { 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 }; -static const u8 enc_assoc038[] __initconst = { }; -static const u8 enc_nonce038[] __initconst = { +static const u8 enc_assoc038[] = { }; +static const u8 enc_nonce038[] = { 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, 0x0b, 0xef, 0x55, 0xd2 }; -static const u8 enc_key038[] __initconst = { +static const u8 enc_key038[] = { 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, @@ -2585,7 +2584,7 @@ static const u8 enc_key038[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input039[] __initconst = { +static const u8 enc_input039[] = { 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, @@ -2593,7 +2592,7 @@ static const u8 enc_input039[] __initconst = { 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 }; -static const u8 enc_output039[] __initconst = { +static const u8 enc_output039[] = { 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, @@ -2603,14 +2602,14 @@ static const u8 enc_output039[] __initconst = { 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 }; -static const u8 enc_assoc039[] __initconst = { +static const u8 enc_assoc039[] = { 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 }; -static const u8 enc_nonce039[] __initconst = { +static const u8 enc_nonce039[] = { 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, 0x31, 0xcd, 0x4d, 0x95 }; -static const u8 enc_key039[] __initconst = { +static const u8 enc_key039[] = { 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, @@ -2618,7 +2617,7 @@ static const u8 enc_key039[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input040[] __initconst = { +static const u8 enc_input040[] = { 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, @@ -2628,7 +2627,7 @@ static const u8 enc_input040[] __initconst = { 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 }; -static const u8 enc_output040[] __initconst = { +static const u8 enc_output040[] = { 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, @@ -2640,12 +2639,12 @@ static const u8 enc_output040[] __initconst = { 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 }; -static const u8 enc_assoc040[] __initconst = { }; -static const u8 enc_nonce040[] __initconst = { +static const u8 enc_assoc040[] = { }; +static const u8 enc_nonce040[] = { 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, 0x23, 0xcc, 0x06, 0x5b }; -static const u8 enc_key040[] __initconst = { +static const u8 enc_key040[] = { 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, @@ -2653,7 +2652,7 @@ static const u8 enc_key040[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input041[] __initconst = { +static const u8 enc_input041[] = { 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, @@ -2663,7 +2662,7 @@ static const u8 enc_input041[] __initconst = { 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 }; -static const u8 enc_output041[] __initconst = { +static const u8 enc_output041[] = { 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, @@ -2675,14 +2674,14 @@ static const u8 enc_output041[] __initconst = { 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec }; -static const u8 enc_assoc041[] __initconst = { +static const u8 enc_assoc041[] = { 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 }; -static const u8 enc_nonce041[] __initconst = { +static const u8 enc_nonce041[] = { 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, 0xe4, 0xeb, 0xb1, 0x9c }; -static const u8 enc_key041[] __initconst = { +static const u8 enc_key041[] = { 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, @@ -2690,7 +2689,7 @@ static const u8 enc_key041[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input042[] __initconst = { +static const u8 enc_input042[] = { 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, @@ -2705,7 +2704,7 @@ static const u8 enc_input042[] __initconst = { 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, 0x92 }; -static const u8 enc_output042[] __initconst = { +static const u8 enc_output042[] = { 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, @@ -2722,12 +2721,12 @@ static const u8 enc_output042[] __initconst = { 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, 0xb0 }; -static const u8 enc_assoc042[] __initconst = { }; -static const u8 enc_nonce042[] __initconst = { +static const u8 enc_assoc042[] = { }; +static const u8 enc_nonce042[] = { 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, 0xac, 0xde, 0x27, 0x99 }; -static const u8 enc_key042[] __initconst = { +static const u8 enc_key042[] = { 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, @@ -2735,7 +2734,7 @@ static const u8 enc_key042[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input043[] __initconst = { +static const u8 enc_input043[] = { 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, @@ -2750,7 +2749,7 @@ static const u8 enc_input043[] __initconst = { 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, 0x76 }; -static const u8 enc_output043[] __initconst = { +static const u8 enc_output043[] = { 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, @@ -2767,14 +2766,14 @@ static const u8 enc_output043[] __initconst = { 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, 0xf6 }; -static const u8 enc_assoc043[] __initconst = { +static const u8 enc_assoc043[] = { 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 }; -static const u8 enc_nonce043[] __initconst = { +static const u8 enc_nonce043[] = { 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, 0xe5, 0x22, 0x94, 0x60 }; -static const u8 enc_key043[] __initconst = { +static const u8 enc_key043[] = { 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, @@ -2782,24 +2781,24 @@ static const u8 enc_key043[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input044[] __initconst = { +static const u8 enc_input044[] = { 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 }; -static const u8 enc_output044[] __initconst = { +static const u8 enc_output044[] = { 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 }; -static const u8 enc_assoc044[] __initconst = { +static const u8 enc_assoc044[] = { 0x02 }; -static const u8 enc_nonce044[] __initconst = { +static const u8 enc_nonce044[] = { 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, 0x02, 0xd5, 0x06, 0x56 }; -static const u8 enc_key044[] __initconst = { +static const u8 enc_key044[] = { 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, @@ -2807,24 +2806,24 @@ static const u8 enc_key044[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input045[] __initconst = { +static const u8 enc_input045[] = { 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 }; -static const u8 enc_output045[] __initconst = { +static const u8 enc_output045[] = { 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 }; -static const u8 enc_assoc045[] __initconst = { +static const u8 enc_assoc045[] = { 0xb6, 0x48 }; -static const u8 enc_nonce045[] __initconst = { +static const u8 enc_nonce045[] = { 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, 0x5b, 0x3a, 0xa7, 0x13 }; -static const u8 enc_key045[] __initconst = { +static const u8 enc_key045[] = { 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, @@ -2832,25 +2831,25 @@ static const u8 enc_key045[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input046[] __initconst = { +static const u8 enc_input046[] = { 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 }; -static const u8 enc_output046[] __initconst = { +static const u8 enc_output046[] = { 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 }; -static const u8 enc_assoc046[] __initconst = { +static const u8 enc_assoc046[] = { 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 }; -static const u8 enc_nonce046[] __initconst = { +static const u8 enc_nonce046[] = { 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, 0xa4, 0x65, 0x96, 0xdf }; -static const u8 enc_key046[] __initconst = { +static const u8 enc_key046[] = { 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, @@ -2858,25 +2857,25 @@ static const u8 enc_key046[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input047[] __initconst = { +static const u8 enc_input047[] = { 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 }; -static const u8 enc_output047[] __initconst = { +static const u8 enc_output047[] = { 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f }; -static const u8 enc_assoc047[] __initconst = { +static const u8 enc_assoc047[] = { 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b }; -static const u8 enc_nonce047[] __initconst = { +static const u8 enc_nonce047[] = { 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, 0x80, 0x92, 0x66, 0xd9 }; -static const u8 enc_key047[] __initconst = { +static const u8 enc_key047[] = { 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, @@ -2884,26 +2883,26 @@ static const u8 enc_key047[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input048[] __initconst = { +static const u8 enc_input048[] = { 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 }; -static const u8 enc_output048[] __initconst = { +static const u8 enc_output048[] = { 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 }; -static const u8 enc_assoc048[] __initconst = { +static const u8 enc_assoc048[] = { 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, 0x0e }; -static const u8 enc_nonce048[] __initconst = { +static const u8 enc_nonce048[] = { 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, 0xfc, 0x10, 0x68, 0xc3 }; -static const u8 enc_key048[] __initconst = { +static const u8 enc_key048[] = { 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, @@ -2911,27 +2910,27 @@ static const u8 enc_key048[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input049[] __initconst = { +static const u8 enc_input049[] = { 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 }; -static const u8 enc_output049[] __initconst = { +static const u8 enc_output049[] = { 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 }; -static const u8 enc_assoc049[] __initconst = { +static const u8 enc_assoc049[] = { 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda }; -static const u8 enc_nonce049[] __initconst = { +static const u8 enc_nonce049[] = { 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, 0xad, 0x14, 0xd5, 0x3e }; -static const u8 enc_key049[] __initconst = { +static const u8 enc_key049[] = { 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, @@ -2939,27 +2938,27 @@ static const u8 enc_key049[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input050[] __initconst = { +static const u8 enc_input050[] = { 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c }; -static const u8 enc_output050[] __initconst = { +static const u8 enc_output050[] = { 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 }; -static const u8 enc_assoc050[] __initconst = { +static const u8 enc_assoc050[] = { 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 }; -static const u8 enc_nonce050[] __initconst = { +static const u8 enc_nonce050[] = { 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, 0x6a, 0x7b, 0x76, 0x51 }; -static const u8 enc_key050[] __initconst = { +static const u8 enc_key050[] = { 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, @@ -2967,28 +2966,28 @@ static const u8 enc_key050[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input051[] __initconst = { +static const u8 enc_input051[] = { 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 }; -static const u8 enc_output051[] __initconst = { +static const u8 enc_output051[] = { 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 }; -static const u8 enc_assoc051[] __initconst = { +static const u8 enc_assoc051[] = { 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, 0x73 }; -static const u8 enc_nonce051[] __initconst = { +static const u8 enc_nonce051[] = { 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, 0xa8, 0xfa, 0x89, 0x49 }; -static const u8 enc_key051[] __initconst = { +static const u8 enc_key051[] = { 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, @@ -2996,17 +2995,17 @@ static const u8 enc_key051[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input052[] __initconst = { +static const u8 enc_input052[] = { 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed }; -static const u8 enc_output052[] __initconst = { +static const u8 enc_output052[] = { 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 }; -static const u8 enc_assoc052[] __initconst = { +static const u8 enc_assoc052[] = { 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, @@ -3014,11 +3013,11 @@ static const u8 enc_assoc052[] __initconst = { 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 }; -static const u8 enc_nonce052[] __initconst = { +static const u8 enc_nonce052[] = { 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, 0x0e, 0xc3, 0x5d, 0x12 }; -static const u8 enc_key052[] __initconst = { +static const u8 enc_key052[] = { 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, @@ -3026,13 +3025,13 @@ static const u8 enc_key052[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input053[] __initconst = { +static const u8 enc_input053[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe }; -static const u8 enc_output053[] __initconst = { +static const u8 enc_output053[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3040,14 +3039,14 @@ static const u8 enc_output053[] __initconst = { 0xe6, 0xd3, 0xd7, 0x32, 0x4a, 0x1c, 0xbb, 0xa7, 0x77, 0xbb, 0xb0, 0xec, 0xdd, 0xa3, 0x78, 0x07 }; -static const u8 enc_assoc053[] __initconst = { +static const u8 enc_assoc053[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce053[] __initconst = { +static const u8 enc_nonce053[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key053[] __initconst = { +static const u8 enc_key053[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3055,7 +3054,7 @@ static const u8 enc_key053[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input054[] __initconst = { +static const u8 enc_input054[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, @@ -3065,7 +3064,7 @@ static const u8 enc_input054[] __initconst = { 0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5, 0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd }; -static const u8 enc_output054[] __initconst = { +static const u8 enc_output054[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3077,14 +3076,14 @@ static const u8 enc_output054[] __initconst = { 0x06, 0x2d, 0xe6, 0x79, 0x5f, 0x27, 0x4f, 0xd2, 0xa3, 0x05, 0xd7, 0x69, 0x80, 0xbc, 0x9c, 0xce }; -static const u8 enc_assoc054[] __initconst = { +static const u8 enc_assoc054[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce054[] __initconst = { +static const u8 enc_nonce054[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key054[] __initconst = { +static const u8 enc_key054[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3092,7 +3091,7 @@ static const u8 enc_key054[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input055[] __initconst = { +static const u8 enc_input055[] = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, @@ -3110,7 +3109,7 @@ static const u8 enc_input055[] __initconst = { 0x77, 0xd3, 0x0b, 0xc5, 0x76, 0x92, 0xed, 0x38, 0xfb, 0xac, 0x01, 0x88, 0x38, 0x04, 0x88, 0xc7 }; -static const u8 enc_output055[] __initconst = { +static const u8 enc_output055[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -3130,14 +3129,14 @@ static const u8 enc_output055[] __initconst = { 0xd8, 0xb4, 0x79, 0x02, 0xba, 0xae, 0xaf, 0xb3, 0x42, 0x03, 0x05, 0x15, 0x29, 0xaf, 0x28, 0x2e }; -static const u8 enc_assoc055[] __initconst = { +static const u8 enc_assoc055[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce055[] __initconst = { +static const u8 enc_nonce055[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key055[] __initconst = { +static const u8 enc_key055[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3145,13 +3144,13 @@ static const u8 enc_key055[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input056[] __initconst = { +static const u8 enc_input056[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output056[] __initconst = { +static const u8 enc_output056[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3159,14 +3158,14 @@ static const u8 enc_output056[] __initconst = { 0xb3, 0x89, 0x1c, 0x84, 0x9c, 0xb5, 0x2c, 0x27, 0x74, 0x7e, 0xdf, 0xcf, 0x31, 0x21, 0x3b, 0xb6 }; -static const u8 enc_assoc056[] __initconst = { +static const u8 enc_assoc056[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce056[] __initconst = { +static const u8 enc_nonce056[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key056[] __initconst = { +static const u8 enc_key056[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3174,7 +3173,7 @@ static const u8 enc_key056[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input057[] __initconst = { +static const u8 enc_input057[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, @@ -3184,7 +3183,7 @@ static const u8 enc_input057[] __initconst = { 0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a, 0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42 }; -static const u8 enc_output057[] __initconst = { +static const u8 enc_output057[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3196,14 +3195,14 @@ static const u8 enc_output057[] __initconst = { 0xf0, 0xc1, 0x2d, 0x26, 0xef, 0x03, 0x02, 0x9b, 0x62, 0xc0, 0x08, 0xda, 0x27, 0xc5, 0xdc, 0x68 }; -static const u8 enc_assoc057[] __initconst = { +static const u8 enc_assoc057[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce057[] __initconst = { +static const u8 enc_nonce057[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key057[] __initconst = { +static const u8 enc_key057[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3211,7 +3210,7 @@ static const u8 enc_key057[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input058[] __initconst = { +static const u8 enc_input058[] = { 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, @@ -3229,7 +3228,7 @@ static const u8 enc_input058[] __initconst = { 0x88, 0x2c, 0xf4, 0x3a, 0x89, 0x6d, 0x12, 0xc7, 0x04, 0x53, 0xfe, 0x77, 0xc7, 0xfb, 0x77, 0x38 }; -static const u8 enc_output058[] __initconst = { +static const u8 enc_output058[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3249,14 +3248,14 @@ static const u8 enc_output058[] __initconst = { 0xee, 0x65, 0x78, 0x30, 0x01, 0xc2, 0x56, 0x91, 0xfa, 0x28, 0xd0, 0xf5, 0xf1, 0xc1, 0xd7, 0x62 }; -static const u8 enc_assoc058[] __initconst = { +static const u8 enc_assoc058[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce058[] __initconst = { +static const u8 enc_nonce058[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key058[] __initconst = { +static const u8 enc_key058[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3264,13 +3263,13 @@ static const u8 enc_key058[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input059[] __initconst = { +static const u8 enc_input059[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e }; -static const u8 enc_output059[] __initconst = { +static const u8 enc_output059[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3278,14 +3277,14 @@ static const u8 enc_output059[] __initconst = { 0x79, 0xba, 0x7a, 0x29, 0xf5, 0xa7, 0xbb, 0x75, 0x79, 0x7a, 0xf8, 0x7a, 0x61, 0x01, 0x29, 0xa4 }; -static const u8 enc_assoc059[] __initconst = { +static const u8 enc_assoc059[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce059[] __initconst = { +static const u8 enc_nonce059[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key059[] __initconst = { +static const u8 enc_key059[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3293,7 +3292,7 @@ static const u8 enc_key059[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input060[] __initconst = { +static const u8 enc_input060[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, @@ -3303,7 +3302,7 @@ static const u8 enc_input060[] __initconst = { 0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65, 0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d }; -static const u8 enc_output060[] __initconst = { +static const u8 enc_output060[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3315,14 +3314,14 @@ static const u8 enc_output060[] __initconst = { 0x36, 0xb1, 0x74, 0x38, 0x19, 0xe1, 0xb9, 0xba, 0x15, 0x51, 0xe8, 0xed, 0x92, 0x2a, 0x95, 0x9a }; -static const u8 enc_assoc060[] __initconst = { +static const u8 enc_assoc060[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce060[] __initconst = { +static const u8 enc_nonce060[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key060[] __initconst = { +static const u8 enc_key060[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3330,7 +3329,7 @@ static const u8 enc_key060[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input061[] __initconst = { +static const u8 enc_input061[] = { 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, @@ -3348,7 +3347,7 @@ static const u8 enc_input061[] __initconst = { 0x77, 0xd3, 0x0b, 0x45, 0x76, 0x92, 0xed, 0xb8, 0xfb, 0xac, 0x01, 0x08, 0x38, 0x04, 0x88, 0x47 }; -static const u8 enc_output061[] __initconst = { +static const u8 enc_output061[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, @@ -3368,14 +3367,14 @@ static const u8 enc_output061[] __initconst = { 0xfe, 0xac, 0x49, 0x55, 0x55, 0x4e, 0x80, 0x6f, 0x3a, 0x19, 0x02, 0xe2, 0x44, 0x32, 0xc0, 0x8a }; -static const u8 enc_assoc061[] __initconst = { +static const u8 enc_assoc061[] = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_nonce061[] __initconst = { +static const u8 enc_nonce061[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key061[] __initconst = { +static const u8 enc_key061[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3383,13 +3382,13 @@ static const u8 enc_key061[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input062[] __initconst = { +static const u8 enc_input062[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1 }; -static const u8 enc_output062[] __initconst = { +static const u8 enc_output062[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3397,14 +3396,14 @@ static const u8 enc_output062[] __initconst = { 0x20, 0xa3, 0x79, 0x8d, 0xf1, 0x29, 0x2c, 0x59, 0x72, 0xbf, 0x97, 0x41, 0xae, 0xc3, 0x8a, 0x19 }; -static const u8 enc_assoc062[] __initconst = { +static const u8 enc_assoc062[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce062[] __initconst = { +static const u8 enc_nonce062[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key062[] __initconst = { +static const u8 enc_key062[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3412,7 +3411,7 @@ static const u8 enc_key062[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input063[] __initconst = { +static const u8 enc_input063[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, @@ -3422,7 +3421,7 @@ static const u8 enc_input063[] __initconst = { 0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a, 0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2 }; -static const u8 enc_output063[] __initconst = { +static const u8 enc_output063[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3434,14 +3433,14 @@ static const u8 enc_output063[] __initconst = { 0xc0, 0x3d, 0x9f, 0x67, 0x35, 0x4a, 0x97, 0xb2, 0xf0, 0x74, 0xf7, 0x55, 0x15, 0x57, 0xe4, 0x9c }; -static const u8 enc_assoc063[] __initconst = { +static const u8 enc_assoc063[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce063[] __initconst = { +static const u8 enc_nonce063[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key063[] __initconst = { +static const u8 enc_key063[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3449,7 +3448,7 @@ static const u8 enc_key063[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input064[] __initconst = { +static const u8 enc_input064[] = { 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, @@ -3467,7 +3466,7 @@ static const u8 enc_input064[] __initconst = { 0x88, 0x2c, 0xf4, 0xba, 0x89, 0x6d, 0x12, 0x47, 0x04, 0x53, 0xfe, 0xf7, 0xc7, 0xfb, 0x77, 0xb8 }; -static const u8 enc_output064[] __initconst = { +static const u8 enc_output064[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, @@ -3487,14 +3486,14 @@ static const u8 enc_output064[] __initconst = { 0xc8, 0x6d, 0xa8, 0xdd, 0x65, 0x22, 0x86, 0xd5, 0x02, 0x13, 0xd3, 0x28, 0xd6, 0x3e, 0x40, 0x06 }; -static const u8 enc_assoc064[] __initconst = { +static const u8 enc_assoc064[] = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_nonce064[] __initconst = { +static const u8 enc_nonce064[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key064[] __initconst = { +static const u8 enc_key064[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3502,13 +3501,13 @@ static const u8 enc_key064[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input065[] __initconst = { +static const u8 enc_input065[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output065[] __initconst = { +static const u8 enc_output065[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3516,14 +3515,14 @@ static const u8 enc_output065[] __initconst = { 0xbe, 0xde, 0x90, 0x83, 0xce, 0xb3, 0x6d, 0xdf, 0xe5, 0xfa, 0x81, 0x1f, 0x95, 0x47, 0x1c, 0x67 }; -static const u8 enc_assoc065[] __initconst = { +static const u8 enc_assoc065[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce065[] __initconst = { +static const u8 enc_nonce065[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key065[] __initconst = { +static const u8 enc_key065[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3531,7 +3530,7 @@ static const u8 enc_key065[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input066[] __initconst = { +static const u8 enc_input066[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, @@ -3541,7 +3540,7 @@ static const u8 enc_input066[] __initconst = { 0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a, 0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42 }; -static const u8 enc_output066[] __initconst = { +static const u8 enc_output066[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3553,14 +3552,14 @@ static const u8 enc_output066[] __initconst = { 0x30, 0x08, 0x74, 0xbb, 0x06, 0x92, 0xb6, 0x89, 0xde, 0xad, 0x9a, 0xe1, 0x5b, 0x06, 0x73, 0x90 }; -static const u8 enc_assoc066[] __initconst = { +static const u8 enc_assoc066[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce066[] __initconst = { +static const u8 enc_nonce066[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key066[] __initconst = { +static const u8 enc_key066[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3568,7 +3567,7 @@ static const u8 enc_key066[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input067[] __initconst = { +static const u8 enc_input067[] = { 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, @@ -3586,7 +3585,7 @@ static const u8 enc_input067[] __initconst = { 0x08, 0x2c, 0xf4, 0x3a, 0x09, 0x6d, 0x12, 0xc7, 0x84, 0x53, 0xfe, 0x77, 0x47, 0xfb, 0x77, 0x38 }; -static const u8 enc_output067[] __initconst = { +static const u8 enc_output067[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, @@ -3606,14 +3605,14 @@ static const u8 enc_output067[] __initconst = { 0x99, 0xca, 0xd8, 0x5f, 0x45, 0xca, 0x40, 0x94, 0x2d, 0x0d, 0x4d, 0x5e, 0x95, 0x0a, 0xde, 0x22 }; -static const u8 enc_assoc067[] __initconst = { +static const u8 enc_assoc067[] = { 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff }; -static const u8 enc_nonce067[] __initconst = { +static const u8 enc_nonce067[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key067[] __initconst = { +static const u8 enc_key067[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3621,13 +3620,13 @@ static const u8 enc_key067[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input068[] __initconst = { +static const u8 enc_input068[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41 }; -static const u8 enc_output068[] __initconst = { +static const u8 enc_output068[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3635,14 +3634,14 @@ static const u8 enc_output068[] __initconst = { 0x8b, 0xbe, 0x14, 0x52, 0x72, 0xe7, 0xc2, 0xd9, 0xa1, 0x89, 0x1a, 0x3a, 0xb0, 0x98, 0x3d, 0x9d }; -static const u8 enc_assoc068[] __initconst = { +static const u8 enc_assoc068[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce068[] __initconst = { +static const u8 enc_nonce068[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key068[] __initconst = { +static const u8 enc_key068[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3650,7 +3649,7 @@ static const u8 enc_key068[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input069[] __initconst = { +static const u8 enc_input069[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, @@ -3660,7 +3659,7 @@ static const u8 enc_input069[] __initconst = { 0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a, 0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42 }; -static const u8 enc_output069[] __initconst = { +static const u8 enc_output069[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3672,14 +3671,14 @@ static const u8 enc_output069[] __initconst = { 0x3b, 0x41, 0x86, 0x19, 0x13, 0xa8, 0xf6, 0xde, 0x7f, 0x61, 0xe2, 0x25, 0x63, 0x1b, 0xc3, 0x82 }; -static const u8 enc_assoc069[] __initconst = { +static const u8 enc_assoc069[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce069[] __initconst = { +static const u8 enc_nonce069[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key069[] __initconst = { +static const u8 enc_key069[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3687,7 +3686,7 @@ static const u8 enc_key069[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input070[] __initconst = { +static const u8 enc_input070[] = { 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, @@ -3705,7 +3704,7 @@ static const u8 enc_input070[] __initconst = { 0x77, 0xd3, 0x0b, 0xc5, 0x89, 0x6d, 0x12, 0xc7, 0xfb, 0xac, 0x01, 0x88, 0xc7, 0xfb, 0x77, 0x38 }; -static const u8 enc_output070[] __initconst = { +static const u8 enc_output070[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, @@ -3725,14 +3724,14 @@ static const u8 enc_output070[] __initconst = { 0x84, 0x28, 0xbc, 0xf0, 0x23, 0xec, 0x6b, 0xf3, 0x1f, 0xd9, 0xef, 0xb2, 0x03, 0xff, 0x08, 0x71 }; -static const u8 enc_assoc070[] __initconst = { +static const u8 enc_assoc070[] = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce070[] __initconst = { +static const u8 enc_nonce070[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key070[] __initconst = { +static const u8 enc_key070[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3740,13 +3739,13 @@ static const u8 enc_key070[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input071[] __initconst = { +static const u8 enc_input071[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe }; -static const u8 enc_output071[] __initconst = { +static const u8 enc_output071[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3754,14 +3753,14 @@ static const u8 enc_output071[] __initconst = { 0x13, 0x9f, 0xdf, 0x64, 0x74, 0xea, 0x24, 0xf5, 0x49, 0xb0, 0x75, 0x82, 0x5f, 0x2c, 0x76, 0x20 }; -static const u8 enc_assoc071[] __initconst = { +static const u8 enc_assoc071[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce071[] __initconst = { +static const u8 enc_nonce071[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key071[] __initconst = { +static const u8 enc_key071[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3769,7 +3768,7 @@ static const u8 enc_key071[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input072[] __initconst = { +static const u8 enc_input072[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, @@ -3779,7 +3778,7 @@ static const u8 enc_input072[] __initconst = { 0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5, 0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd }; -static const u8 enc_output072[] __initconst = { +static const u8 enc_output072[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3791,14 +3790,14 @@ static const u8 enc_output072[] __initconst = { 0xbb, 0xad, 0x8d, 0x86, 0x3b, 0x83, 0x5a, 0x8e, 0x86, 0x64, 0xfd, 0x1d, 0x45, 0x66, 0xb6, 0xb4 }; -static const u8 enc_assoc072[] __initconst = { +static const u8 enc_assoc072[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce072[] __initconst = { +static const u8 enc_nonce072[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key072[] __initconst = { +static const u8 enc_key072[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3806,7 +3805,7 @@ static const u8 enc_key072[] __initconst = { }; /* wycheproof - misc */ -static const u8 enc_input073[] __initconst = { +static const u8 enc_input073[] = { 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, @@ -3824,7 +3823,7 @@ static const u8 enc_input073[] __initconst = { 0x88, 0x2c, 0xf4, 0x3a, 0x76, 0x92, 0xed, 0x38, 0x04, 0x53, 0xfe, 0x77, 0x38, 0x04, 0x88, 0xc7 }; -static const u8 enc_output073[] __initconst = { +static const u8 enc_output073[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, @@ -3844,14 +3843,14 @@ static const u8 enc_output073[] __initconst = { 0x42, 0xf2, 0x35, 0x42, 0x97, 0x84, 0x9a, 0x51, 0x1d, 0x53, 0xe5, 0x57, 0x17, 0x72, 0xf7, 0x1f }; -static const u8 enc_assoc073[] __initconst = { +static const u8 enc_assoc073[] = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_nonce073[] __initconst = { +static const u8 enc_nonce073[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 }; -static const u8 enc_key073[] __initconst = { +static const u8 enc_key073[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -3859,7 +3858,7 @@ static const u8 enc_key073[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input074[] __initconst = { +static const u8 enc_input074[] = { 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, @@ -3877,7 +3876,7 @@ static const u8 enc_input074[] __initconst = { 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a }; -static const u8 enc_output074[] __initconst = { +static const u8 enc_output074[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3897,7 +3896,7 @@ static const u8 enc_output074[] __initconst = { 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca }; -static const u8 enc_assoc074[] __initconst = { +static const u8 enc_assoc074[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3907,11 +3906,11 @@ static const u8 enc_assoc074[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce074[] __initconst = { +static const u8 enc_nonce074[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x00, 0x02, 0x50, 0x6e }; -static const u8 enc_key074[] __initconst = { +static const u8 enc_key074[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -3919,7 +3918,7 @@ static const u8 enc_key074[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input075[] __initconst = { +static const u8 enc_input075[] = { 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, @@ -3937,7 +3936,7 @@ static const u8 enc_input075[] __initconst = { 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 }; -static const u8 enc_output075[] __initconst = { +static const u8 enc_output075[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3957,7 +3956,7 @@ static const u8 enc_output075[] __initconst = { 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 }; -static const u8 enc_assoc075[] __initconst = { +static const u8 enc_assoc075[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -3967,11 +3966,11 @@ static const u8 enc_assoc075[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce075[] __initconst = { +static const u8 enc_nonce075[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x00, 0x03, 0x18, 0xa5 }; -static const u8 enc_key075[] __initconst = { +static const u8 enc_key075[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -3979,7 +3978,7 @@ static const u8 enc_key075[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input076[] __initconst = { +static const u8 enc_input076[] = { 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, 0xde, 0x22, 0xff, 0x5b, 0x8a, 0xdd, 0x95, 0x02, 0xce, 0x03, 0xa0, 0xfa, 0xf5, 0x99, 0x2a, 0x09, @@ -3997,7 +3996,7 @@ static const u8 enc_input076[] __initconst = { 0xc3, 0x53, 0xd0, 0xcf, 0x93, 0x8d, 0xcc, 0xb9, 0xef, 0xad, 0x8f, 0xed, 0xbe, 0x46, 0xda, 0xa5 }; -static const u8 enc_output076[] __initconst = { +static const u8 enc_output076[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4017,7 +4016,7 @@ static const u8 enc_output076[] __initconst = { 0x4b, 0x0b, 0xda, 0x8a, 0xd0, 0x43, 0x83, 0x0d, 0x83, 0x19, 0xab, 0x82, 0xc5, 0x0c, 0x76, 0x63 }; -static const u8 enc_assoc076[] __initconst = { +static const u8 enc_assoc076[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4027,10 +4026,10 @@ static const u8 enc_assoc076[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce076[] __initconst = { +static const u8 enc_nonce076[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb4, 0xf0 }; -static const u8 enc_key076[] __initconst = { +static const u8 enc_key076[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4038,7 +4037,7 @@ static const u8 enc_key076[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input077[] __initconst = { +static const u8 enc_input077[] = { 0x86, 0xcb, 0xac, 0xae, 0x4d, 0x3f, 0x74, 0xae, 0x01, 0x21, 0x3e, 0x05, 0x51, 0xcc, 0x15, 0x16, 0x0e, 0xa1, 0xbe, 0x84, 0x08, 0xe3, 0xd5, 0xd7, @@ -4056,7 +4055,7 @@ static const u8 enc_input077[] __initconst = { 0x0e, 0x76, 0x2b, 0x43, 0x0c, 0x4d, 0x51, 0x7c, 0x97, 0x10, 0x70, 0x68, 0xf4, 0x98, 0xef, 0x7f }; -static const u8 enc_output077[] __initconst = { +static const u8 enc_output077[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4076,7 +4075,7 @@ static const u8 enc_output077[] __initconst = { 0x4b, 0xc9, 0x8f, 0x72, 0xc4, 0x94, 0xc2, 0xa4, 0x3c, 0x2b, 0x15, 0xa1, 0x04, 0x3f, 0x1c, 0xfa }; -static const u8 enc_assoc077[] __initconst = { +static const u8 enc_assoc077[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4086,10 +4085,10 @@ static const u8 enc_assoc077[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce077[] __initconst = { +static const u8 enc_nonce077[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xfb, 0x66 }; -static const u8 enc_key077[] __initconst = { +static const u8 enc_key077[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4097,7 +4096,7 @@ static const u8 enc_key077[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input078[] __initconst = { +static const u8 enc_input078[] = { 0xfa, 0xb1, 0xcd, 0xdf, 0x4f, 0xe1, 0x98, 0xef, 0x63, 0xad, 0xd8, 0x81, 0xd6, 0xea, 0xd6, 0xc5, 0x76, 0x37, 0xbb, 0xe9, 0x20, 0x18, 0xca, 0x7c, @@ -4115,7 +4114,7 @@ static const u8 enc_input078[] __initconst = { 0xc3, 0x24, 0x60, 0x41, 0x43, 0x21, 0x43, 0xe9, 0xab, 0x3a, 0x6d, 0x2c, 0xcc, 0x2f, 0x4d, 0x62 }; -static const u8 enc_output078[] __initconst = { +static const u8 enc_output078[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4135,7 +4134,7 @@ static const u8 enc_output078[] __initconst = { 0xf7, 0xe9, 0xe1, 0x51, 0xb0, 0x25, 0x33, 0xc7, 0x46, 0x58, 0xbf, 0xc7, 0x73, 0x7c, 0x68, 0x0d }; -static const u8 enc_assoc078[] __initconst = { +static const u8 enc_assoc078[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4145,10 +4144,10 @@ static const u8 enc_assoc078[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce078[] __initconst = { +static const u8 enc_nonce078[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0xbb, 0x90 }; -static const u8 enc_key078[] __initconst = { +static const u8 enc_key078[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4156,7 +4155,7 @@ static const u8 enc_key078[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input079[] __initconst = { +static const u8 enc_input079[] = { 0x22, 0x72, 0x02, 0xbe, 0x7f, 0x35, 0x15, 0xe9, 0xd1, 0xc0, 0x2e, 0xea, 0x2f, 0x19, 0x50, 0xb6, 0x48, 0x1b, 0x04, 0x8a, 0x4c, 0x91, 0x50, 0x6c, @@ -4174,7 +4173,7 @@ static const u8 enc_input079[] __initconst = { 0xfb, 0xe1, 0xee, 0x45, 0xb1, 0xb2, 0x1f, 0x71, 0x62, 0xe2, 0xfc, 0xaa, 0x74, 0x2a, 0xbe, 0xfd }; -static const u8 enc_output079[] __initconst = { +static const u8 enc_output079[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4194,7 +4193,7 @@ static const u8 enc_output079[] __initconst = { 0x79, 0x5b, 0xcf, 0xf6, 0x47, 0xc5, 0x53, 0xc2, 0xe4, 0xeb, 0x6e, 0x0e, 0xaf, 0xd9, 0xe0, 0x4e }; -static const u8 enc_assoc079[] __initconst = { +static const u8 enc_assoc079[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4204,10 +4203,10 @@ static const u8 enc_assoc079[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce079[] __initconst = { +static const u8 enc_nonce079[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x48, 0x4a }; -static const u8 enc_key079[] __initconst = { +static const u8 enc_key079[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4215,7 +4214,7 @@ static const u8 enc_key079[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input080[] __initconst = { +static const u8 enc_input080[] = { 0xfa, 0xe5, 0x83, 0x45, 0xc1, 0x6c, 0xb0, 0xf5, 0xcc, 0x53, 0x7f, 0x2b, 0x1b, 0x34, 0x69, 0xc9, 0x69, 0x46, 0x3b, 0x3e, 0xa7, 0x1b, 0xcf, 0x6b, @@ -4233,7 +4232,7 @@ static const u8 enc_input080[] __initconst = { 0x18, 0x01, 0xce, 0x33, 0xc4, 0xe4, 0xa7, 0x7d, 0x83, 0x1d, 0x3c, 0xe3, 0x4e, 0x84, 0x10, 0xe1 }; -static const u8 enc_output080[] __initconst = { +static const u8 enc_output080[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4253,7 +4252,7 @@ static const u8 enc_output080[] __initconst = { 0x19, 0x46, 0xd6, 0x53, 0x96, 0x0f, 0x94, 0x7a, 0x74, 0xd3, 0xe8, 0x09, 0x3c, 0xf4, 0x85, 0x02 }; -static const u8 enc_assoc080[] __initconst = { +static const u8 enc_assoc080[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4263,10 +4262,10 @@ static const u8 enc_assoc080[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce080[] __initconst = { +static const u8 enc_nonce080[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x2f, 0x40 }; -static const u8 enc_key080[] __initconst = { +static const u8 enc_key080[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4274,7 +4273,7 @@ static const u8 enc_key080[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input081[] __initconst = { +static const u8 enc_input081[] = { 0xeb, 0xb2, 0x16, 0xdd, 0xd7, 0xca, 0x70, 0x92, 0x15, 0xf5, 0x03, 0xdf, 0x9c, 0xe6, 0x3c, 0x5c, 0xd2, 0x19, 0x4e, 0x7d, 0x90, 0x99, 0xe8, 0xa9, @@ -4292,7 +4291,7 @@ static const u8 enc_input081[] __initconst = { 0x83, 0xfd, 0xca, 0x39, 0xd2, 0xe1, 0x4f, 0x23, 0xd0, 0x0a, 0x58, 0x26, 0x64, 0xf4, 0xec, 0xb1 }; -static const u8 enc_output081[] __initconst = { +static const u8 enc_output081[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4312,7 +4311,7 @@ static const u8 enc_output081[] __initconst = { 0x36, 0xc3, 0x00, 0x29, 0x85, 0xdd, 0x21, 0xba, 0xf8, 0x95, 0xd6, 0x33, 0x57, 0x3f, 0x12, 0xc0 }; -static const u8 enc_assoc081[] __initconst = { +static const u8 enc_assoc081[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4322,10 +4321,10 @@ static const u8 enc_assoc081[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce081[] __initconst = { +static const u8 enc_nonce081[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x93, 0x35 }; -static const u8 enc_key081[] __initconst = { +static const u8 enc_key081[] = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, @@ -4333,7 +4332,7 @@ static const u8 enc_key081[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input082[] __initconst = { +static const u8 enc_input082[] = { 0x40, 0x8a, 0xe6, 0xef, 0x1c, 0x7e, 0xf0, 0xfb, 0x2c, 0x2d, 0x61, 0x08, 0x16, 0xfc, 0x78, 0x49, 0xef, 0xa5, 0x8f, 0x78, 0x27, 0x3f, 0x5f, 0x16, @@ -4351,7 +4350,7 @@ static const u8 enc_input082[] __initconst = { 0x2b, 0x68, 0x9f, 0x93, 0x55, 0xd9, 0xc1, 0x83, 0x80, 0x1f, 0x6a, 0xcc, 0x31, 0x3f, 0x89, 0x07 }; -static const u8 enc_output082[] __initconst = { +static const u8 enc_output082[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4371,7 +4370,7 @@ static const u8 enc_output082[] __initconst = { 0x65, 0x14, 0x51, 0x8e, 0x0a, 0x26, 0x41, 0x42, 0xe0, 0xb7, 0x35, 0x1f, 0x96, 0x7f, 0xc2, 0xae }; -static const u8 enc_assoc082[] __initconst = { +static const u8 enc_assoc082[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4381,10 +4380,10 @@ static const u8 enc_assoc082[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce082[] __initconst = { +static const u8 enc_nonce082[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xf7, 0xd5 }; -static const u8 enc_key082[] __initconst = { +static const u8 enc_key082[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4392,7 +4391,7 @@ static const u8 enc_key082[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input083[] __initconst = { +static const u8 enc_input083[] = { 0x0a, 0x0a, 0x24, 0x49, 0x9b, 0xca, 0xde, 0x58, 0xcf, 0x15, 0x76, 0xc3, 0x12, 0xac, 0xa9, 0x84, 0x71, 0x8c, 0xb4, 0xcc, 0x7e, 0x01, 0x53, 0xf5, @@ -4410,7 +4409,7 @@ static const u8 enc_input083[] __initconst = { 0xb3, 0x06, 0xa2, 0x1b, 0x42, 0xd4, 0xc3, 0xba, 0x6e, 0x6f, 0x0c, 0xbc, 0xc8, 0x1e, 0x87, 0x7a }; -static const u8 enc_output083[] __initconst = { +static const u8 enc_output083[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4430,7 +4429,7 @@ static const u8 enc_output083[] __initconst = { 0x4c, 0x19, 0x4d, 0xa6, 0xa9, 0x9f, 0xd6, 0x5b, 0x40, 0xe9, 0xca, 0xd7, 0x98, 0xf4, 0x4b, 0x19 }; -static const u8 enc_assoc083[] __initconst = { +static const u8 enc_assoc083[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4440,10 +4439,10 @@ static const u8 enc_assoc083[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce083[] __initconst = { +static const u8 enc_nonce083[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xfc, 0xe4 }; -static const u8 enc_key083[] __initconst = { +static const u8 enc_key083[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4451,7 +4450,7 @@ static const u8 enc_key083[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input084[] __initconst = { +static const u8 enc_input084[] = { 0x4a, 0x0a, 0xaf, 0xf8, 0x49, 0x47, 0x29, 0x18, 0x86, 0x91, 0x70, 0x13, 0x40, 0xf3, 0xce, 0x2b, 0x8a, 0x78, 0xee, 0xd3, 0xa0, 0xf0, 0x65, 0x99, @@ -4469,7 +4468,7 @@ static const u8 enc_input084[] __initconst = { 0x5d, 0x7e, 0x51, 0x3b, 0xe5, 0xb8, 0xea, 0x97, 0x13, 0x10, 0xd5, 0xbf, 0x16, 0xba, 0x7a, 0xee }; -static const u8 enc_output084[] __initconst = { +static const u8 enc_output084[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4489,7 +4488,7 @@ static const u8 enc_output084[] __initconst = { 0xc8, 0xae, 0x77, 0x88, 0xcd, 0x28, 0x74, 0xab, 0xc1, 0x38, 0x54, 0x1e, 0x11, 0xfd, 0x05, 0x87 }; -static const u8 enc_assoc084[] __initconst = { +static const u8 enc_assoc084[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4499,10 +4498,10 @@ static const u8 enc_assoc084[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce084[] __initconst = { +static const u8 enc_nonce084[] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x84, 0x86, 0xa8 }; -static const u8 enc_key084[] __initconst = { +static const u8 enc_key084[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4510,7 +4509,7 @@ static const u8 enc_key084[] __initconst = { }; /* wycheproof - checking for int overflows */ -static const u8 enc_input085[] __initconst = { +static const u8 enc_input085[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x78, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -4528,7 +4527,7 @@ static const u8 enc_input085[] __initconst = { 0xf7, 0x4d, 0x5b, 0xf8, 0x67, 0x1c, 0x5a, 0x8a, 0x50, 0x92, 0xf6, 0x1d, 0x54, 0xc9, 0xaa, 0x5b }; -static const u8 enc_output085[] __initconst = { +static const u8 enc_output085[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4548,7 +4547,7 @@ static const u8 enc_output085[] __initconst = { 0x93, 0x3a, 0x51, 0x63, 0xc7, 0xf6, 0x23, 0x68, 0x32, 0x7b, 0x3f, 0xbc, 0x10, 0x36, 0xc9, 0x43 }; -static const u8 enc_assoc085[] __initconst = { +static const u8 enc_assoc085[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4558,10 +4557,10 @@ static const u8 enc_assoc085[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce085[] __initconst = { +static const u8 enc_nonce085[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key085[] __initconst = { +static const u8 enc_key085[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4569,7 +4568,7 @@ static const u8 enc_key085[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input086[] __initconst = { +static const u8 enc_input086[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4579,7 +4578,7 @@ static const u8 enc_input086[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output086[] __initconst = { +static const u8 enc_output086[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4591,17 +4590,17 @@ static const u8 enc_output086[] __initconst = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }; -static const u8 enc_assoc086[] __initconst = { +static const u8 enc_assoc086[] = { 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 }; -static const u8 enc_nonce086[] __initconst = { +static const u8 enc_nonce086[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key086[] __initconst = { +static const u8 enc_key086[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4609,7 +4608,7 @@ static const u8 enc_key086[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input087[] __initconst = { +static const u8 enc_input087[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4619,7 +4618,7 @@ static const u8 enc_input087[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output087[] __initconst = { +static const u8 enc_output087[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4631,17 +4630,17 @@ static const u8 enc_output087[] __initconst = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc087[] __initconst = { +static const u8 enc_assoc087[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 }; -static const u8 enc_nonce087[] __initconst = { +static const u8 enc_nonce087[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key087[] __initconst = { +static const u8 enc_key087[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4649,7 +4648,7 @@ static const u8 enc_key087[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input088[] __initconst = { +static const u8 enc_input088[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4659,7 +4658,7 @@ static const u8 enc_input088[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output088[] __initconst = { +static const u8 enc_output088[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4671,17 +4670,17 @@ static const u8 enc_output088[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_assoc088[] __initconst = { +static const u8 enc_assoc088[] = { 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 }; -static const u8 enc_nonce088[] __initconst = { +static const u8 enc_nonce088[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key088[] __initconst = { +static const u8 enc_key088[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4689,7 +4688,7 @@ static const u8 enc_key088[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input089[] __initconst = { +static const u8 enc_input089[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4699,7 +4698,7 @@ static const u8 enc_input089[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output089[] __initconst = { +static const u8 enc_output089[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4711,17 +4710,17 @@ static const u8 enc_output089[] __initconst = { 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 }; -static const u8 enc_assoc089[] __initconst = { +static const u8 enc_assoc089[] = { 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 }; -static const u8 enc_nonce089[] __initconst = { +static const u8 enc_nonce089[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key089[] __initconst = { +static const u8 enc_key089[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4729,7 +4728,7 @@ static const u8 enc_key089[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input090[] __initconst = { +static const u8 enc_input090[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4739,7 +4738,7 @@ static const u8 enc_input090[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output090[] __initconst = { +static const u8 enc_output090[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4751,17 +4750,17 @@ static const u8 enc_output090[] __initconst = { 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f }; -static const u8 enc_assoc090[] __initconst = { +static const u8 enc_assoc090[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 }; -static const u8 enc_nonce090[] __initconst = { +static const u8 enc_nonce090[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key090[] __initconst = { +static const u8 enc_key090[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4769,7 +4768,7 @@ static const u8 enc_key090[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input091[] __initconst = { +static const u8 enc_input091[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4779,7 +4778,7 @@ static const u8 enc_input091[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output091[] __initconst = { +static const u8 enc_output091[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4791,17 +4790,17 @@ static const u8 enc_output091[] __initconst = { 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc091[] __initconst = { +static const u8 enc_assoc091[] = { 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 }; -static const u8 enc_nonce091[] __initconst = { +static const u8 enc_nonce091[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key091[] __initconst = { +static const u8 enc_key091[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4809,7 +4808,7 @@ static const u8 enc_key091[] __initconst = { }; /* wycheproof - special case tag */ -static const u8 enc_input092[] __initconst = { +static const u8 enc_input092[] = { 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, @@ -4819,7 +4818,7 @@ static const u8 enc_input092[] __initconst = { 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d }; -static const u8 enc_output092[] __initconst = { +static const u8 enc_output092[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4831,17 +4830,17 @@ static const u8 enc_output092[] __initconst = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; -static const u8 enc_assoc092[] __initconst = { +static const u8 enc_assoc092[] = { 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 }; -static const u8 enc_nonce092[] __initconst = { +static const u8 enc_nonce092[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b }; -static const u8 enc_key092[] __initconst = { +static const u8 enc_key092[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4849,7 +4848,7 @@ static const u8 enc_key092[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input093[] __initconst = { +static const u8 enc_input093[] = { 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, 0x3d, 0xb7, 0x66, 0x4a, 0x34, 0xae, 0x6b, 0x44, 0x4d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -4861,7 +4860,7 @@ static const u8 enc_input093[] __initconst = { 0x83, 0xdc, 0xe9, 0xf3, 0x07, 0x3e, 0xfa, 0xdb, 0x7d, 0x23, 0xb8, 0x7a, 0xce, 0x35, 0x16, 0x8c }; -static const u8 enc_output093[] __initconst = { +static const u8 enc_output093[] = { 0x00, 0x39, 0xe2, 0xfd, 0x2f, 0xd3, 0x12, 0x14, 0x9e, 0x98, 0x98, 0x80, 0x88, 0x48, 0x13, 0xe7, 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -4875,13 +4874,13 @@ static const u8 enc_output093[] __initconst = { 0xa5, 0x19, 0xac, 0x1a, 0x35, 0xb4, 0xa5, 0x77, 0x87, 0x51, 0x0a, 0xf7, 0x8d, 0x8d, 0x20, 0x0a }; -static const u8 enc_assoc093[] __initconst = { +static const u8 enc_assoc093[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce093[] __initconst = { +static const u8 enc_nonce093[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key093[] __initconst = { +static const u8 enc_key093[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4889,7 +4888,7 @@ static const u8 enc_key093[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input094[] __initconst = { +static const u8 enc_input094[] = { 0xd3, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xe5, 0xda, 0x78, 0x76, 0x6f, 0xa1, 0x92, 0x90, @@ -4903,7 +4902,7 @@ static const u8 enc_input094[] __initconst = { 0x01, 0x49, 0xef, 0x50, 0x4b, 0x71, 0xb1, 0x20, 0xca, 0x4f, 0xf3, 0x95, 0x19, 0xc2, 0xc2, 0x10 }; -static const u8 enc_output094[] __initconst = { +static const u8 enc_output094[] = { 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x62, 0x18, 0xb2, 0x7f, 0x83, 0xb8, 0xb4, 0x66, @@ -4919,13 +4918,13 @@ static const u8 enc_output094[] __initconst = { 0x30, 0x2f, 0xe8, 0x2a, 0xb0, 0xa0, 0x9a, 0xf6, 0x44, 0x00, 0xd0, 0x15, 0xae, 0x83, 0xd9, 0xcc }; -static const u8 enc_assoc094[] __initconst = { +static const u8 enc_assoc094[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce094[] __initconst = { +static const u8 enc_nonce094[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key094[] __initconst = { +static const u8 enc_key094[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4933,7 +4932,7 @@ static const u8 enc_key094[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input095[] __initconst = { +static const u8 enc_input095[] = { 0xe9, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x6d, 0xf1, 0x39, 0x4e, 0xdc, 0x53, 0x9b, 0x5b, @@ -4947,7 +4946,7 @@ static const u8 enc_input095[] __initconst = { 0x99, 0x52, 0xae, 0x08, 0x18, 0xc3, 0x89, 0x79, 0xc0, 0x74, 0x13, 0x71, 0x1a, 0x9a, 0xf7, 0x13 }; -static const u8 enc_output095[] __initconst = { +static const u8 enc_output095[] = { 0xe9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xea, 0x33, 0xf3, 0x47, 0x30, 0x4a, 0xbd, 0xad, @@ -4963,13 +4962,13 @@ static const u8 enc_output095[] __initconst = { 0x98, 0xa7, 0xe8, 0x36, 0xe0, 0xee, 0x4d, 0x02, 0x35, 0x00, 0xd0, 0x55, 0x7e, 0xc2, 0xcb, 0xe0 }; -static const u8 enc_assoc095[] __initconst = { +static const u8 enc_assoc095[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce095[] __initconst = { +static const u8 enc_nonce095[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key095[] __initconst = { +static const u8 enc_key095[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -4977,7 +4976,7 @@ static const u8 enc_key095[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input096[] __initconst = { +static const u8 enc_input096[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x64, 0xf9, 0x0f, 0x5b, 0x26, 0x92, 0xb8, 0x60, @@ -4987,7 +4986,7 @@ static const u8 enc_input096[] __initconst = { 0x03, 0x35, 0x61, 0xe7, 0xca, 0xca, 0x6d, 0x94, 0x1d, 0xc3, 0xcd, 0x69, 0x14, 0xad, 0x69, 0x04 }; -static const u8 enc_output096[] __initconst = { +static const u8 enc_output096[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe3, 0x3b, 0xc5, 0x52, 0xca, 0x8b, 0x9e, 0x96, @@ -4999,13 +4998,13 @@ static const u8 enc_output096[] __initconst = { 0x6a, 0xb8, 0xdc, 0xe2, 0xc5, 0x9d, 0xa4, 0x73, 0x71, 0x30, 0xb0, 0x25, 0x2f, 0x68, 0xa8, 0xd8 }; -static const u8 enc_assoc096[] __initconst = { +static const u8 enc_assoc096[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce096[] __initconst = { +static const u8 enc_nonce096[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key096[] __initconst = { +static const u8 enc_key096[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5013,7 +5012,7 @@ static const u8 enc_key096[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input097[] __initconst = { +static const u8 enc_input097[] = { 0x68, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xb0, 0x8f, 0x25, 0x67, 0x5b, 0x9b, 0xcb, 0xf6, @@ -5027,7 +5026,7 @@ static const u8 enc_input097[] __initconst = { 0x65, 0x0e, 0xc6, 0x2d, 0x75, 0x70, 0x72, 0xce, 0xe6, 0xff, 0x23, 0x31, 0x86, 0xdd, 0x1c, 0x8f }; -static const u8 enc_output097[] __initconst = { +static const u8 enc_output097[] = { 0x68, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x37, 0x4d, 0xef, 0x6e, 0xb7, 0x82, 0xed, 0x00, @@ -5043,13 +5042,13 @@ static const u8 enc_output097[] __initconst = { 0x04, 0x4d, 0xea, 0x60, 0x88, 0x80, 0x41, 0x2b, 0xfd, 0xff, 0xcf, 0x35, 0x57, 0x9e, 0x9b, 0x26 }; -static const u8 enc_assoc097[] __initconst = { +static const u8 enc_assoc097[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce097[] __initconst = { +static const u8 enc_nonce097[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key097[] __initconst = { +static const u8 enc_key097[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5057,7 +5056,7 @@ static const u8 enc_key097[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input098[] __initconst = { +static const u8 enc_input098[] = { 0x6d, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xa1, 0x61, 0xb5, 0xab, 0x04, 0x09, 0x00, 0x62, @@ -5071,7 +5070,7 @@ static const u8 enc_input098[] __initconst = { 0x8e, 0xdc, 0x36, 0x6c, 0xd6, 0x97, 0x65, 0x6f, 0xca, 0x81, 0xfb, 0x13, 0x3c, 0xed, 0x79, 0xa1 }; -static const u8 enc_output098[] __initconst = { +static const u8 enc_output098[] = { 0x6d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x26, 0xa3, 0x7f, 0xa2, 0xe8, 0x10, 0x26, 0x94, @@ -5087,13 +5086,13 @@ static const u8 enc_output098[] __initconst = { 0x1e, 0x6b, 0xea, 0x63, 0x14, 0x54, 0x2e, 0x2e, 0xf9, 0xff, 0xcf, 0x45, 0x0b, 0x2e, 0x98, 0x2b }; -static const u8 enc_assoc098[] __initconst = { +static const u8 enc_assoc098[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce098[] __initconst = { +static const u8 enc_nonce098[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key098[] __initconst = { +static const u8 enc_key098[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5101,7 +5100,7 @@ static const u8 enc_key098[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input099[] __initconst = { +static const u8 enc_input099[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xfc, 0x01, 0xb8, 0x91, 0xe5, 0xf0, 0xf9, 0x12, @@ -5111,7 +5110,7 @@ static const u8 enc_input099[] __initconst = { 0x60, 0xcd, 0x9e, 0xa1, 0x0c, 0x29, 0xa3, 0x66, 0x54, 0xe7, 0xa2, 0x8e, 0x76, 0x1b, 0xec, 0xd8 }; -static const u8 enc_output099[] __initconst = { +static const u8 enc_output099[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7b, 0xc3, 0x72, 0x98, 0x09, 0xe9, 0xdf, 0xe4, @@ -5123,13 +5122,13 @@ static const u8 enc_output099[] __initconst = { 0xed, 0x20, 0x17, 0xc8, 0xdb, 0xa4, 0x77, 0x56, 0x29, 0x04, 0x9d, 0x78, 0x6e, 0x3b, 0xce, 0xb1 }; -static const u8 enc_assoc099[] __initconst = { +static const u8 enc_assoc099[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce099[] __initconst = { +static const u8 enc_nonce099[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key099[] __initconst = { +static const u8 enc_key099[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5137,7 +5136,7 @@ static const u8 enc_key099[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input100[] __initconst = { +static const u8 enc_input100[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x6b, 0x6d, 0xc9, 0xd2, 0x1a, 0x81, 0x9e, 0x70, @@ -5147,7 +5146,7 @@ static const u8 enc_input100[] __initconst = { 0x10, 0xb9, 0x2f, 0x5f, 0xfe, 0xf9, 0x8b, 0x84, 0x7c, 0xf1, 0x7a, 0x9c, 0x98, 0xd8, 0x83, 0xe5 }; -static const u8 enc_output100[] __initconst = { +static const u8 enc_output100[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xec, 0xaf, 0x03, 0xdb, 0xf6, 0x98, 0xb8, 0x86, @@ -5159,13 +5158,13 @@ static const u8 enc_output100[] __initconst = { 0x07, 0x3f, 0x17, 0xcb, 0x67, 0x78, 0x64, 0x59, 0x25, 0x04, 0x9d, 0x88, 0x22, 0xcb, 0xca, 0xb6 }; -static const u8 enc_assoc100[] __initconst = { +static const u8 enc_assoc100[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce100[] __initconst = { +static const u8 enc_nonce100[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key100[] __initconst = { +static const u8 enc_key100[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5173,7 +5172,7 @@ static const u8 enc_key100[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input101[] __initconst = { +static const u8 enc_input101[] = { 0xff, 0xcb, 0x2b, 0x11, 0x06, 0xf8, 0x23, 0x4c, 0x5e, 0x99, 0xd4, 0xdb, 0x4c, 0x70, 0x48, 0xde, 0x32, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5185,7 +5184,7 @@ static const u8 enc_input101[] __initconst = { 0xce, 0xbe, 0xf5, 0xe9, 0x88, 0x5a, 0x80, 0xea, 0x76, 0xd9, 0x75, 0xc1, 0x44, 0xa4, 0x18, 0x88 }; -static const u8 enc_output101[] __initconst = { +static const u8 enc_output101[] = { 0xff, 0xa0, 0xfc, 0x3e, 0x80, 0x32, 0xc3, 0xd5, 0xfd, 0xb6, 0x2a, 0x11, 0xf0, 0x96, 0x30, 0x7d, 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5199,13 +5198,13 @@ static const u8 enc_output101[] __initconst = { 0x8b, 0x9b, 0xb4, 0xb4, 0x86, 0x12, 0x89, 0x65, 0x8c, 0x69, 0x6a, 0x83, 0x40, 0x15, 0x04, 0x05 }; -static const u8 enc_assoc101[] __initconst = { +static const u8 enc_assoc101[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce101[] __initconst = { +static const u8 enc_nonce101[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key101[] __initconst = { +static const u8 enc_key101[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5213,7 +5212,7 @@ static const u8 enc_key101[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input102[] __initconst = { +static const u8 enc_input102[] = { 0x6f, 0x9e, 0x70, 0xed, 0x3b, 0x8b, 0xac, 0xa0, 0x26, 0xe4, 0x6a, 0x5a, 0x09, 0x43, 0x15, 0x8d, 0x21, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5225,7 +5224,7 @@ static const u8 enc_input102[] __initconst = { 0xd4, 0x36, 0x51, 0xfd, 0x14, 0x9c, 0x26, 0x0b, 0xcb, 0xdd, 0x7b, 0x12, 0x68, 0x01, 0x31, 0x8c }; -static const u8 enc_output102[] __initconst = { +static const u8 enc_output102[] = { 0x6f, 0xf5, 0xa7, 0xc2, 0xbd, 0x41, 0x4c, 0x39, 0x85, 0xcb, 0x94, 0x90, 0xb5, 0xa5, 0x6d, 0x2e, 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5239,13 +5238,13 @@ static const u8 enc_output102[] __initconst = { 0x8b, 0x3b, 0xbd, 0x51, 0x64, 0x44, 0x59, 0x56, 0x8d, 0x81, 0xca, 0x1f, 0xa7, 0x2c, 0xe4, 0x04 }; -static const u8 enc_assoc102[] __initconst = { +static const u8 enc_assoc102[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce102[] __initconst = { +static const u8 enc_nonce102[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key102[] __initconst = { +static const u8 enc_key102[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5253,7 +5252,7 @@ static const u8 enc_key102[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input103[] __initconst = { +static const u8 enc_input103[] = { 0x41, 0x2b, 0x08, 0x0a, 0x3e, 0x19, 0xc1, 0x0d, 0x44, 0xa1, 0xaf, 0x1e, 0xab, 0xde, 0xb4, 0xce, 0x35, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5265,7 +5264,7 @@ static const u8 enc_input103[] __initconst = { 0xb3, 0xd4, 0xe9, 0x90, 0x90, 0x34, 0xc6, 0x14, 0xb1, 0x0a, 0xff, 0x55, 0x25, 0xd0, 0x9d, 0x8d }; -static const u8 enc_output103[] __initconst = { +static const u8 enc_output103[] = { 0x41, 0x40, 0xdf, 0x25, 0xb8, 0xd3, 0x21, 0x94, 0xe7, 0x8e, 0x51, 0xd4, 0x17, 0x38, 0xcc, 0x6d, 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5279,13 +5278,13 @@ static const u8 enc_output103[] __initconst = { 0x86, 0xfb, 0xab, 0x2b, 0x4a, 0x94, 0xf4, 0x7a, 0xa5, 0x6f, 0x0a, 0xea, 0x65, 0xd1, 0x10, 0x08 }; -static const u8 enc_assoc103[] __initconst = { +static const u8 enc_assoc103[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce103[] __initconst = { +static const u8 enc_nonce103[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key103[] __initconst = { +static const u8 enc_key103[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5293,7 +5292,7 @@ static const u8 enc_key103[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input104[] __initconst = { +static const u8 enc_input104[] = { 0xb2, 0x47, 0xa7, 0x47, 0x23, 0x49, 0x1a, 0xac, 0xac, 0xaa, 0xd7, 0x09, 0xc9, 0x1e, 0x93, 0x2b, 0x31, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5305,7 +5304,7 @@ static const u8 enc_input104[] __initconst = { 0x42, 0x89, 0x79, 0x44, 0xc2, 0xa2, 0x8f, 0xa1, 0x76, 0x11, 0xd7, 0xfa, 0x5c, 0x22, 0xad, 0x8f }; -static const u8 enc_output104[] __initconst = { +static const u8 enc_output104[] = { 0xb2, 0x2c, 0x70, 0x68, 0xa5, 0x83, 0xfa, 0x35, 0x0f, 0x85, 0x29, 0xc3, 0x75, 0xf8, 0xeb, 0x88, 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5319,13 +5318,13 @@ static const u8 enc_output104[] __initconst = { 0xa0, 0x19, 0xac, 0x2e, 0xd6, 0x67, 0xe1, 0x7d, 0xa1, 0x6f, 0x0a, 0xfa, 0x19, 0x61, 0x0d, 0x0d }; -static const u8 enc_assoc104[] __initconst = { +static const u8 enc_assoc104[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce104[] __initconst = { +static const u8 enc_nonce104[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key104[] __initconst = { +static const u8 enc_key104[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5333,7 +5332,7 @@ static const u8 enc_key104[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input105[] __initconst = { +static const u8 enc_input105[] = { 0x74, 0x0f, 0x9e, 0x49, 0xf6, 0x10, 0xef, 0xa5, 0x85, 0xb6, 0x59, 0xca, 0x6e, 0xd8, 0xb4, 0x99, 0x2d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5345,7 +5344,7 @@ static const u8 enc_input105[] __initconst = { 0x99, 0x7a, 0xeb, 0x0c, 0x27, 0x95, 0x62, 0x46, 0x69, 0xc3, 0x87, 0xf9, 0x11, 0x6a, 0xc1, 0x8d }; -static const u8 enc_output105[] __initconst = { +static const u8 enc_output105[] = { 0x74, 0x64, 0x49, 0x66, 0x70, 0xda, 0x0f, 0x3c, 0x26, 0x99, 0xa7, 0x00, 0xd2, 0x3e, 0xcc, 0x3a, 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5359,13 +5358,13 @@ static const u8 enc_output105[] __initconst = { 0x73, 0x6e, 0x18, 0x18, 0x16, 0x96, 0xa5, 0x88, 0x9c, 0x31, 0x59, 0xfa, 0xab, 0xab, 0x20, 0xfd }; -static const u8 enc_assoc105[] __initconst = { +static const u8 enc_assoc105[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce105[] __initconst = { +static const u8 enc_nonce105[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key105[] __initconst = { +static const u8 enc_key105[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5373,7 +5372,7 @@ static const u8 enc_key105[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input106[] __initconst = { +static const u8 enc_input106[] = { 0xad, 0xba, 0x5d, 0x10, 0x5b, 0xc8, 0xaa, 0x06, 0x2c, 0x23, 0x36, 0xcb, 0x88, 0x9d, 0xdb, 0xd5, 0x37, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5385,7 +5384,7 @@ static const u8 enc_input106[] __initconst = { 0xcf, 0x2b, 0x22, 0x5d, 0xb1, 0x60, 0x7a, 0x10, 0xe6, 0xd5, 0x40, 0x1e, 0x53, 0xb4, 0x2a, 0x8d }; -static const u8 enc_output106[] __initconst = { +static const u8 enc_output106[] = { 0xad, 0xd1, 0x8a, 0x3f, 0xdd, 0x02, 0x4a, 0x9f, 0x8f, 0x0c, 0xc8, 0x01, 0x34, 0x7b, 0xa3, 0x76, 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5399,13 +5398,13 @@ static const u8 enc_output106[] __initconst = { 0xba, 0xd5, 0x8f, 0x10, 0xa9, 0x1e, 0x6a, 0x88, 0x9a, 0xba, 0x32, 0xfd, 0x17, 0xd8, 0x33, 0x1a }; -static const u8 enc_assoc106[] __initconst = { +static const u8 enc_assoc106[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce106[] __initconst = { +static const u8 enc_nonce106[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key106[] __initconst = { +static const u8 enc_key106[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5413,7 +5412,7 @@ static const u8 enc_key106[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input107[] __initconst = { +static const u8 enc_input107[] = { 0xfe, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xc0, 0x01, 0xed, 0xc5, 0xda, 0x44, 0x2e, 0x71, @@ -5427,7 +5426,7 @@ static const u8 enc_input107[] __initconst = { 0x00, 0x26, 0x6e, 0xa1, 0xe4, 0x36, 0x44, 0xa3, 0x4d, 0x8d, 0xd1, 0xdc, 0x93, 0xf2, 0xfa, 0x13 }; -static const u8 enc_output107[] __initconst = { +static const u8 enc_output107[] = { 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x47, 0xc3, 0x27, 0xcc, 0x36, 0x5d, 0x08, 0x87, @@ -5443,13 +5442,13 @@ static const u8 enc_output107[] __initconst = { 0xd6, 0x8c, 0xe1, 0x74, 0x07, 0x9a, 0xdd, 0x02, 0x8d, 0xd0, 0x5c, 0xf8, 0x14, 0x63, 0x04, 0x88 }; -static const u8 enc_assoc107[] __initconst = { +static const u8 enc_assoc107[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce107[] __initconst = { +static const u8 enc_nonce107[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key107[] __initconst = { +static const u8 enc_key107[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5457,7 +5456,7 @@ static const u8 enc_key107[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input108[] __initconst = { +static const u8 enc_input108[] = { 0xb5, 0x13, 0xb0, 0x6a, 0xb9, 0xac, 0x14, 0x43, 0x5a, 0xcb, 0x8a, 0xa3, 0xa3, 0x7a, 0xfd, 0xb6, 0x54, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5469,7 +5468,7 @@ static const u8 enc_input108[] __initconst = { 0xb9, 0xc2, 0x7c, 0x30, 0x28, 0xaa, 0x8d, 0x69, 0xef, 0x06, 0xaf, 0xc0, 0xb5, 0x9e, 0xda, 0x8e }; -static const u8 enc_output108[] __initconst = { +static const u8 enc_output108[] = { 0xb5, 0x78, 0x67, 0x45, 0x3f, 0x66, 0xf4, 0xda, 0xf9, 0xe4, 0x74, 0x69, 0x1f, 0x9c, 0x85, 0x15, 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5483,13 +5482,13 @@ static const u8 enc_output108[] __initconst = { 0xaa, 0x48, 0xa3, 0x88, 0x7d, 0x4b, 0x05, 0x96, 0x99, 0xc2, 0xfd, 0xf9, 0xc6, 0x78, 0x7e, 0x0a }; -static const u8 enc_assoc108[] __initconst = { +static const u8 enc_assoc108[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce108[] __initconst = { +static const u8 enc_nonce108[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key108[] __initconst = { +static const u8 enc_key108[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5497,7 +5496,7 @@ static const u8 enc_key108[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input109[] __initconst = { +static const u8 enc_input109[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xd4, 0xf1, 0x09, 0xe8, 0x14, 0xce, 0xa8, 0x5a, @@ -5511,7 +5510,7 @@ static const u8 enc_input109[] __initconst = { 0x1b, 0x64, 0x89, 0xba, 0x84, 0xd8, 0xf5, 0x59, 0x82, 0x9e, 0xd9, 0xbd, 0xa2, 0x29, 0x0f, 0x16 }; -static const u8 enc_output109[] __initconst = { +static const u8 enc_output109[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x53, 0x33, 0xc3, 0xe1, 0xf8, 0xd7, 0x8e, 0xac, @@ -5527,13 +5526,13 @@ static const u8 enc_output109[] __initconst = { 0xb9, 0x36, 0xa8, 0x17, 0xf2, 0x21, 0x1a, 0xf1, 0x29, 0xe2, 0xcf, 0x16, 0x0f, 0xd4, 0x2b, 0xcb }; -static const u8 enc_assoc109[] __initconst = { +static const u8 enc_assoc109[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce109[] __initconst = { +static const u8 enc_nonce109[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key109[] __initconst = { +static const u8 enc_key109[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5541,7 +5540,7 @@ static const u8 enc_key109[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input110[] __initconst = { +static const u8 enc_input110[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xdf, 0x4c, 0x62, 0x03, 0x2d, 0x41, 0x19, 0xb5, @@ -5555,7 +5554,7 @@ static const u8 enc_input110[] __initconst = { 0xb2, 0xa0, 0xc1, 0x84, 0x4b, 0x4e, 0x35, 0xd4, 0x1e, 0x5d, 0xa2, 0x10, 0xf6, 0x2f, 0x84, 0x12 }; -static const u8 enc_output110[] __initconst = { +static const u8 enc_output110[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x58, 0x8e, 0xa8, 0x0a, 0xc1, 0x58, 0x3f, 0x43, @@ -5571,13 +5570,13 @@ static const u8 enc_output110[] __initconst = { 0x9f, 0x7a, 0xc4, 0x35, 0x1f, 0x6b, 0x91, 0xe6, 0x30, 0x97, 0xa7, 0x13, 0x11, 0x5d, 0x05, 0xbe }; -static const u8 enc_assoc110[] __initconst = { +static const u8 enc_assoc110[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce110[] __initconst = { +static const u8 enc_nonce110[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key110[] __initconst = { +static const u8 enc_key110[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5585,7 +5584,7 @@ static const u8 enc_key110[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input111[] __initconst = { +static const u8 enc_input111[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x13, 0xf8, 0x0a, 0x00, 0x6d, 0xc1, 0xbb, 0xda, @@ -5599,7 +5598,7 @@ static const u8 enc_input111[] __initconst = { 0x2b, 0x6c, 0x89, 0x1d, 0x37, 0xc7, 0xe1, 0x1a, 0x56, 0x41, 0x91, 0x9c, 0x49, 0x4d, 0x95, 0x16 }; -static const u8 enc_output111[] __initconst = { +static const u8 enc_output111[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x94, 0x3a, 0xc0, 0x09, 0x81, 0xd8, 0x9d, 0x2c, @@ -5615,13 +5614,13 @@ static const u8 enc_output111[] __initconst = { 0x9a, 0x18, 0xa8, 0x28, 0x07, 0x02, 0x69, 0xf4, 0x47, 0x00, 0xd0, 0x09, 0xe7, 0x17, 0x1c, 0xc9 }; -static const u8 enc_assoc111[] __initconst = { +static const u8 enc_assoc111[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce111[] __initconst = { +static const u8 enc_nonce111[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key111[] __initconst = { +static const u8 enc_key111[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5629,7 +5628,7 @@ static const u8 enc_key111[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input112[] __initconst = { +static const u8 enc_input112[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x82, 0xe5, 0x9b, 0x45, 0x82, 0x91, 0x50, 0x38, @@ -5643,7 +5642,7 @@ static const u8 enc_input112[] __initconst = { 0xfe, 0x55, 0xf9, 0x2a, 0xdc, 0x08, 0xb5, 0xaa, 0x95, 0x48, 0xa9, 0x2d, 0x63, 0xaf, 0xe1, 0x13 }; -static const u8 enc_output112[] __initconst = { +static const u8 enc_output112[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x05, 0x27, 0x51, 0x4c, 0x6e, 0x88, 0x76, 0xce, @@ -5659,13 +5658,13 @@ static const u8 enc_output112[] __initconst = { 0xb4, 0x36, 0xa8, 0x2b, 0x93, 0xd5, 0x55, 0xf7, 0x43, 0x00, 0xd0, 0x19, 0x9b, 0xa7, 0x18, 0xce }; -static const u8 enc_assoc112[] __initconst = { +static const u8 enc_assoc112[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce112[] __initconst = { +static const u8 enc_nonce112[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key112[] __initconst = { +static const u8 enc_key112[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5673,7 +5672,7 @@ static const u8 enc_key112[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input113[] __initconst = { +static const u8 enc_input113[] = { 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0xf1, 0xd1, 0x28, 0x87, 0xb7, 0x21, 0x69, 0x86, @@ -5687,7 +5686,7 @@ static const u8 enc_input113[] __initconst = { 0xef, 0xe3, 0x69, 0x79, 0xed, 0x9e, 0x7d, 0x3e, 0xc9, 0x52, 0x41, 0x4e, 0x49, 0xb1, 0x30, 0x16 }; -static const u8 enc_output113[] __initconst = { +static const u8 enc_output113[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x76, 0x13, 0xe2, 0x8e, 0x5b, 0x38, 0x4f, 0x70, @@ -5703,13 +5702,13 @@ static const u8 enc_output113[] __initconst = { 0xce, 0x54, 0xa8, 0x2e, 0x1f, 0xa9, 0x42, 0xfa, 0x3f, 0x00, 0xd0, 0x29, 0x4f, 0x37, 0x15, 0xd3 }; -static const u8 enc_assoc113[] __initconst = { +static const u8 enc_assoc113[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce113[] __initconst = { +static const u8 enc_nonce113[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key113[] __initconst = { +static const u8 enc_key113[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5717,7 +5716,7 @@ static const u8 enc_key113[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input114[] __initconst = { +static const u8 enc_input114[] = { 0xcb, 0xf1, 0xda, 0x9e, 0x0b, 0xa9, 0x37, 0x73, 0x74, 0xe6, 0x9e, 0x1c, 0x0e, 0x60, 0x0c, 0xfc, 0x34, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5729,7 +5728,7 @@ static const u8 enc_input114[] __initconst = { 0x66, 0x68, 0xdb, 0xc8, 0xf5, 0xf2, 0x0e, 0xf2, 0xb3, 0xf3, 0x8f, 0x00, 0xe2, 0x03, 0x17, 0x88 }; -static const u8 enc_output114[] __initconst = { +static const u8 enc_output114[] = { 0xcb, 0x9a, 0x0d, 0xb1, 0x8d, 0x63, 0xd7, 0xea, 0xd7, 0xc9, 0x60, 0xd6, 0xb2, 0x86, 0x74, 0x5f, 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5743,13 +5742,13 @@ static const u8 enc_output114[] __initconst = { 0x23, 0x83, 0xab, 0x0b, 0x79, 0x92, 0x05, 0x69, 0x9b, 0x51, 0x0a, 0xa7, 0x09, 0xbf, 0x31, 0xf1 }; -static const u8 enc_assoc114[] __initconst = { +static const u8 enc_assoc114[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce114[] __initconst = { +static const u8 enc_nonce114[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key114[] __initconst = { +static const u8 enc_key114[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5757,7 +5756,7 @@ static const u8 enc_key114[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input115[] __initconst = { +static const u8 enc_input115[] = { 0x8f, 0x27, 0x86, 0x94, 0xc4, 0xe9, 0xda, 0xeb, 0xd5, 0x8d, 0x3e, 0x5b, 0x96, 0x6e, 0x8b, 0x68, 0x42, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, @@ -5769,7 +5768,7 @@ static const u8 enc_input115[] __initconst = { 0xde, 0x04, 0x9a, 0x00, 0xa8, 0x64, 0x06, 0x4b, 0xbc, 0xd4, 0x6f, 0xe4, 0xe4, 0x5b, 0x42, 0x8f }; -static const u8 enc_output115[] __initconst = { +static const u8 enc_output115[] = { 0x8f, 0x4c, 0x51, 0xbb, 0x42, 0x23, 0x3a, 0x72, 0x76, 0xa2, 0xc0, 0x91, 0x2a, 0x88, 0xf3, 0xcb, 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -5783,13 +5782,13 @@ static const u8 enc_output115[] __initconst = { 0x8b, 0xfb, 0xab, 0x17, 0xa9, 0xe0, 0xb8, 0x74, 0x8b, 0x51, 0x0a, 0xe7, 0xd9, 0xfd, 0x23, 0x05 }; -static const u8 enc_assoc115[] __initconst = { +static const u8 enc_assoc115[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce115[] __initconst = { +static const u8 enc_nonce115[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key115[] __initconst = { +static const u8 enc_key115[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5797,7 +5796,7 @@ static const u8 enc_key115[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input116[] __initconst = { +static const u8 enc_input116[] = { 0xd5, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x9a, 0x22, 0xd7, 0x0a, 0x48, 0xe2, 0x4f, 0xdd, @@ -5811,7 +5810,7 @@ static const u8 enc_input116[] __initconst = { 0x3f, 0x91, 0xf8, 0xe7, 0xc7, 0xb1, 0x96, 0x25, 0x64, 0x61, 0x9c, 0x5e, 0x7e, 0x9b, 0xf6, 0x13 }; -static const u8 enc_output116[] __initconst = { +static const u8 enc_output116[] = { 0xd5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1d, 0xe0, 0x1d, 0x03, 0xa4, 0xfb, 0x69, 0x2b, @@ -5827,13 +5826,13 @@ static const u8 enc_output116[] __initconst = { 0x49, 0xbc, 0x6e, 0x9f, 0xc5, 0x1c, 0x4d, 0x50, 0x30, 0x36, 0x64, 0x4d, 0x84, 0x27, 0x73, 0xd2 }; -static const u8 enc_assoc116[] __initconst = { +static const u8 enc_assoc116[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce116[] __initconst = { +static const u8 enc_nonce116[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key116[] __initconst = { +static const u8 enc_key116[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5841,7 +5840,7 @@ static const u8 enc_key116[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input117[] __initconst = { +static const u8 enc_input117[] = { 0xdb, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x75, 0xd5, 0x64, 0x3a, 0xa5, 0xaf, 0x93, 0x4d, @@ -5855,7 +5854,7 @@ static const u8 enc_input117[] __initconst = { 0x28, 0x3f, 0x6b, 0x32, 0x18, 0x07, 0x5f, 0xc9, 0x5f, 0x6b, 0xb4, 0xff, 0x45, 0x6d, 0xc1, 0x11 }; -static const u8 enc_output117[] __initconst = { +static const u8 enc_output117[] = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf2, 0x17, 0xae, 0x33, 0x49, 0xb6, 0xb5, 0xbb, @@ -5871,13 +5870,13 @@ static const u8 enc_output117[] __initconst = { 0x63, 0xda, 0x6e, 0xa2, 0x51, 0xf0, 0x39, 0x53, 0x2c, 0x36, 0x64, 0x5d, 0x38, 0xb7, 0x6f, 0xd7 }; -static const u8 enc_assoc117[] __initconst = { +static const u8 enc_assoc117[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce117[] __initconst = { +static const u8 enc_nonce117[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key117[] __initconst = { +static const u8 enc_key117[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5885,7 +5884,7 @@ static const u8 enc_key117[] __initconst = { }; /* wycheproof - edge case intermediate sums in poly1305 */ -static const u8 enc_input118[] __initconst = { +static const u8 enc_input118[] = { 0x93, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, 0x62, 0x48, 0x39, 0x60, 0x42, 0x16, 0xe4, 0x03, @@ -5899,7 +5898,7 @@ static const u8 enc_input118[] __initconst = { 0x90, 0xec, 0xf2, 0x1a, 0x04, 0xe6, 0x30, 0x85, 0x8b, 0xb6, 0x56, 0x52, 0xb5, 0xb1, 0x80, 0x16 }; -static const u8 enc_output118[] __initconst = { +static const u8 enc_output118[] = { 0x93, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xe5, 0x8a, 0xf3, 0x69, 0xae, 0x0f, 0xc2, 0xf5, @@ -5915,13 +5914,13 @@ static const u8 enc_output118[] __initconst = { 0x73, 0xeb, 0x27, 0x24, 0xb5, 0xc4, 0x05, 0xf0, 0x4d, 0x00, 0xd0, 0xf1, 0x58, 0x40, 0xa1, 0xc1 }; -static const u8 enc_assoc118[] __initconst = { +static const u8 enc_assoc118[] = { 0xff, 0xff, 0xff, 0xff }; -static const u8 enc_nonce118[] __initconst = { +static const u8 enc_nonce118[] = { 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 }; -static const u8 enc_key118[] __initconst = { +static const u8 enc_key118[] = { 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, @@ -5929,7 +5928,7 @@ static const u8 enc_key118[] __initconst = { }; static const struct chacha20poly1305_testvec -chacha20poly1305_enc_vectors[] __initconst = { +chacha20poly1305_enc_vectors[] = { { enc_input001, enc_output001, enc_assoc001, enc_nonce001, enc_key001, sizeof(enc_input001), sizeof(enc_assoc001), sizeof(enc_nonce001) }, { enc_input002, enc_output002, enc_assoc002, enc_nonce002, enc_key002, @@ -6168,7 +6167,7 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input118), sizeof(enc_assoc118), sizeof(enc_nonce118) } }; -static const u8 dec_input001[] __initconst = { +static const u8 dec_input001[] = { 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, @@ -6206,7 +6205,7 @@ static const u8 dec_input001[] __initconst = { 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, 0x38 }; -static const u8 dec_output001[] __initconst = { +static const u8 dec_output001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -6242,95 +6241,95 @@ static const u8 dec_output001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 dec_assoc001[] __initconst = { +static const u8 dec_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 dec_nonce001[] __initconst = { +static const u8 dec_nonce001[] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 }; -static const u8 dec_key001[] __initconst = { +static const u8 dec_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 }; -static const u8 dec_input002[] __initconst = { +static const u8 dec_input002[] = { 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 }; -static const u8 dec_output002[] __initconst = { }; -static const u8 dec_assoc002[] __initconst = { }; -static const u8 dec_nonce002[] __initconst = { +static const u8 dec_output002[] = { }; +static const u8 dec_assoc002[] = { }; +static const u8 dec_nonce002[] = { 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e }; -static const u8 dec_key002[] __initconst = { +static const u8 dec_key002[] = { 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 }; -static const u8 dec_input003[] __initconst = { +static const u8 dec_input003[] = { 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 }; -static const u8 dec_output003[] __initconst = { }; -static const u8 dec_assoc003[] __initconst = { +static const u8 dec_output003[] = { }; +static const u8 dec_assoc003[] = { 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b }; -static const u8 dec_nonce003[] __initconst = { +static const u8 dec_nonce003[] = { 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d }; -static const u8 dec_key003[] __initconst = { +static const u8 dec_key003[] = { 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d }; -static const u8 dec_input004[] __initconst = { +static const u8 dec_input004[] = { 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, 0x89 }; -static const u8 dec_output004[] __initconst = { +static const u8 dec_output004[] = { 0xa4 }; -static const u8 dec_assoc004[] __initconst = { +static const u8 dec_assoc004[] = { 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 }; -static const u8 dec_nonce004[] __initconst = { +static const u8 dec_nonce004[] = { 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 }; -static const u8 dec_key004[] __initconst = { +static const u8 dec_key004[] = { 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e }; -static const u8 dec_input005[] __initconst = { +static const u8 dec_input005[] = { 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, 0xac }; -static const u8 dec_output005[] __initconst = { +static const u8 dec_output005[] = { 0x2d }; -static const u8 dec_assoc005[] __initconst = { }; -static const u8 dec_nonce005[] __initconst = { +static const u8 dec_assoc005[] = { }; +static const u8 dec_nonce005[] = { 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 }; -static const u8 dec_key005[] __initconst = { +static const u8 dec_key005[] = { 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 }; -static const u8 dec_input006[] __initconst = { +static const u8 dec_input006[] = { 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, @@ -6351,7 +6350,7 @@ static const u8 dec_input006[] __initconst = { 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, 0xeb }; -static const u8 dec_output006[] __initconst = { +static const u8 dec_output006[] = { 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, @@ -6370,20 +6369,20 @@ static const u8 dec_output006[] __initconst = { 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, 0x8f }; -static const u8 dec_assoc006[] __initconst = { +static const u8 dec_assoc006[] = { 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b }; -static const u8 dec_nonce006[] __initconst = { +static const u8 dec_nonce006[] = { 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c }; -static const u8 dec_key006[] __initconst = { +static const u8 dec_key006[] = { 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 }; -static const u8 dec_input007[] __initconst = { +static const u8 dec_input007[] = { 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, @@ -6419,7 +6418,7 @@ static const u8 dec_input007[] __initconst = { 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 }; -static const u8 dec_output007[] __initconst = { +static const u8 dec_output007[] = { 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, @@ -6453,18 +6452,18 @@ static const u8 dec_output007[] __initconst = { 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 }; -static const u8 dec_assoc007[] __initconst = { }; -static const u8 dec_nonce007[] __initconst = { +static const u8 dec_assoc007[] = { }; +static const u8 dec_nonce007[] = { 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 }; -static const u8 dec_key007[] __initconst = { +static const u8 dec_key007[] = { 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 }; -static const u8 dec_input008[] __initconst = { +static const u8 dec_input008[] = { 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, @@ -6532,7 +6531,7 @@ static const u8 dec_input008[] __initconst = { 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 }; -static const u8 dec_output008[] __initconst = { +static const u8 dec_output008[] = { 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, @@ -6598,18 +6597,18 @@ static const u8 dec_output008[] __initconst = { 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 }; -static const u8 dec_assoc008[] __initconst = { }; -static const u8 dec_nonce008[] __initconst = { +static const u8 dec_assoc008[] = { }; +static const u8 dec_nonce008[] = { 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 }; -static const u8 dec_key008[] __initconst = { +static const u8 dec_key008[] = { 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba }; -static const u8 dec_input009[] __initconst = { +static const u8 dec_input009[] = { 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, @@ -6678,7 +6677,7 @@ static const u8 dec_input009[] __initconst = { 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, 0xae }; -static const u8 dec_output009[] __initconst = { +static const u8 dec_output009[] = { 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, @@ -6745,21 +6744,21 @@ static const u8 dec_output009[] __initconst = { 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, 0x65 }; -static const u8 dec_assoc009[] __initconst = { +static const u8 dec_assoc009[] = { 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, 0xef }; -static const u8 dec_nonce009[] __initconst = { +static const u8 dec_nonce009[] = { 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 }; -static const u8 dec_key009[] __initconst = { +static const u8 dec_key009[] = { 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b }; -static const u8 dec_input010[] __initconst = { +static const u8 dec_input010[] = { 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, @@ -6891,7 +6890,7 @@ static const u8 dec_input010[] __initconst = { 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 }; -static const u8 dec_output010[] __initconst = { +static const u8 dec_output010[] = { 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, @@ -7021,21 +7020,21 @@ static const u8 dec_output010[] __initconst = { 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f }; -static const u8 dec_assoc010[] __initconst = { +static const u8 dec_assoc010[] = { 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 }; -static const u8 dec_nonce010[] __initconst = { +static const u8 dec_nonce010[] = { 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 }; -static const u8 dec_key010[] __initconst = { +static const u8 dec_key010[] = { 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 }; -static const u8 dec_input011[] __initconst = { +static const u8 dec_input011[] = { 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, @@ -7281,7 +7280,7 @@ static const u8 dec_input011[] __initconst = { 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, 0x2b, 0xdf, 0xcd, 0xf9, 0x3c }; -static const u8 dec_output011[] __initconst = { +static const u8 dec_output011[] = { 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, @@ -7525,20 +7524,20 @@ static const u8 dec_output011[] __initconst = { 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, 0x10, 0x1e, 0xbf, 0xec, 0xa8 }; -static const u8 dec_assoc011[] __initconst = { +static const u8 dec_assoc011[] = { 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 }; -static const u8 dec_nonce011[] __initconst = { +static const u8 dec_nonce011[] = { 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa }; -static const u8 dec_key011[] __initconst = { +static const u8 dec_key011[] = { 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 }; -static const u8 dec_input012[] __initconst = { +static const u8 dec_input012[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -7794,7 +7793,7 @@ static const u8 dec_input012[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd6 }; -static const u8 dec_output012[] __initconst = { +static const u8 dec_output012[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -8048,7 +8047,7 @@ static const u8 dec_output012[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 dec_assoc012[] __initconst = { +static const u8 dec_assoc012[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -8058,17 +8057,17 @@ static const u8 dec_assoc012[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 dec_nonce012[] __initconst = { +static const u8 dec_nonce012[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 dec_key012[] __initconst = { +static const u8 dec_key012[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; -static const u8 dec_input013[] __initconst = { +static const u8 dec_input013[] = { 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, @@ -8324,7 +8323,7 @@ static const u8 dec_input013[] __initconst = { 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, 0x70, 0xcf, 0xd7 }; -static const u8 dec_output013[] __initconst = { +static const u8 dec_output013[] = { 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, @@ -8578,7 +8577,7 @@ static const u8 dec_output013[] __initconst = { 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, 0x78, 0xec, 0x00 }; -static const u8 dec_assoc013[] __initconst = { +static const u8 dec_assoc013[] = { 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, @@ -8588,10 +8587,10 @@ static const u8 dec_assoc013[] __initconst = { 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 }; -static const u8 dec_nonce013[] __initconst = { +static const u8 dec_nonce013[] = { 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 }; -static const u8 dec_key013[] __initconst = { +static const u8 dec_key013[] = { 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, @@ -8599,7 +8598,7 @@ static const u8 dec_key013[] __initconst = { }; static const struct chacha20poly1305_testvec -chacha20poly1305_dec_vectors[] __initconst = { +chacha20poly1305_dec_vectors[] = { { dec_input001, dec_output001, dec_assoc001, dec_nonce001, dec_key001, sizeof(dec_input001), sizeof(dec_assoc001), sizeof(dec_nonce001) }, { dec_input002, dec_output002, dec_assoc002, dec_nonce002, dec_key002, @@ -8629,7 +8628,7 @@ chacha20poly1305_dec_vectors[] __initconst = { true } }; -static const u8 xenc_input001[] __initconst = { +static const u8 xenc_input001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -8665,7 +8664,7 @@ static const u8 xenc_input001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 xenc_output001[] __initconst = { +static const u8 xenc_output001[] = { 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, @@ -8703,16 +8702,16 @@ static const u8 xenc_output001[] __initconst = { 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, 0x9c }; -static const u8 xenc_assoc001[] __initconst = { +static const u8 xenc_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 xenc_nonce001[] __initconst = { +static const u8 xenc_nonce001[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; -static const u8 xenc_key001[] __initconst = { +static const u8 xenc_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, @@ -8720,12 +8719,12 @@ static const u8 xenc_key001[] __initconst = { }; static const struct chacha20poly1305_testvec -xchacha20poly1305_enc_vectors[] __initconst = { +xchacha20poly1305_enc_vectors[] = { { xenc_input001, xenc_output001, xenc_assoc001, xenc_nonce001, xenc_key001, sizeof(xenc_input001), sizeof(xenc_assoc001), sizeof(xenc_nonce001) } }; -static const u8 xdec_input001[] __initconst = { +static const u8 xdec_input001[] = { 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, @@ -8763,7 +8762,7 @@ static const u8 xdec_input001[] __initconst = { 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, 0x9c }; -static const u8 xdec_output001[] __initconst = { +static const u8 xdec_output001[] = { 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, @@ -8799,16 +8798,16 @@ static const u8 xdec_output001[] __initconst = { 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, 0x9d }; -static const u8 xdec_assoc001[] __initconst = { +static const u8 xdec_assoc001[] = { 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x91 }; -static const u8 xdec_nonce001[] __initconst = { +static const u8 xdec_nonce001[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 }; -static const u8 xdec_key001[] __initconst = { +static const u8 xdec_key001[] = { 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, @@ -8816,15 +8815,15 @@ static const u8 xdec_key001[] __initconst = { }; static const struct chacha20poly1305_testvec -xchacha20poly1305_dec_vectors[] __initconst = { +xchacha20poly1305_dec_vectors[] = { { xdec_input001, xdec_output001, xdec_assoc001, xdec_nonce001, xdec_key001, sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } }; -/* This is for the selftests-only, since it is only useful for the purpose of +/* This is for the tests only, since it is only useful for the purpose of * testing the underlying primitives and interactions. */ -static void __init +static void chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, const u8 nonce[12], @@ -8858,11 +8857,12 @@ chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, poly1305_final(&poly1305_state, dst + src_len); } -static void __init -chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, - const u8 *ad, const size_t ad_len, - const u8 *nonce, const size_t nonce_len, - const u8 key[CHACHA20POLY1305_KEY_SIZE]) +static void +chacha20poly1305_test_encrypt(struct kunit *test, u8 *dst, + const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 *nonce, const size_t nonce_len, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) { if (nonce_len == 8) chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, @@ -8871,10 +8871,10 @@ chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, chacha20poly1305_encrypt_bignonce(dst, src, src_len, ad, ad_len, nonce, key); else - BUG(); + KUNIT_FAIL(test, "bad nonce_len: %zu", nonce_len); } -static bool __init +static bool decryption_success(bool func_ret, bool expect_failure, int memcmp_result) { if (expect_failure) @@ -8882,25 +8882,23 @@ decryption_success(bool func_ret, bool expect_failure, int memcmp_result) return func_ret && !memcmp_result; } -bool __init chacha20poly1305_selftest(void) +static void test_chacha20poly1305(struct kunit *test) { enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; size_t i, j, k, total_len; u8 *computed_output = NULL, *input = NULL; - bool success = true, ret; + bool ret; struct scatterlist sg_src[3]; - computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - if (!computed_output || !input) { - pr_err("chacha20poly1305 self-test malloc: FAIL\n"); - success = false; - goto out; - } + computed_output = kunit_kmalloc(test, MAXIMUM_TEST_BUFFER_LEN, + GFP_KERNEL); + input = kunit_kmalloc(test, MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, computed_output); + KUNIT_ASSERT_NOT_NULL(test, input); for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); - chacha20poly1305_selftest_encrypt(computed_output, + chacha20poly1305_test_encrypt(test, computed_output, chacha20poly1305_enc_vectors[i].input, chacha20poly1305_enc_vectors[i].ilen, chacha20poly1305_enc_vectors[i].assoc, @@ -8908,14 +8906,13 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_enc_vectors[i].nonce, chacha20poly1305_enc_vectors[i].nlen, chacha20poly1305_enc_vectors[i].key); - if (memcmp(computed_output, - chacha20poly1305_enc_vectors[i].output, - chacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("chacha20poly1305 encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "chacha20poly1305 encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { @@ -8931,14 +8928,13 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_enc_vectors[i].alen, get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), chacha20poly1305_enc_vectors[i].key); - if (!ret || memcmp(computed_output, - chacha20poly1305_enc_vectors[i].output, - chacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("chacha20poly1305 sg encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + ret && memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "chacha20poly1305 sg encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { @@ -8950,16 +8946,15 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].alen, get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), chacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - chacha20poly1305_dec_vectors[i].failure, + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, chacha20poly1305_dec_vectors[i].failure, memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, chacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("chacha20poly1305 decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + POLY1305_DIGEST_SIZE)), + "chacha20poly1305 decryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { @@ -8973,15 +8968,15 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].alen, get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), chacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - chacha20poly1305_dec_vectors[i].failure, - memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, - chacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, chacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, + chacha20poly1305_dec_vectors[i].output, + chacha20poly1305_dec_vectors[i].ilen - + POLY1305_DIGEST_SIZE)), + "chacha20poly1305 sg decryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_enc_vectors); ++i) { @@ -8993,14 +8988,13 @@ bool __init chacha20poly1305_selftest(void) xchacha20poly1305_enc_vectors[i].alen, xchacha20poly1305_enc_vectors[i].nonce, xchacha20poly1305_enc_vectors[i].key); - if (memcmp(computed_output, - xchacha20poly1305_enc_vectors[i].output, - xchacha20poly1305_enc_vectors[i].ilen + - POLY1305_DIGEST_SIZE)) { - pr_err("xchacha20poly1305 encryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + KUNIT_EXPECT_TRUE_MSG( + test, + memcmp(computed_output, + xchacha20poly1305_enc_vectors[i].output, + xchacha20poly1305_enc_vectors[i].ilen + + POLY1305_DIGEST_SIZE) == 0, + "xchacha20poly1305 encryption test %zu: FAIL", i + 1); } for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { @@ -9012,16 +9006,15 @@ bool __init chacha20poly1305_selftest(void) xchacha20poly1305_dec_vectors[i].alen, xchacha20poly1305_dec_vectors[i].nonce, xchacha20poly1305_dec_vectors[i].key); - if (!decryption_success(ret, - xchacha20poly1305_dec_vectors[i].failure, + KUNIT_EXPECT_TRUE_MSG( + test, + decryption_success( + ret, xchacha20poly1305_dec_vectors[i].failure, memcmp(computed_output, xchacha20poly1305_dec_vectors[i].output, xchacha20poly1305_dec_vectors[i].ilen - - POLY1305_DIGEST_SIZE))) { - pr_err("xchacha20poly1305 decryption self-test %zu: FAIL\n", - i + 1); - success = false; - } + POLY1305_DIGEST_SIZE)), + "xchacha20poly1305 decryption test %zu: FAIL", i + 1); } for (total_len = POLY1305_DIGEST_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) @@ -9067,16 +9060,26 @@ bool __init chacha20poly1305_selftest(void) continue; chunkfail: - pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", - total_len, i, j); - success = false; + KUNIT_FAIL( + test, + "chacha20poly1305 chunked test %zu/%zu/%zu: FAIL\n", + total_len, i, j); } } } - -out: - kfree(computed_output); - kfree(input); - return success; } + +static struct kunit_case chacha20poly1305_test_cases[] = { + KUNIT_CASE(test_chacha20poly1305), + {}, +}; + +static struct kunit_suite chacha20poly1305_test_suite = { + .name = "chacha20poly1305", + .test_cases = chacha20poly1305_test_cases, +}; +kunit_test_suite(chacha20poly1305_test_suite); + +MODULE_DESCRIPTION("KUnit tests for ChaCha20Poly1305"); +MODULE_LICENSE("GPL"); From 1aa82df3eb4d1a28c02a9d0062c6ed0db1a59bac Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Mar 2026 19:44:14 -0700 Subject: [PATCH 54/66] lib/crypto: aescfb: Don't disable IRQs during AES block encryption aes_encrypt() now uses AES instructions when available instead of always using table-based code. AES instructions are constant-time and don't benefit from disabling IRQs as a constant-time hardening measure. In fact, on two architectures (arm and riscv) disabling IRQs is counterproductive because it prevents the AES instructions from being used. (See the may_use_simd() implementation on those architectures.) Therefore, let's remove the IRQ disabling/enabling and leave the choice of constant-time hardening measures to the AES library code. Note that currently the arm table-based AES code (which runs on arm kernels that don't have ARMv8 CE) disables IRQs, while the generic table-based AES code does not. So this does technically regress in constant-time hardening when that generic code is used. But as discussed in commit a22fd0e3c495 ("lib/crypto: aes: Introduce improved AES library") I think just leaving IRQs enabled is the right choice. Disabling them is slow and can cause problems, and AES instructions (which modern CPUs have) solve the problem in a much better way anyway. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260331024414.51545-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/aescfb.c | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c index 147e5211728f..e38848d101e3 100644 --- a/lib/crypto/aescfb.c +++ b/lib/crypto/aescfb.c @@ -9,25 +9,6 @@ #include #include #include -#include - -static void aescfb_encrypt_block(const struct aes_enckey *key, void *dst, - const void *src) -{ - unsigned long flags; - - /* - * In AES-CFB, the AES encryption operates on known 'plaintext' (the IV - * and ciphertext), making it susceptible to timing attacks on the - * encryption key. The AES library already mitigates this risk to some - * extent by pulling the entire S-box into the caches before doing any - * substitutions, but this strategy is more effective when running with - * interrupts disabled. - */ - local_irq_save(flags); - aes_encrypt(key, dst, src); - local_irq_restore(flags); -} /** * aescfb_encrypt - Perform AES-CFB encryption on a block of data @@ -45,7 +26,7 @@ void aescfb_encrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, const u8 *v = iv; while (len > 0) { - aescfb_encrypt_block(key, ks, v); + aes_encrypt(key, ks, v); crypto_xor_cpy(dst, src, ks, min(len, AES_BLOCK_SIZE)); v = dst; @@ -72,7 +53,7 @@ void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, { u8 ks[2][AES_BLOCK_SIZE]; - aescfb_encrypt_block(key, ks[0], iv); + aes_encrypt(key, ks[0], iv); for (int i = 0; len > 0; i ^= 1) { if (len > AES_BLOCK_SIZE) @@ -81,7 +62,7 @@ void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, * performing the XOR, as that may update in place and * overwrite the ciphertext. */ - aescfb_encrypt_block(key, ks[!i], src); + aes_encrypt(key, ks[!i], src); crypto_xor_cpy(dst, src, ks[i], min(len, AES_BLOCK_SIZE)); From 8f45af945fce60c8656b5113d80af7fe221c88f5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Mar 2026 19:44:30 -0700 Subject: [PATCH 55/66] lib/crypto: aesgcm: Don't disable IRQs during AES block encryption aes_encrypt() now uses AES instructions when available instead of always using table-based code. AES instructions are constant-time and don't benefit from disabling IRQs as a constant-time hardening measure. In fact, on two architectures (arm and riscv) disabling IRQs is counterproductive because it prevents the AES instructions from being used. (See the may_use_simd() implementation on those architectures.) Therefore, let's remove the IRQ disabling/enabling and leave the choice of constant-time hardening measures to the AES library code. Note that currently the arm table-based AES code (which runs on arm kernels that don't have ARMv8 CE) disables IRQs, while the generic table-based AES code does not. So this does technically regress in constant-time hardening when that generic code is used. But as discussed in commit a22fd0e3c495 ("lib/crypto: aes: Introduce improved AES library") I think just leaving IRQs enabled is the right choice. Disabling them is slow and can cause problems, and AES instructions (which modern CPUs have) solve the problem in a much better way anyway. Link: https://lore.kernel.org/r/20260331024430.51755-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/aesgcm.c | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c index 8c7e74d2d147..1da31e1f747d 100644 --- a/lib/crypto/aesgcm.c +++ b/lib/crypto/aesgcm.c @@ -9,25 +9,6 @@ #include #include #include -#include - -static void aesgcm_encrypt_block(const struct aes_enckey *key, void *dst, - const void *src) -{ - unsigned long flags; - - /* - * In AES-GCM, both the GHASH key derivation and the CTR mode - * encryption operate on known plaintext, making them susceptible to - * timing attacks on the encryption key. The AES library already - * mitigates this risk to some extent by pulling the entire S-box into - * the caches before doing any substitutions, but this strategy is more - * effective when running with interrupts disabled. - */ - local_irq_save(flags); - aes_encrypt(key, dst, src); - local_irq_restore(flags); -} /** * aesgcm_expandkey - Expands the AES and GHASH keys for the AES-GCM key @@ -53,7 +34,7 @@ int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, return ret; ctx->authsize = authsize; - aesgcm_encrypt_block(&ctx->aes_key, h, h); + aes_encrypt(&ctx->aes_key, h, h); ghash_preparekey(&ctx->ghash_key, h); memzero_explicit(h, sizeof(h)); return 0; @@ -98,7 +79,7 @@ static void aesgcm_mac(const struct aesgcm_ctx *ctx, const u8 *src, int src_len, ghash_final(&ghash, ghash_out); ctr[3] = cpu_to_be32(1); - aesgcm_encrypt_block(&ctx->aes_key, enc_ctr, ctr); + aes_encrypt(&ctx->aes_key, enc_ctr, (const u8 *)ctr); crypto_xor_cpy(authtag, ghash_out, enc_ctr, ctx->authsize); memzero_explicit(ghash_out, sizeof(ghash_out)); @@ -120,7 +101,7 @@ static void aesgcm_crypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, * len', this cannot happen, so no explicit test is necessary. */ ctr[3] = cpu_to_be32(n++); - aesgcm_encrypt_block(&ctx->aes_key, buf, ctr); + aes_encrypt(&ctx->aes_key, buf, (const u8 *)ctr); crypto_xor_cpy(dst, src, buf, min(len, AES_BLOCK_SIZE)); dst += AES_BLOCK_SIZE; From 8aeeb5255d5e0001f2af6786e2a7564fef416acf Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 30 Mar 2026 19:44:38 -0700 Subject: [PATCH 56/66] lib/crypto: Include instead of Since the lib/crypto/ files that include need it only for the transitive inclusion of (and not all the traditional crypto API stuff that the rest of is filled with), replace these inclusions with direct inclusions of . Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260331024438.51783-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/aescfb.c | 2 +- lib/crypto/chacha.c | 2 +- lib/crypto/memneq.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c index e38848d101e3..82cd55436055 100644 --- a/lib/crypto/aescfb.c +++ b/lib/crypto/aescfb.c @@ -6,7 +6,7 @@ */ #include -#include +#include #include #include diff --git a/lib/crypto/chacha.c b/lib/crypto/chacha.c index e0c7cb4af318..86e5d382a4e0 100644 --- a/lib/crypto/chacha.c +++ b/lib/crypto/chacha.c @@ -5,8 +5,8 @@ * Copyright (C) 2015 Martin Willi */ -#include // for crypto_xor_cpy #include +#include #include #include #include diff --git a/lib/crypto/memneq.c b/lib/crypto/memneq.c index 44daacb8cb51..08924acd44bc 100644 --- a/lib/crypto/memneq.c +++ b/lib/crypto/memneq.c @@ -59,7 +59,7 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include +#include #include #include #include @@ -159,7 +159,7 @@ static inline unsigned long __crypto_memneq_16(const void *a, const void *b) /* Compare two areas of memory without leaking timing information, * and with special optimizations for common sizes. Users should * not call this function directly, but should instead use - * crypto_memneq defined in crypto/algapi.h. + * crypto_memneq defined in crypto/utils.h. */ noinline unsigned long __crypto_memneq(const void *a, const void *b, size_t size) From 11d6bc70fff310cf0c4bbfa740144b0e350cd706 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 31 Mar 2026 17:05:40 -0700 Subject: [PATCH 57/66] lib/crypto: arm64/aes: Remove obsolete chunking logic Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the AES-CBC-MAC code accordingly. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260401000548.133151-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/crypto/aes-ce-ccm-glue.c | 13 ++++------- include/crypto/aes.h | 6 ++--- lib/crypto/arm64/aes-modes.S | 8 +++---- lib/crypto/arm64/aes.h | 35 +++++++++++------------------ 4 files changed, 23 insertions(+), 39 deletions(-) diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index 45aed0073283..a304375ce724 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -101,16 +101,11 @@ static u32 ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, u32 blocks = abytes / AES_BLOCK_SIZE; if (macp == AES_BLOCK_SIZE || (!macp && blocks > 0)) { - u32 rem = ce_aes_mac_update(in, rk, rounds, blocks, mac, - macp, enc_after); - u32 adv = (blocks - rem) * AES_BLOCK_SIZE; - + ce_aes_mac_update(in, rk, rounds, blocks, mac, macp, + enc_after); macp = enc_after ? 0 : AES_BLOCK_SIZE; - in += adv; - abytes -= adv; - - if (unlikely(rem)) - macp = 0; + in += blocks * AES_BLOCK_SIZE; + abytes -= blocks * AES_BLOCK_SIZE; } else { u32 l = min(AES_BLOCK_SIZE - macp, abytes); diff --git a/include/crypto/aes.h b/include/crypto/aes.h index 3feb4105c2a2..16fbfd93e2bd 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -230,9 +230,9 @@ asmlinkage void ce_aes_essiv_cbc_encrypt(u8 out[], u8 const in[], asmlinkage void ce_aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], int rounds, int blocks, u8 iv[], u32 const rk2[]); -asmlinkage size_t ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds, - size_t blocks, u8 dg[], int enc_before, - int enc_after); +asmlinkage void ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + size_t blocks, u8 dg[], int enc_before, + int enc_after); #elif defined(CONFIG_PPC) void ppc_expand_key_128(u32 *key_enc, const u8 *key); void ppc_expand_key_192(u32 *key_enc, const u8 *key); diff --git a/lib/crypto/arm64/aes-modes.S b/lib/crypto/arm64/aes-modes.S index fc89cd02b642..10e537317eaf 100644 --- a/lib/crypto/arm64/aes-modes.S +++ b/lib/crypto/arm64/aes-modes.S @@ -817,9 +817,9 @@ AES_FUNC_END(aes_xts_decrypt) #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) /* - * size_t aes_mac_update(u8 const in[], u32 const rk[], int rounds, - * size_t blocks, u8 dg[], int enc_before, - * int enc_after); + * void aes_mac_update(u8 const in[], u32 const rk[], int rounds, + * size_t blocks, u8 dg[], int enc_before, + * int enc_after); */ AES_FUNC_START(aes_mac_update) ld1 {v0.16b}, [x4] /* get dg */ @@ -844,7 +844,6 @@ AES_FUNC_START(aes_mac_update) cbz w5, .Lmacout encrypt_block v0, w2, x1, x7, w8 st1 {v0.16b}, [x4] /* return dg */ - cond_yield .Lmacout, x7, x8 b .Lmacloop4x .Lmac1x: add x3, x3, #4 @@ -863,7 +862,6 @@ AES_FUNC_START(aes_mac_update) .Lmacout: st1 {v0.16b}, [x4] /* return dg */ - mov x0, x3 ret AES_FUNC_END(aes_mac_update) #endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h index 135d3324a30a..9e9e45a6f787 100644 --- a/lib/crypto/arm64/aes.h +++ b/lib/crypto/arm64/aes.h @@ -29,9 +29,9 @@ asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE], asmlinkage u32 __aes_ce_sub(u32 l); asmlinkage void __aes_ce_invert(struct aes_block *out, const struct aes_block *in); -asmlinkage size_t neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds, - size_t blocks, u8 dg[], int enc_before, - int enc_after); +asmlinkage void neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + size_t blocks, u8 dg[], int enc_before, + int enc_after); /* * Expand an AES key using the crypto extensions if supported and usable or @@ -192,25 +192,16 @@ static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], bool enc_after) { if (static_branch_likely(&have_neon) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() { - if (static_branch_likely(&have_aes)) - rem = ce_aes_mac_update( - data, key->k.rndkeys, - key->nrounds, nblocks, h, - enc_before, enc_after); - else - rem = neon_aes_mac_update( - data, key->k.rndkeys, - key->nrounds, nblocks, h, - enc_before, enc_after); - } - data += (nblocks - rem) * AES_BLOCK_SIZE; - nblocks = rem; - enc_before = false; - } while (nblocks); + scoped_ksimd() { + if (static_branch_likely(&have_aes)) + ce_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + else + neon_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + } return true; } return false; From 63fcc765e1a30b7e04d395d7adc440443ae00338 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 31 Mar 2026 17:05:41 -0700 Subject: [PATCH 58/66] lib/crypto: arm64/chacha: Remove obsolete chunking logic Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the ChaCha code accordingly. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260401000548.133151-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm64/chacha.h | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/lib/crypto/arm64/chacha.h b/lib/crypto/arm64/chacha.h index ca8c6a8b0578..c6f8ddf98e2d 100644 --- a/lib/crypto/arm64/chacha.h +++ b/lib/crypto/arm64/chacha.h @@ -36,9 +36,9 @@ asmlinkage void hchacha_block_neon(const struct chacha_state *state, static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); static void chacha_doneon(struct chacha_state *state, u8 *dst, const u8 *src, - int bytes, int nrounds) + unsigned int bytes, int nrounds) { - while (bytes > 0) { + while (bytes) { int l = min(bytes, CHACHA_BLOCK_SIZE * 5); if (l <= CHACHA_BLOCK_SIZE) { @@ -76,16 +76,8 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst, !crypto_simd_usable()) return chacha_crypt_generic(state, dst, src, bytes, nrounds); - do { - unsigned int todo = min_t(unsigned int, bytes, SZ_4K); - - scoped_ksimd() - chacha_doneon(state, dst, src, todo, nrounds); - - bytes -= todo; - src += todo; - dst += todo; - } while (bytes); + scoped_ksimd() + chacha_doneon(state, dst, src, bytes, nrounds); } #define chacha_mod_init_arch chacha_mod_init_arch From d3a5cc5c9237cd6ffd1f84c1af306e315cf0cf3d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 31 Mar 2026 17:05:42 -0700 Subject: [PATCH 59/66] lib/crypto: arm64/gf128hash: Remove obsolete chunking logic Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the GHASH and POLYVAL code accordingly. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260401000548.133151-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm64/gf128hash.h | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/lib/crypto/arm64/gf128hash.h b/lib/crypto/arm64/gf128hash.h index b2c85585b758..1d1179f87f8d 100644 --- a/lib/crypto/arm64/gf128hash.h +++ b/lib/crypto/arm64/gf128hash.h @@ -89,16 +89,8 @@ static void ghash_blocks_arch(struct polyval_elem *acc, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_asimd) && may_use_simd()) { - do { - /* Allow rescheduling every 4 KiB. */ - size_t n = min_t(size_t, nblocks, - 4096 / GHASH_BLOCK_SIZE); - - scoped_ksimd() - pmull_ghash_update_p8(n, acc, data, &key->h); - data += n * GHASH_BLOCK_SIZE; - nblocks -= n; - } while (nblocks); + scoped_ksimd() + pmull_ghash_update_p8(nblocks, acc, data, &key->h); } else { ghash_blocks_generic(acc, &key->h, data, nblocks); } @@ -110,16 +102,8 @@ static void polyval_blocks_arch(struct polyval_elem *acc, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_pmull) && may_use_simd()) { - do { - /* Allow rescheduling every 4 KiB. */ - size_t n = min_t(size_t, nblocks, - 4096 / POLYVAL_BLOCK_SIZE); - - scoped_ksimd() - polyval_blocks_pmull(acc, key, data, n); - data += n * POLYVAL_BLOCK_SIZE; - nblocks -= n; - } while (nblocks); + scoped_ksimd() + polyval_blocks_pmull(acc, key, data, nblocks); } else { polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], data, nblocks); From dec1061f0ae9e00f13b9e141f3b5cae053da1346 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 31 Mar 2026 17:05:43 -0700 Subject: [PATCH 60/66] lib/crypto: arm64/poly1305: Remove obsolete chunking logic Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the Poly1305 code accordingly. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260401000548.133151-5-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm64/poly1305.h | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/crypto/arm64/poly1305.h b/lib/crypto/arm64/poly1305.h index b77669767cd6..3d4bde857699 100644 --- a/lib/crypto/arm64/poly1305.h +++ b/lib/crypto/arm64/poly1305.h @@ -27,17 +27,11 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { if (static_branch_likely(&have_neon) && likely(may_use_simd())) { - do { - unsigned int todo = min_t(unsigned int, len, SZ_4K); - - scoped_ksimd() - poly1305_blocks_neon(state, src, todo, padbit); - - len -= todo; - src += todo; - } while (len); - } else + scoped_ksimd() + poly1305_blocks_neon(state, src, len, padbit); + } else { poly1305_blocks_arm64(state, src, len, padbit); + } } #define poly1305_mod_init_arch poly1305_mod_init_arch From fd5017138ce03f34d0e67758df51e8bb30c0d91b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 31 Mar 2026 17:05:44 -0700 Subject: [PATCH 61/66] lib/crypto: arm64/sha1: Remove obsolete chunking logic Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the SHA-1 code accordingly. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260401000548.133151-6-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm64/sha1-ce-core.S | 14 +++++--------- lib/crypto/arm64/sha1.h | 15 ++++----------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/lib/crypto/arm64/sha1-ce-core.S b/lib/crypto/arm64/sha1-ce-core.S index 8fbd4767f0f0..59d27fda0714 100644 --- a/lib/crypto/arm64/sha1-ce-core.S +++ b/lib/crypto/arm64/sha1-ce-core.S @@ -62,10 +62,10 @@ .endm /* - * size_t __sha1_ce_transform(struct sha1_block_state *state, - * const u8 *data, size_t nblocks); + * void sha1_ce_transform(struct sha1_block_state *state, + * const u8 *data, size_t nblocks); */ -SYM_FUNC_START(__sha1_ce_transform) +SYM_FUNC_START(sha1_ce_transform) /* load round constants */ loadrc k0.4s, 0x5a827999, w6 loadrc k1.4s, 0x6ed9eba1, w6 @@ -116,15 +116,11 @@ CPU_LE( rev32 v11.16b, v11.16b ) add dgbv.2s, dgbv.2s, dg1v.2s add dgav.4s, dgav.4s, dg0v.4s - /* return early if voluntary preemption is needed */ - cond_yield 1f, x5, x6 - /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -1: st1 {dgav.4s}, [x0] + st1 {dgav.4s}, [x0] str dgb, [x0, #16] - mov x0, x2 ret -SYM_FUNC_END(__sha1_ce_transform) +SYM_FUNC_END(sha1_ce_transform) diff --git a/lib/crypto/arm64/sha1.h b/lib/crypto/arm64/sha1.h index bc7071f1be09..112c5d443c56 100644 --- a/lib/crypto/arm64/sha1.h +++ b/lib/crypto/arm64/sha1.h @@ -9,22 +9,15 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce); -asmlinkage size_t __sha1_ce_transform(struct sha1_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha1_ce_transform(struct sha1_block_state *state, + const u8 *data, size_t nblocks); static void sha1_blocks(struct sha1_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_ce) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha1_ce_transform(state, data, nblocks); - - data += (nblocks - rem) * SHA1_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha1_ce_transform(state, data, nblocks); } else { sha1_blocks_generic(state, data, nblocks); } From fe1233c2eb69c040f737dd10b881efff5f4ccee0 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 31 Mar 2026 17:05:45 -0700 Subject: [PATCH 62/66] lib/crypto: arm64/sha256: Remove obsolete chunking logic Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the SHA-256 code accordingly. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260401000548.133151-7-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm64/sha256-ce.S | 14 +++++--------- lib/crypto/arm64/sha256.h | 29 ++++++++--------------------- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/lib/crypto/arm64/sha256-ce.S b/lib/crypto/arm64/sha256-ce.S index e4bfe42a61a9..b54ad977afa3 100644 --- a/lib/crypto/arm64/sha256-ce.S +++ b/lib/crypto/arm64/sha256-ce.S @@ -79,11 +79,11 @@ .endm /* - * size_t __sha256_ce_transform(struct sha256_block_state *state, - * const u8 *data, size_t nblocks); + * void sha256_ce_transform(struct sha256_block_state *state, + * const u8 *data, size_t nblocks); */ .text -SYM_FUNC_START(__sha256_ce_transform) +SYM_FUNC_START(sha256_ce_transform) load_round_constants x8 @@ -127,17 +127,13 @@ CPU_LE( rev32 v19.16b, v19.16b ) add dgav.4s, dgav.4s, dg0v.4s add dgbv.4s, dgbv.4s, dg1v.4s - /* return early if voluntary preemption is needed */ - cond_yield 1f, x5, x6 - /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -1: st1 {dgav.4s, dgbv.4s}, [x0] - mov x0, x2 + st1 {dgav.4s, dgbv.4s}, [x0] ret -SYM_FUNC_END(__sha256_ce_transform) +SYM_FUNC_END(sha256_ce_transform) .unreq dga .unreq dgav diff --git a/lib/crypto/arm64/sha256.h b/lib/crypto/arm64/sha256.h index 1fad3d7baa9a..b4353d3c4dd0 100644 --- a/lib/crypto/arm64/sha256.h +++ b/lib/crypto/arm64/sha256.h @@ -14,26 +14,17 @@ asmlinkage void sha256_block_data_order(struct sha256_block_state *state, const u8 *data, size_t nblocks); asmlinkage void sha256_block_neon(struct sha256_block_state *state, const u8 *data, size_t nblocks); -asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha256_ce_transform(struct sha256_block_state *state, + const u8 *data, size_t nblocks); static void sha256_blocks(struct sha256_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_neon) && likely(may_use_simd())) { - if (static_branch_likely(&have_ce)) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha256_ce_transform(state, data, - nblocks); - - data += (nblocks - rem) * SHA256_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); - } else { - scoped_ksimd() + scoped_ksimd() { + if (static_branch_likely(&have_ce)) + sha256_ce_transform(state, data, nblocks); + else sha256_block_neon(state, data, nblocks); } } else { @@ -55,13 +46,9 @@ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx, u8 out1[SHA256_DIGEST_SIZE], u8 out2[SHA256_DIGEST_SIZE]) { - /* - * The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX. - * Further limit len to 65536 to avoid spending too long with preemption - * disabled. (Of course, in practice len is nearly always 4096 anyway.) - */ + /* The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX. */ if (static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE && - len <= 65536 && likely(may_use_simd())) { + len <= INT_MAX && likely(may_use_simd())) { scoped_ksimd() sha256_ce_finup2x(ctx, data1, data2, len, out1, out2); kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE); From 7116418f6b00faf43e56f0e052b968b04fc75989 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 31 Mar 2026 17:05:46 -0700 Subject: [PATCH 63/66] lib/crypto: arm64/sha512: Remove obsolete chunking logic Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the SHA-512 code accordingly. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260401000548.133151-8-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm64/sha512-ce-core.S | 12 +++++------- lib/crypto/arm64/sha512.h | 15 ++++----------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/lib/crypto/arm64/sha512-ce-core.S b/lib/crypto/arm64/sha512-ce-core.S index ffd51acfd1ee..26834921e8d6 100644 --- a/lib/crypto/arm64/sha512-ce-core.S +++ b/lib/crypto/arm64/sha512-ce-core.S @@ -93,11 +93,11 @@ .endm /* - * size_t __sha512_ce_transform(struct sha512_block_state *state, - * const u8 *data, size_t nblocks); + * void sha512_ce_transform(struct sha512_block_state *state, + * const u8 *data, size_t nblocks); */ .text -SYM_FUNC_START(__sha512_ce_transform) +SYM_FUNC_START(sha512_ce_transform) /* load state */ ld1 {v8.2d-v11.2d}, [x0] @@ -186,12 +186,10 @@ CPU_LE( rev64 v19.16b, v19.16b ) add v10.2d, v10.2d, v2.2d add v11.2d, v11.2d, v3.2d - cond_yield 3f, x4, x5 /* handled all input blocks? */ cbnz x2, 0b /* store new state */ -3: st1 {v8.2d-v11.2d}, [x0] - mov x0, x2 + st1 {v8.2d-v11.2d}, [x0] ret -SYM_FUNC_END(__sha512_ce_transform) +SYM_FUNC_END(sha512_ce_transform) diff --git a/lib/crypto/arm64/sha512.h b/lib/crypto/arm64/sha512.h index d978c4d07e90..5da27e6e23ea 100644 --- a/lib/crypto/arm64/sha512.h +++ b/lib/crypto/arm64/sha512.h @@ -12,23 +12,16 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha512_insns); asmlinkage void sha512_block_data_order(struct sha512_block_state *state, const u8 *data, size_t nblocks); -asmlinkage size_t __sha512_ce_transform(struct sha512_block_state *state, - const u8 *data, size_t nblocks); +asmlinkage void sha512_ce_transform(struct sha512_block_state *state, + const u8 *data, size_t nblocks); static void sha512_blocks(struct sha512_block_state *state, const u8 *data, size_t nblocks) { if (static_branch_likely(&have_sha512_insns) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = __sha512_ce_transform(state, data, nblocks); - - data += (nblocks - rem) * SHA512_BLOCK_SIZE; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha512_ce_transform(state, data, nblocks); } else { sha512_block_data_order(state, data, nblocks); } From 6d575f11c70b0ceff7db47813ebb7aec09e8d01f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 31 Mar 2026 17:05:47 -0700 Subject: [PATCH 64/66] lib/crypto: arm64/sha3: Remove obsolete chunking logic Since commit aefbab8e77eb ("arm64: fpsimd: Preserve/restore kernel mode NEON at context switch"), kernel-mode NEON sections have been preemptible on arm64. And since commit 7dadeaa6e851 ("sched: Further restrict the preemption modes"), voluntary preemption is no longer supported on arm64 either. Therefore, there's no longer any need to limit the length of kernel-mode NEON sections on arm64. Simplify the SHA-3 code accordingly. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260401000548.133151-9-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm64/sha3-ce-core.S | 8 +++----- lib/crypto/arm64/sha3.h | 15 ++++----------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/lib/crypto/arm64/sha3-ce-core.S b/lib/crypto/arm64/sha3-ce-core.S index ace90b506490..b8ab01987ae0 100644 --- a/lib/crypto/arm64/sha3-ce-core.S +++ b/lib/crypto/arm64/sha3-ce-core.S @@ -37,8 +37,8 @@ .endm /* - * size_t sha3_ce_transform(struct sha3_state *state, const u8 *data, - * size_t nblocks, size_t block_size) + * void sha3_ce_transform(struct sha3_state *state, const u8 *data, + * size_t nblocks, size_t block_size) * * block_size is assumed to be one of 72 (SHA3-512), 104 (SHA3-384), 136 * (SHA3-256 and SHAKE256), 144 (SHA3-224), or 168 (SHAKE128). @@ -185,18 +185,16 @@ SYM_FUNC_START(sha3_ce_transform) eor v0.16b, v0.16b, v31.16b cbnz w8, 3b - cond_yield 4f, x8, x9 cbnz x2, 0b /* save state */ -4: st1 { v0.1d- v3.1d}, [x0], #32 + st1 { v0.1d- v3.1d}, [x0], #32 st1 { v4.1d- v7.1d}, [x0], #32 st1 { v8.1d-v11.1d}, [x0], #32 st1 {v12.1d-v15.1d}, [x0], #32 st1 {v16.1d-v19.1d}, [x0], #32 st1 {v20.1d-v23.1d}, [x0], #32 st1 {v24.1d}, [x0] - mov x0, x2 ret SYM_FUNC_END(sha3_ce_transform) diff --git a/lib/crypto/arm64/sha3.h b/lib/crypto/arm64/sha3.h index b602f1b3b282..eaaba3224acc 100644 --- a/lib/crypto/arm64/sha3.h +++ b/lib/crypto/arm64/sha3.h @@ -12,22 +12,15 @@ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3); -asmlinkage size_t sha3_ce_transform(struct sha3_state *state, const u8 *data, - size_t nblocks, size_t block_size); +asmlinkage void sha3_ce_transform(struct sha3_state *state, const u8 *data, + size_t nblocks, size_t block_size); static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data, size_t nblocks, size_t block_size) { if (static_branch_likely(&have_sha3) && likely(may_use_simd())) { - do { - size_t rem; - - scoped_ksimd() - rem = sha3_ce_transform(state, data, nblocks, - block_size); - data += (nblocks - rem) * block_size; - nblocks = rem; - } while (nblocks); + scoped_ksimd() + sha3_ce_transform(state, data, nblocks, block_size); } else { sha3_absorb_blocks_generic(state, data, nblocks, block_size); } From 180e92df9a4f7498d6549c8c23839bd9554b3449 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 31 Mar 2026 17:05:48 -0700 Subject: [PATCH 65/66] arm64: fpsimd: Remove obsolete cond_yield macro All invocations of the cond_yield macro have been removed, so remove the macro definition as well. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260401000548.133151-10-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/arm64/include/asm/assembler.h | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index d3d46e5f7188..9d7c9ae5ac96 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -748,28 +748,6 @@ alternative_else_nop_endif set_sctlr sctlr_el2, \reg .endm - /* - * Check whether asm code should yield as soon as it is able. This is - * the case if we are currently running in task context, and the - * TIF_NEED_RESCHED flag is set. (Note that the TIF_NEED_RESCHED flag - * is stored negated in the top word of the thread_info::preempt_count - * field) - */ - .macro cond_yield, lbl:req, tmp:req, tmp2 -#ifdef CONFIG_PREEMPT_VOLUNTARY - get_current_task \tmp - ldr \tmp, [\tmp, #TSK_TI_PREEMPT] - /* - * If we are serving a softirq, there is no point in yielding: the - * softirq will not be preempted no matter what we do, so we should - * run to completion as quickly as we can. The preempt_count field will - * have BIT(SOFTIRQ_SHIFT) set in this case, so the zero check will - * catch this case too. - */ - cbz \tmp, \lbl -#endif - .endm - /* * Branch Target Identifier (BTI) */ From 12b11e47f126d097839fd2f077636e2139b0151b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 31 Mar 2026 17:33:31 -0700 Subject: [PATCH 66/66] lib/crypto: arm64: Assume a little-endian kernel Since support for big-endian arm64 kernels was removed, the CPU_LE() macro now unconditionally emits the code it is passed, and the CPU_BE() macro now unconditionally discards the code it is passed. Simplify the assembly code in lib/crypto/arm64/ accordingly. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260401003331.144065-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- lib/crypto/arm64/aes-cipher-core.S | 10 ------- lib/crypto/arm64/chacha-neon-core.S | 16 ----------- lib/crypto/arm64/ghash-neon-core.S | 2 +- lib/crypto/arm64/sha1-ce-core.S | 8 +++--- lib/crypto/arm64/sha256-ce.S | 41 +++++++++++++---------------- lib/crypto/arm64/sha512-ce-core.S | 16 +++++------ lib/crypto/arm64/sm3-ce-core.S | 8 +++--- 7 files changed, 36 insertions(+), 65 deletions(-) diff --git a/lib/crypto/arm64/aes-cipher-core.S b/lib/crypto/arm64/aes-cipher-core.S index 651f701c56a8..0b05ec4be65f 100644 --- a/lib/crypto/arm64/aes-cipher-core.S +++ b/lib/crypto/arm64/aes-cipher-core.S @@ -87,11 +87,6 @@ ldp w8, w9, [rk], #16 ldp w10, w11, [rk, #-8] -CPU_BE( rev w4, w4 ) -CPU_BE( rev w5, w5 ) -CPU_BE( rev w6, w6 ) -CPU_BE( rev w7, w7 ) - eor w4, w4, w8 eor w5, w5, w9 eor w6, w6, w10 @@ -112,11 +107,6 @@ CPU_BE( rev w7, w7 ) 3: adr_l tt, \ltab \round w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b -CPU_BE( rev w4, w4 ) -CPU_BE( rev w5, w5 ) -CPU_BE( rev w6, w6 ) -CPU_BE( rev w7, w7 ) - stp w4, w5, [out] stp w6, w7, [out, #8] ret diff --git a/lib/crypto/arm64/chacha-neon-core.S b/lib/crypto/arm64/chacha-neon-core.S index 80079586ecc7..cb18eec968bd 100644 --- a/lib/crypto/arm64/chacha-neon-core.S +++ b/lib/crypto/arm64/chacha-neon-core.S @@ -531,10 +531,6 @@ SYM_FUNC_START(chacha_4block_xor_neon) add v3.4s, v3.4s, v19.4s add a2, a2, w8 add a3, a3, w9 -CPU_BE( rev a0, a0 ) -CPU_BE( rev a1, a1 ) -CPU_BE( rev a2, a2 ) -CPU_BE( rev a3, a3 ) ld4r {v24.4s-v27.4s}, [x0], #16 ld4r {v28.4s-v31.4s}, [x0] @@ -555,10 +551,6 @@ CPU_BE( rev a3, a3 ) add v7.4s, v7.4s, v23.4s add a6, a6, w8 add a7, a7, w9 -CPU_BE( rev a4, a4 ) -CPU_BE( rev a5, a5 ) -CPU_BE( rev a6, a6 ) -CPU_BE( rev a7, a7 ) // x8[0-3] += s2[0] // x9[0-3] += s2[1] @@ -576,10 +568,6 @@ CPU_BE( rev a7, a7 ) add v11.4s, v11.4s, v27.4s add a10, a10, w8 add a11, a11, w9 -CPU_BE( rev a8, a8 ) -CPU_BE( rev a9, a9 ) -CPU_BE( rev a10, a10 ) -CPU_BE( rev a11, a11 ) // x12[0-3] += s3[0] // x13[0-3] += s3[1] @@ -597,10 +585,6 @@ CPU_BE( rev a11, a11 ) add v15.4s, v15.4s, v31.4s add a14, a14, w8 add a15, a15, w9 -CPU_BE( rev a12, a12 ) -CPU_BE( rev a13, a13 ) -CPU_BE( rev a14, a14 ) -CPU_BE( rev a15, a15 ) // interleave 32-bit words in state n, n+1 ldp w6, w7, [x2], #64 diff --git a/lib/crypto/arm64/ghash-neon-core.S b/lib/crypto/arm64/ghash-neon-core.S index 85b20fcd98fe..4c5799172b49 100644 --- a/lib/crypto/arm64/ghash-neon-core.S +++ b/lib/crypto/arm64/ghash-neon-core.S @@ -192,7 +192,7 @@ SYM_FUNC_START(pmull_ghash_update_p8) sub x0, x0, #1 /* multiply XL by SHASH in GF(2^128) */ -CPU_LE( rev64 T1.16b, T1.16b ) + rev64 T1.16b, T1.16b ext T2.16b, XL.16b, XL.16b, #8 ext IN1.16b, T1.16b, T1.16b, #8 diff --git a/lib/crypto/arm64/sha1-ce-core.S b/lib/crypto/arm64/sha1-ce-core.S index 59d27fda0714..40f2a6c8d0c5 100644 --- a/lib/crypto/arm64/sha1-ce-core.S +++ b/lib/crypto/arm64/sha1-ce-core.S @@ -80,10 +80,10 @@ SYM_FUNC_START(sha1_ce_transform) 0: ld1 {v8.4s-v11.4s}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev32 v8.16b, v8.16b ) -CPU_LE( rev32 v9.16b, v9.16b ) -CPU_LE( rev32 v10.16b, v10.16b ) -CPU_LE( rev32 v11.16b, v11.16b ) + rev32 v8.16b, v8.16b + rev32 v9.16b, v9.16b + rev32 v10.16b, v10.16b + rev32 v11.16b, v11.16b add t0.4s, v8.4s, k0.4s mov dg0v.16b, dgav.16b diff --git a/lib/crypto/arm64/sha256-ce.S b/lib/crypto/arm64/sha256-ce.S index b54ad977afa3..8fdbf0a9ff89 100644 --- a/lib/crypto/arm64/sha256-ce.S +++ b/lib/crypto/arm64/sha256-ce.S @@ -94,10 +94,10 @@ SYM_FUNC_START(sha256_ce_transform) 0: ld1 {v16.4s-v19.4s}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev32 v16.16b, v16.16b ) -CPU_LE( rev32 v17.16b, v17.16b ) -CPU_LE( rev32 v18.16b, v18.16b ) -CPU_LE( rev32 v19.16b, v19.16b ) + rev32 v16.16b, v16.16b + rev32 v17.16b, v17.16b + rev32 v18.16b, v18.16b + rev32 v19.16b, v19.16b add t0.4s, v16.4s, v0.4s mov dg0v.16b, dgav.16b @@ -289,14 +289,14 @@ SYM_FUNC_START(sha256_ce_finup2x) ld1 {v20.4s-v23.4s}, [data2], #64 .Lfinup2x_loop_have_data: // Convert the words of the data blocks from big endian. -CPU_LE( rev32 v16.16b, v16.16b ) -CPU_LE( rev32 v17.16b, v17.16b ) -CPU_LE( rev32 v18.16b, v18.16b ) -CPU_LE( rev32 v19.16b, v19.16b ) -CPU_LE( rev32 v20.16b, v20.16b ) -CPU_LE( rev32 v21.16b, v21.16b ) -CPU_LE( rev32 v22.16b, v22.16b ) -CPU_LE( rev32 v23.16b, v23.16b ) + rev32 v16.16b, v16.16b + rev32 v17.16b, v17.16b + rev32 v18.16b, v18.16b + rev32 v19.16b, v19.16b + rev32 v20.16b, v20.16b + rev32 v21.16b, v21.16b + rev32 v22.16b, v22.16b + rev32 v23.16b, v23.16b .Lfinup2x_loop_have_bswapped_data: // Save the original state for each block. @@ -336,11 +336,8 @@ CPU_LE( rev32 v23.16b, v23.16b ) sub w8, len, #64 // w8 = len - 64 add data1, data1, w8, sxtw // data1 += len - 64 add data2, data2, w8, sxtw // data2 += len - 64 -CPU_LE( mov x9, #0x80 ) -CPU_LE( fmov d16, x9 ) -CPU_BE( movi v16.16b, #0 ) -CPU_BE( mov x9, #0x8000000000000000 ) -CPU_BE( mov v16.d[1], x9 ) + mov x9, #0x80 + fmov d16, x9 movi v17.16b, #0 stp q16, q17, [sp, #64] stp q17, q17, [sp, #96] @@ -348,7 +345,7 @@ CPU_BE( mov v16.d[1], x9 ) cmp len, #56 b.ge 1f // will count spill into its own block? lsl count, count, #3 -CPU_LE( rev count, count ) + rev count, count str count, [x9, #56] mov final_step, #2 // won't need count-only block b 2f @@ -393,10 +390,10 @@ CPU_LE( rev count, count ) .Lfinup2x_done: // Write the two digests with all bytes in the correct order. -CPU_LE( rev32 state0_a.16b, state0_a.16b ) -CPU_LE( rev32 state1_a.16b, state1_a.16b ) -CPU_LE( rev32 state0_b.16b, state0_b.16b ) -CPU_LE( rev32 state1_b.16b, state1_b.16b ) + rev32 state0_a.16b, state0_a.16b + rev32 state1_a.16b, state1_a.16b + rev32 state0_b.16b, state0_b.16b + rev32 state1_b.16b, state1_b.16b st1 {state0_a.4s-state1_a.4s}, [out1] st1 {state0_b.4s-state1_b.4s}, [out2] add sp, sp, #128 diff --git a/lib/crypto/arm64/sha512-ce-core.S b/lib/crypto/arm64/sha512-ce-core.S index 26834921e8d6..daa2c63a20c0 100644 --- a/lib/crypto/arm64/sha512-ce-core.S +++ b/lib/crypto/arm64/sha512-ce-core.S @@ -110,14 +110,14 @@ SYM_FUNC_START(sha512_ce_transform) ld1 {v16.2d-v19.2d}, [x1], #64 sub x2, x2, #1 -CPU_LE( rev64 v12.16b, v12.16b ) -CPU_LE( rev64 v13.16b, v13.16b ) -CPU_LE( rev64 v14.16b, v14.16b ) -CPU_LE( rev64 v15.16b, v15.16b ) -CPU_LE( rev64 v16.16b, v16.16b ) -CPU_LE( rev64 v17.16b, v17.16b ) -CPU_LE( rev64 v18.16b, v18.16b ) -CPU_LE( rev64 v19.16b, v19.16b ) + rev64 v12.16b, v12.16b + rev64 v13.16b, v13.16b + rev64 v14.16b, v14.16b + rev64 v15.16b, v15.16b + rev64 v16.16b, v16.16b + rev64 v17.16b, v17.16b + rev64 v18.16b, v18.16b + rev64 v19.16b, v19.16b mov x4, x3 // rc pointer diff --git a/lib/crypto/arm64/sm3-ce-core.S b/lib/crypto/arm64/sm3-ce-core.S index 9cef7ea7f34f..ee7f900d7cff 100644 --- a/lib/crypto/arm64/sm3-ce-core.S +++ b/lib/crypto/arm64/sm3-ce-core.S @@ -91,10 +91,10 @@ SYM_FUNC_START(sm3_ce_transform) mov v15.16b, v8.16b mov v16.16b, v9.16b -CPU_LE( rev32 v0.16b, v0.16b ) -CPU_LE( rev32 v1.16b, v1.16b ) -CPU_LE( rev32 v2.16b, v2.16b ) -CPU_LE( rev32 v3.16b, v3.16b ) + rev32 v0.16b, v0.16b + rev32 v1.16b, v1.16b + rev32 v2.16b, v2.16b + rev32 v3.16b, v3.16b ext v11.16b, v13.16b, v13.16b, #4