lib/crypto: x86/sm3: Migrate optimized code into library
Instead of exposing the x86-optimized SM3 code via an x86-specific crypto_shash algorithm, instead just implement the sm3_blocks() library function. This is much simpler, it makes the SM3 library functions be x86-optimized, and it fixes the longstanding issue where the x86-optimized SM3 code was disabled by default. SM3 still remains available through crypto_shash, but individual architectures no longer need to handle it. Tweak the prototype of sm3_transform_avx() to match what the library expects, including changing the block count to size_t. Note that the assembly code actually already treated this argument as size_t. Acked-by: Ard Biesheuvel <ardb@kernel.org> Link: https://lore.kernel.org/r/20260321040935.410034-10-ebiggers@kernel.org Signed-off-by: Eric Biggers <ebiggers@kernel.org>master
Eric Biggers
parent
5f6bbba5e9
commit
17ba6108d3
|
|
@ -331,17 +331,4 @@ config CRYPTO_AEGIS128_AESNI_SSE2
|
|||
- AES-NI (AES New Instructions)
|
||||
- SSE4.1 (Streaming SIMD Extensions 4.1)
|
||||
|
||||
config CRYPTO_SM3_AVX_X86_64
|
||||
tristate "Hash functions: SM3 (AVX)"
|
||||
depends on 64BIT
|
||||
select CRYPTO_HASH
|
||||
select CRYPTO_LIB_SM3
|
||||
help
|
||||
SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3
|
||||
|
||||
Architecture: x86_64 using:
|
||||
- AVX (Advanced Vector Extensions)
|
||||
|
||||
If unsure, say N.
|
||||
|
||||
endmenu
|
||||
|
|
|
|||
|
|
@ -50,9 +50,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \
|
|||
aes-gcm-vaes-avx512.o \
|
||||
aes-xts-avx-x86_64.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o
|
||||
sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX_X86_64) += sm4-aesni-avx-x86_64.o
|
||||
sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o
|
||||
|
||||
|
|
|
|||
|
|
@ -1,100 +0,0 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
/*
|
||||
* SM3 Secure Hash Algorithm, AVX assembler accelerated.
|
||||
* specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02
|
||||
*
|
||||
* Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
|
||||
*/
|
||||
|
||||
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
||||
|
||||
#include <crypto/internal/hash.h>
|
||||
#include <crypto/internal/simd.h>
|
||||
#include <crypto/sm3.h>
|
||||
#include <crypto/sm3_base.h>
|
||||
#include <linux/cpufeature.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
|
||||
asmlinkage void sm3_transform_avx(struct sm3_state *state,
|
||||
const u8 *data, int nblocks);
|
||||
|
||||
static int sm3_avx_update(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len)
|
||||
{
|
||||
int remain;
|
||||
|
||||
/*
|
||||
* Make sure struct sm3_state begins directly with the SM3
|
||||
* 256-bit internal state, as this is what the asm functions expect.
|
||||
*/
|
||||
BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0);
|
||||
|
||||
kernel_fpu_begin();
|
||||
remain = sm3_base_do_update_blocks(desc, data, len, sm3_transform_avx);
|
||||
kernel_fpu_end();
|
||||
return remain;
|
||||
}
|
||||
|
||||
static int sm3_avx_finup(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len, u8 *out)
|
||||
{
|
||||
kernel_fpu_begin();
|
||||
sm3_base_do_finup(desc, data, len, sm3_transform_avx);
|
||||
kernel_fpu_end();
|
||||
return sm3_base_finish(desc, out);
|
||||
}
|
||||
|
||||
static struct shash_alg sm3_avx_alg = {
|
||||
.digestsize = SM3_DIGEST_SIZE,
|
||||
.init = sm3_base_init,
|
||||
.update = sm3_avx_update,
|
||||
.finup = sm3_avx_finup,
|
||||
.descsize = SM3_STATE_SIZE,
|
||||
.base = {
|
||||
.cra_name = "sm3",
|
||||
.cra_driver_name = "sm3-avx",
|
||||
.cra_priority = 300,
|
||||
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
|
||||
CRYPTO_AHASH_ALG_FINUP_MAX,
|
||||
.cra_blocksize = SM3_BLOCK_SIZE,
|
||||
.cra_module = THIS_MODULE,
|
||||
}
|
||||
};
|
||||
|
||||
static int __init sm3_avx_mod_init(void)
|
||||
{
|
||||
const char *feature_name;
|
||||
|
||||
if (!boot_cpu_has(X86_FEATURE_AVX)) {
|
||||
pr_info("AVX instruction are not detected.\n");
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
if (!boot_cpu_has(X86_FEATURE_BMI2)) {
|
||||
pr_info("BMI2 instruction are not detected.\n");
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
|
||||
&feature_name)) {
|
||||
pr_info("CPU feature '%s' is not supported.\n", feature_name);
|
||||
return -ENODEV;
|
||||
}
|
||||
|
||||
return crypto_register_shash(&sm3_avx_alg);
|
||||
}
|
||||
|
||||
static void __exit sm3_avx_mod_exit(void)
|
||||
{
|
||||
crypto_unregister_shash(&sm3_avx_alg);
|
||||
}
|
||||
|
||||
module_init(sm3_avx_mod_init);
|
||||
module_exit(sm3_avx_mod_exit);
|
||||
|
||||
MODULE_LICENSE("GPL v2");
|
||||
MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
|
||||
MODULE_DESCRIPTION("SM3 Secure Hash Algorithm, AVX assembler accelerated");
|
||||
MODULE_ALIAS_CRYPTO("sm3");
|
||||
MODULE_ALIAS_CRYPTO("sm3-avx");
|
||||
|
|
@ -282,6 +282,7 @@ config CRYPTO_LIB_SM3_ARCH
|
|||
default y if ARM64
|
||||
default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
|
||||
RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
|
||||
default y if X86_64
|
||||
|
||||
source "lib/crypto/tests/Kconfig"
|
||||
|
||||
|
|
|
|||
|
|
@ -375,6 +375,7 @@ CFLAGS_sm3.o += -I$(src)/$(SRCARCH)
|
|||
libsm3-$(CONFIG_ARM64) += arm64/sm3-ce-core.o \
|
||||
arm64/sm3-neon-core.o
|
||||
libsm3-$(CONFIG_RISCV) += riscv/sm3-riscv64-zvksh-zvkb.o
|
||||
libsm3-$(CONFIG_X86) += x86/sm3-avx-asm_64.o
|
||||
endif # CONFIG_CRYPTO_LIB_SM3_ARCH
|
||||
|
||||
################################################################################
|
||||
|
|
|
|||
|
|
@ -12,10 +12,9 @@
|
|||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/cfi_types.h>
|
||||
#include <asm/frame.h>
|
||||
|
||||
/* Context structure */
|
||||
/* State structure */
|
||||
|
||||
#define state_h0 0
|
||||
#define state_h1 4
|
||||
|
|
@ -325,13 +324,13 @@
|
|||
/*
|
||||
* Transform nblocks*64 bytes (nblocks*16 32-bit words) at DATA.
|
||||
*
|
||||
* void sm3_transform_avx(struct sm3_state *state,
|
||||
* const u8 *data, int nblocks);
|
||||
* void sm3_transform_avx(struct sm3_block_state *state,
|
||||
* const u8 *data, size_t nblocks);
|
||||
*/
|
||||
SYM_TYPED_FUNC_START(sm3_transform_avx)
|
||||
SYM_FUNC_START(sm3_transform_avx)
|
||||
/* input:
|
||||
* %rdi: ctx, CTX
|
||||
* %rsi: data (64*nblks bytes)
|
||||
* %rdi: state
|
||||
* %rsi: data
|
||||
* %rdx: nblocks
|
||||
*/
|
||||
vzeroupper;
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-or-later */
|
||||
/*
|
||||
* SM3 optimized for x86_64
|
||||
*
|
||||
* Copyright 2026 Google LLC
|
||||
*/
|
||||
#include <asm/fpu/api.h>
|
||||
#include <linux/static_call.h>
|
||||
|
||||
asmlinkage void sm3_transform_avx(struct sm3_block_state *state,
|
||||
const u8 *data, size_t nblocks);
|
||||
|
||||
static void sm3_blocks_avx(struct sm3_block_state *state,
|
||||
const u8 *data, size_t nblocks)
|
||||
{
|
||||
if (likely(irq_fpu_usable())) {
|
||||
kernel_fpu_begin();
|
||||
sm3_transform_avx(state, data, nblocks);
|
||||
kernel_fpu_end();
|
||||
} else {
|
||||
sm3_blocks_generic(state, data, nblocks);
|
||||
}
|
||||
}
|
||||
|
||||
DEFINE_STATIC_CALL(sm3_blocks_x86, sm3_blocks_generic);
|
||||
|
||||
static void sm3_blocks(struct sm3_block_state *state,
|
||||
const u8 *data, size_t nblocks)
|
||||
{
|
||||
static_call(sm3_blocks_x86)(state, data, nblocks);
|
||||
}
|
||||
|
||||
#define sm3_mod_init_arch sm3_mod_init_arch
|
||||
static void sm3_mod_init_arch(void)
|
||||
{
|
||||
if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_BMI2) &&
|
||||
cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
|
||||
static_call_update(sm3_blocks_x86, sm3_blocks_avx);
|
||||
}
|
||||
Loading…
Reference in New Issue